Add retry loop for fetching authentication token if any 'Internal Fai…

…lure' occurs (#368) * Add retry loop for fetching authentication token if any 'Internal failure' occurs This is to prevent errors while client tries to connect with server and fails with 'Internal Failure' because it was unable to fetch OAuth2 token. * Add retry loop for fetching authentication token if any 'Internal failure' occurs This is to prevent errors while client tries to connect with server and fails with 'Internal Failure' because it was unable to fetch OAuth2 token.
googleapis · Sep 9, 2019 · eae1dcb · eae1dcb
1 parent 7a8641a
commit eae1dcb
Show file tree

Hide file tree

Showing 22 changed files with 106 additions and 90 deletions.
diff --git a/docs/reference/google.auth.app_engine.rst b/docs/reference/google.auth.app_engine.rst
@@ -1,7 +1,7 @@
-google.auth.app_engine module
-=============================
+google.auth.app\_engine module
+==============================
 
 .. automodule:: google.auth.app_engine
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
diff --git a/docs/reference/google.auth.compute_engine.credentials.rst b/docs/reference/google.auth.compute_engine.credentials.rst
@@ -1,7 +1,7 @@
-google.auth.compute_engine.credentials module
-=============================================
+google.auth.compute\_engine.credentials module
+==============================================
 
 .. automodule:: google.auth.compute_engine.credentials
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
diff --git a/docs/reference/google.auth.compute_engine.rst b/docs/reference/google.auth.compute_engine.rst
@@ -1,15 +1,14 @@
-google.auth.compute_engine package
-==================================
+google.auth.compute\_engine package
+===================================
 
 .. automodule:: google.auth.compute_engine
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
 
 Submodules
 ----------
 
 .. toctree::
 
    google.auth.compute_engine.credentials
-
diff --git a/docs/reference/google.auth.credentials.rst b/docs/reference/google.auth.credentials.rst
@@ -2,6 +2,6 @@ google.auth.credentials module
 ==============================
 
 .. automodule:: google.auth.credentials
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
diff --git a/docs/reference/google.auth.crypt.rst b/docs/reference/google.auth.crypt.rst
@@ -2,9 +2,9 @@ google.auth.crypt package
 =========================
 
 .. automodule:: google.auth.crypt
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
 
 Submodules
 ----------
@@ -13,4 +13,3 @@ Submodules
 
    google.auth.crypt.base
    google.auth.crypt.rsa
-
diff --git a/docs/reference/google.auth.environment_vars.rst b/docs/reference/google.auth.environment_vars.rst
@@ -1,7 +1,7 @@
-google.auth.environment_vars module
-===================================
+google.auth.environment\_vars module
+====================================
 
 .. automodule:: google.auth.environment_vars
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
diff --git a/docs/reference/google.auth.exceptions.rst b/docs/reference/google.auth.exceptions.rst
@@ -2,6 +2,6 @@ google.auth.exceptions module
 =============================
 
 .. automodule:: google.auth.exceptions
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
diff --git a/docs/reference/google.auth.iam.rst b/docs/reference/google.auth.iam.rst
@@ -2,6 +2,6 @@ google.auth.iam module
 ======================
 
 .. automodule:: google.auth.iam
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
diff --git a/docs/reference/google.auth.impersonated_credentials.rst b/docs/reference/google.auth.impersonated_credentials.rst
@@ -2,6 +2,6 @@ google.auth.impersonated\_credentials module
 ============================================
 
 .. automodule:: google.auth.impersonated_credentials
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
diff --git a/docs/reference/google.auth.jwt.rst b/docs/reference/google.auth.jwt.rst
@@ -2,6 +2,6 @@ google.auth.jwt module
 ======================
 
 .. automodule:: google.auth.jwt
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
diff --git a/docs/reference/google.auth.rst b/docs/reference/google.auth.rst
@@ -2,18 +2,18 @@ google.auth package
 ===================
 
 .. automodule:: google.auth
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
 
 Subpackages
 -----------
 
 .. toctree::
 
-    google.auth.compute_engine
-    google.auth.crypt
-    google.auth.transport
+   google.auth.compute_engine
+   google.auth.crypt
+   google.auth.transport
 
 Submodules
 ----------
@@ -27,4 +27,3 @@ Submodules
    google.auth.iam
    google.auth.impersonated_credentials
    google.auth.jwt
-
diff --git a/docs/reference/google.auth.transport.grpc.rst b/docs/reference/google.auth.transport.grpc.rst
@@ -2,6 +2,6 @@ google.auth.transport.grpc module
 =================================
 
 .. automodule:: google.auth.transport.grpc
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
diff --git a/docs/reference/google.auth.transport.requests.rst b/docs/reference/google.auth.transport.requests.rst
@@ -2,6 +2,6 @@ google.auth.transport.requests module
 =====================================
 
 .. automodule:: google.auth.transport.requests
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
diff --git a/docs/reference/google.auth.transport.rst b/docs/reference/google.auth.transport.rst
@@ -2,9 +2,9 @@ google.auth.transport package
 =============================
 
 .. automodule:: google.auth.transport
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
 
 Submodules
 ----------
@@ -14,4 +14,3 @@ Submodules
    google.auth.transport.grpc
    google.auth.transport.requests
    google.auth.transport.urllib3
-
diff --git a/docs/reference/google.auth.transport.urllib3.rst b/docs/reference/google.auth.transport.urllib3.rst
@@ -2,6 +2,6 @@ google.auth.transport.urllib3 module
 ====================================
 
 .. automodule:: google.auth.transport.urllib3
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
diff --git a/docs/reference/google.oauth2.credentials.rst b/docs/reference/google.oauth2.credentials.rst
@@ -2,6 +2,6 @@ google.oauth2.credentials module
 ================================
 
 .. automodule:: google.oauth2.credentials
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
diff --git a/docs/reference/google.oauth2.id_token.rst b/docs/reference/google.oauth2.id_token.rst
@@ -1,7 +1,7 @@
-google.oauth2.id_token module
-=============================
+google.oauth2.id\_token module
+==============================
 
 .. automodule:: google.oauth2.id_token
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
diff --git a/docs/reference/google.oauth2.rst b/docs/reference/google.oauth2.rst
@@ -2,9 +2,9 @@ google.oauth2 package
 =====================
 
 .. automodule:: google.oauth2
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
 
 Submodules
 ----------
@@ -14,4 +14,3 @@ Submodules
    google.oauth2.credentials
    google.oauth2.id_token
    google.oauth2.service_account
-
diff --git a/docs/reference/google.oauth2.service_account.rst b/docs/reference/google.oauth2.service_account.rst
@@ -1,7 +1,7 @@
-google.oauth2.service_account module
-====================================
+google.oauth2.service\_account module
+=====================================
 
 .. automodule:: google.oauth2.service_account
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
diff --git a/docs/reference/google.rst b/docs/reference/google.rst
@@ -2,15 +2,14 @@ google package
 ==============
 
 .. automodule:: google
-    :members:
-    :inherited-members:
-    :show-inheritance:
+   :members:
+   :inherited-members:
+   :show-inheritance:
 
 Subpackages
 -----------
 
 .. toctree::
 
-    google.auth
-    google.oauth2
-
+   google.auth
+   google.oauth2
diff --git a/google/oauth2/_client.py b/google/oauth2/_client.py
@@ -102,13 +102,23 @@ def _token_endpoint_request(request, token_uri, body):
         'content-type': _URLENCODED_CONTENT_TYPE,
     }
 
-    response = request(
-        method='POST', url=token_uri, headers=headers, body=body)
-
-    response_body = response.data.decode('utf-8')
-
-    if response.status != http_client.OK:
-        _handle_error_response(response_body)
+    retry = 0
+    # retry to fetch token for maximum of two times if any internal failure
+    # occurs.
+    while True:
+        response = request(
+            method='POST', url=token_uri, headers=headers, body=body)
+        response_body = response.data.decode('utf-8')
+
+        if response.status == http_client.OK:
+            break
+        else:
+            error_desc = json.loads(
+                response_body).get('error_description') or ''
+            if error_desc == 'internal_failure' and retry < 1:
+                retry += 1
+                continue
+            _handle_error_response(response_body)
 
     response_data = json.loads(response_body)
 

diff --git a/tests/oauth2/test__client.py b/tests/oauth2/test__client.py
@@ -106,6 +106,18 @@ def test__token_endpoint_request_error():
         _client._token_endpoint_request(request, 'http://example.com', {})
 
 
+def test__token_endpoint_request_internal_failure_error():
+    request = make_request({'error': 'internal_failure',
+                            'error_description': 'internal_failure'},
+                           status=http_client.BAD_REQUEST)
+
+    with pytest.raises(exceptions.RefreshError):
+        _client._token_endpoint_request(
+            request, 'http://example.com',
+            {'error': 'internal_failure',
+             'error_description': 'internal_failure'})
+
+
 def verify_request_params(request, params):
     request_body = request.call_args[1]['body']
     request_params = urllib.parse.parse_qs(request_body)