Replace oauth2client by google-auth (#2361)

* Replace deprecated oauth2client with google-auth

* Replace oauth2client by google.auth in other packages

* Disable cache discovery to reduce log

luigi/contrib/bigquery.py

@@ -126,7 +126,7 @@ def __init__(self, oauth_credentials=None, descriptor='', http_=None):
         if descriptor:
             self.client = discovery.build_from_document(descriptor, **authenticate_kwargs)
         else:
-            self.client = discovery.build('bigquery', 'v2', **authenticate_kwargs)
+            self.client = discovery.build('bigquery', 'v2', cache_discovery=False, **authenticate_kwargs)
 
     def dataset_exists(self, dataset):
         """Returns whether the given dataset exists.

luigi/contrib/dataproc.py

@@ -11,15 +11,15 @@
 _dataproc_client = None
 
 try:
-    import oauth2client.client
+    import google.auth
     from googleapiclient import discovery
     from googleapiclient.errors import HttpError
 
-    DEFAULT_CREDENTIALS = oauth2client.client.GoogleCredentials.get_application_default()
+    DEFAULT_CREDENTIALS, _ = google.auth.default()
     authenticate_kwargs = gcp.get_authenticate_kwargs(DEFAULT_CREDENTIALS)
-    _dataproc_client = discovery.build('dataproc', 'v1', **authenticate_kwargs)
+    _dataproc_client = discovery.build('dataproc', 'v1', cache_discovery=False, **authenticate_kwargs)
 except ImportError:
-    logger.warning("Loading Dataproc module without the python packages googleapiclient & oauth2client. \
+    logger.warning("Loading Dataproc module without the python packages googleapiclient & google-auth. \
         This will crash at runtime if Dataproc functionality is used.")
 
 

luigi/contrib/gcp.py

@@ -6,9 +6,9 @@
 
 try:
     import httplib2
-    import oauth2client
+    import google.auth
 except ImportError:
-    logger.warning("Loading GCP module without the python packages httplib2, oauth2client. \
+    logger.warning("Loading GCP module without the python packages httplib2, google-auth. \
         This *could* crash at runtime if no other credentials are provided.")
 
 
@@ -33,11 +33,11 @@ def get_authenticate_kwargs(oauth_credentials=None, http_=None):
         # neither http_ or credentials provided
         try:
             # try default credentials
-            oauth_credentials = oauth2client.client.GoogleCredentials.get_application_default()
+            credentials, _ = google.auth.default()
             authenticate_kwargs = {
-                "credentials": oauth_credentials
+                "credentials": credentials
             }
-        except oauth2client.client.GoogleCredentials.ApplicationDefaultCredentialsError:
+        except google.auth.exceptions.DefaultCredentialsError:
             # try http using httplib2
             authenticate_kwargs = {
                 "http": httplib2.Http()

luigi/contrib/gcs.py

@@ -42,7 +42,7 @@
     from googleapiclient import discovery
     from googleapiclient import http
 except ImportError:
-    logger.warning("Loading GCS module without the python packages googleapiclient & oauth2client. \
+    logger.warning("Loading GCS module without the python packages googleapiclient & google-auth. \
         This will crash at runtime if GCS functionality is used.")
 else:
     # Retry transport and file IO errors.
@@ -89,9 +89,9 @@ class GCSClient(luigi.target.FileSystem):
 
        There are several ways to use this class. By default it will use the app
        default credentials, as described at https://developers.google.com/identity/protocols/application-default-credentials .
-       Alternatively, you may pass an oauth2client credentials object. e.g. to use a service account::
+       Alternatively, you may pass an google-auth credentials object. e.g. to use a service account::
 
-         credentials = oauth2client.client.SignedJwtAssertionCredentials(
+         credentials = google.auth.jwt.Credentials.from_service_account_info(
              '012345678912-ThisIsARandomServiceAccountEmail@developer.gserviceaccount.com',
              'These are the contents of the p12 file that came with the service account',
              scope='https://www.googleapis.com/auth/devstorage.read_write')
@@ -115,7 +115,7 @@ def __init__(self, oauth_credentials=None, descriptor='', http_=None,
         if descriptor:
             self.client = discovery.build_from_document(descriptor, **authenticate_kwargs)
         else:
-            self.client = discovery.build('storage', 'v1', **authenticate_kwargs)
+            self.client = discovery.build('storage', 'v1', cache_discovery=False, **authenticate_kwargs)
 
     def _path_to_bucket_and_key(self, path):
         (scheme, netloc, path, _, _) = urlsplit(path)

test/contrib/bigquery_gcloud_test.py

@@ -30,7 +30,7 @@
 
 try:
     import googleapiclient.errors
-    import oauth2client
+    import google.auth
 except ImportError:
     raise unittest.SkipTest('Unable to load googleapiclient module')
 from luigi.contrib import bigquery, bigquery_avro, gcs
@@ -52,7 +52,7 @@
 EU_LOCATION = 'EU'
 US_LOCATION = 'US'
 
-CREDENTIALS = oauth2client.client.GoogleCredentials.get_application_default()
+CREDENTIALS, _ = google.auth.default()
 
 
 def bucket_url(suffix):

test/contrib/dataproc_test.py

@@ -6,12 +6,12 @@
 import unittest
 
 try:
-    import oauth2client
+    import google.auth
     from luigi.contrib import dataproc
     from googleapiclient import discovery
 
-    default_credentials = oauth2client.client.GoogleCredentials.get_application_default()
-    default_client = discovery.build('dataproc', 'v1', credentials=default_credentials)
+    default_credentials, _ = google.auth.default()
+    default_client = discovery.build('dataproc', 'v1', cache_discovery=False, credentials=default_credentials)
     dataproc.set_dataproc_client(default_client)
 except ImportError:
     raise unittest.SkipTest('Unable to load google cloud dependencies')

test/contrib/gcs_test.py

@@ -24,7 +24,7 @@
 from helpers import unittest
 try:
     import googleapiclient.errors
-    import oauth2client
+    import google.auth
 except ImportError:
     raise unittest.SkipTest('Unable to load googleapiclient module')
 import os
@@ -41,7 +41,7 @@
 BUCKET_NAME = os.environ.get('GCS_TEST_BUCKET', 'your_test_bucket_here')
 TEST_FOLDER = os.environ.get('TRAVIS_BUILD_ID', 'gcs_test_folder')
 
-CREDENTIALS = oauth2client.client.GoogleCredentials.get_application_default()
+CREDENTIALS, _ = google.auth.default()
 ATTEMPTED_BUCKET_CREATE = False
 
 

tox.ini

@@ -23,7 +23,7 @@ deps=
   gcloud: google-api-python-client>=1.4.0,<2.0
   py27-gcloud: avro
   py33-gcloud,py34-gcloud,py35-gcloud,py36-gcloud: avro-python3
-  gcloud: oauth2client<4.0.0
+  gcloud: google-auth==1.4.1
   google-compute-engine
   coverage>=4.1,<4.2
   codecov>=1.4.0