From bd638c9c1a9c6e6107481af2d00198205642faaa Mon Sep 17 00:00:00 2001 From: Thibaut Mattio Date: Thu, 22 Feb 2018 13:58:58 -0500 Subject: [PATCH 1/3] Replace deprecated oauth2client with google-auth --- luigi/contrib/gcp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/luigi/contrib/gcp.py b/luigi/contrib/gcp.py index 140b6b541e..8eaed6cc5a 100644 --- a/luigi/contrib/gcp.py +++ b/luigi/contrib/gcp.py @@ -6,7 +6,7 @@ try: import httplib2 - import oauth2client + import google.auth except ImportError: logger.warning("Loading GCP module without the python packages httplib2, oauth2client. \ This *could* crash at runtime if no other credentials are provided.") @@ -33,11 +33,11 @@ def get_authenticate_kwargs(oauth_credentials=None, http_=None): # neither http_ or credentials provided try: # try default credentials - oauth_credentials = oauth2client.client.GoogleCredentials.get_application_default() + credentials, _ = google.auth.default() authenticate_kwargs = { - "credentials": oauth_credentials + "credentials": credentials } - except oauth2client.client.GoogleCredentials.ApplicationDefaultCredentialsError: + except google.auth.exceptions.DefaultCredentialsError: # try http using httplib2 authenticate_kwargs = { "http": httplib2.Http() From 828055248dbbdac5da220471e49349aa956d8286 Mon Sep 17 00:00:00 2001 From: Thibaut Mattio Date: Thu, 22 Feb 2018 14:08:29 -0500 Subject: [PATCH 2/3] Replace oauth2client by google.auth in other packages --- luigi/contrib/dataproc.py | 6 +++--- luigi/contrib/gcp.py | 2 +- luigi/contrib/gcs.py | 6 +++--- test/contrib/bigquery_gcloud_test.py | 4 ++-- test/contrib/dataproc_test.py | 4 ++-- test/contrib/gcs_test.py | 4 ++-- tox.ini | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/luigi/contrib/dataproc.py b/luigi/contrib/dataproc.py index 67f9f11f03..533e96bb6a 100644 --- a/luigi/contrib/dataproc.py +++ b/luigi/contrib/dataproc.py @@ -11,15 +11,15 @@ _dataproc_client = None try: - import oauth2client.client + import google.auth from googleapiclient import discovery from googleapiclient.errors import HttpError - DEFAULT_CREDENTIALS = oauth2client.client.GoogleCredentials.get_application_default() + DEFAULT_CREDENTIALS, _ = google.auth.default() authenticate_kwargs = gcp.get_authenticate_kwargs(DEFAULT_CREDENTIALS) _dataproc_client = discovery.build('dataproc', 'v1', **authenticate_kwargs) except ImportError: - logger.warning("Loading Dataproc module without the python packages googleapiclient & oauth2client. \ + logger.warning("Loading Dataproc module without the python packages googleapiclient & google-auth. \ This will crash at runtime if Dataproc functionality is used.") diff --git a/luigi/contrib/gcp.py b/luigi/contrib/gcp.py index 8eaed6cc5a..bb768b94f6 100644 --- a/luigi/contrib/gcp.py +++ b/luigi/contrib/gcp.py @@ -8,7 +8,7 @@ import httplib2 import google.auth except ImportError: - logger.warning("Loading GCP module without the python packages httplib2, oauth2client. \ + logger.warning("Loading GCP module without the python packages httplib2, google-auth. \ This *could* crash at runtime if no other credentials are provided.") diff --git a/luigi/contrib/gcs.py b/luigi/contrib/gcs.py index 7a60b509b5..3b1f554bb2 100644 --- a/luigi/contrib/gcs.py +++ b/luigi/contrib/gcs.py @@ -42,7 +42,7 @@ from googleapiclient import discovery from googleapiclient import http except ImportError: - logger.warning("Loading GCS module without the python packages googleapiclient & oauth2client. \ + logger.warning("Loading GCS module without the python packages googleapiclient & google-auth. \ This will crash at runtime if GCS functionality is used.") else: # Retry transport and file IO errors. @@ -89,9 +89,9 @@ class GCSClient(luigi.target.FileSystem): There are several ways to use this class. By default it will use the app default credentials, as described at https://developers.google.com/identity/protocols/application-default-credentials . - Alternatively, you may pass an oauth2client credentials object. e.g. to use a service account:: + Alternatively, you may pass an google-auth credentials object. e.g. to use a service account:: - credentials = oauth2client.client.SignedJwtAssertionCredentials( + credentials = google.auth.jwt.Credentials.from_service_account_info( '012345678912-ThisIsARandomServiceAccountEmail@developer.gserviceaccount.com', 'These are the contents of the p12 file that came with the service account', scope='https://www.googleapis.com/auth/devstorage.read_write') diff --git a/test/contrib/bigquery_gcloud_test.py b/test/contrib/bigquery_gcloud_test.py index 75bc9bb8b8..68f71b3a08 100644 --- a/test/contrib/bigquery_gcloud_test.py +++ b/test/contrib/bigquery_gcloud_test.py @@ -30,7 +30,7 @@ try: import googleapiclient.errors - import oauth2client + import google.auth except ImportError: raise unittest.SkipTest('Unable to load googleapiclient module') from luigi.contrib import bigquery, bigquery_avro, gcs @@ -52,7 +52,7 @@ EU_LOCATION = 'EU' US_LOCATION = 'US' -CREDENTIALS = oauth2client.client.GoogleCredentials.get_application_default() +CREDENTIALS, _ = google.auth.default() def bucket_url(suffix): diff --git a/test/contrib/dataproc_test.py b/test/contrib/dataproc_test.py index 5de1cfbd8a..a9b9db9e8f 100644 --- a/test/contrib/dataproc_test.py +++ b/test/contrib/dataproc_test.py @@ -6,11 +6,11 @@ import unittest try: - import oauth2client + import google.auth from luigi.contrib import dataproc from googleapiclient import discovery - default_credentials = oauth2client.client.GoogleCredentials.get_application_default() + default_credentials, _ = google.auth.default() default_client = discovery.build('dataproc', 'v1', credentials=default_credentials) dataproc.set_dataproc_client(default_client) except ImportError: diff --git a/test/contrib/gcs_test.py b/test/contrib/gcs_test.py index ff8405c04c..24e014071d 100644 --- a/test/contrib/gcs_test.py +++ b/test/contrib/gcs_test.py @@ -24,7 +24,7 @@ from helpers import unittest try: import googleapiclient.errors - import oauth2client + import google.auth except ImportError: raise unittest.SkipTest('Unable to load googleapiclient module') import os @@ -41,7 +41,7 @@ BUCKET_NAME = os.environ.get('GCS_TEST_BUCKET', 'your_test_bucket_here') TEST_FOLDER = os.environ.get('TRAVIS_BUILD_ID', 'gcs_test_folder') -CREDENTIALS = oauth2client.client.GoogleCredentials.get_application_default() +CREDENTIALS, _ = google.auth.default() ATTEMPTED_BUCKET_CREATE = False diff --git a/tox.ini b/tox.ini index ea1eaa0221..1482c89db7 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ deps= gcloud: google-api-python-client>=1.4.0,<2.0 py27-gcloud: avro py33-gcloud,py34-gcloud,py35-gcloud,py36-gcloud: avro-python3 - gcloud: oauth2client<4.0.0 + gcloud: google-auth==1.4.1 google-compute-engine coverage>=4.1,<4.2 codecov>=1.4.0 From b9357b0b31c8290bac2c3c5f9582bdb31716c7ab Mon Sep 17 00:00:00 2001 From: Thibaut Mattio Date: Fri, 23 Feb 2018 09:25:38 -0500 Subject: [PATCH 3/3] Disable cache discovery to reduce log --- luigi/contrib/bigquery.py | 2 +- luigi/contrib/dataproc.py | 2 +- luigi/contrib/gcs.py | 2 +- test/contrib/dataproc_test.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/luigi/contrib/bigquery.py b/luigi/contrib/bigquery.py index 54c933799a..72b50eba8d 100644 --- a/luigi/contrib/bigquery.py +++ b/luigi/contrib/bigquery.py @@ -126,7 +126,7 @@ def __init__(self, oauth_credentials=None, descriptor='', http_=None): if descriptor: self.client = discovery.build_from_document(descriptor, **authenticate_kwargs) else: - self.client = discovery.build('bigquery', 'v2', **authenticate_kwargs) + self.client = discovery.build('bigquery', 'v2', cache_discovery=False, **authenticate_kwargs) def dataset_exists(self, dataset): """Returns whether the given dataset exists. diff --git a/luigi/contrib/dataproc.py b/luigi/contrib/dataproc.py index 533e96bb6a..a20ab7abd9 100644 --- a/luigi/contrib/dataproc.py +++ b/luigi/contrib/dataproc.py @@ -17,7 +17,7 @@ DEFAULT_CREDENTIALS, _ = google.auth.default() authenticate_kwargs = gcp.get_authenticate_kwargs(DEFAULT_CREDENTIALS) - _dataproc_client = discovery.build('dataproc', 'v1', **authenticate_kwargs) + _dataproc_client = discovery.build('dataproc', 'v1', cache_discovery=False, **authenticate_kwargs) except ImportError: logger.warning("Loading Dataproc module without the python packages googleapiclient & google-auth. \ This will crash at runtime if Dataproc functionality is used.") diff --git a/luigi/contrib/gcs.py b/luigi/contrib/gcs.py index 3b1f554bb2..49f95543f2 100644 --- a/luigi/contrib/gcs.py +++ b/luigi/contrib/gcs.py @@ -115,7 +115,7 @@ def __init__(self, oauth_credentials=None, descriptor='', http_=None, if descriptor: self.client = discovery.build_from_document(descriptor, **authenticate_kwargs) else: - self.client = discovery.build('storage', 'v1', **authenticate_kwargs) + self.client = discovery.build('storage', 'v1', cache_discovery=False, **authenticate_kwargs) def _path_to_bucket_and_key(self, path): (scheme, netloc, path, _, _) = urlsplit(path) diff --git a/test/contrib/dataproc_test.py b/test/contrib/dataproc_test.py index a9b9db9e8f..9e53b5fb12 100644 --- a/test/contrib/dataproc_test.py +++ b/test/contrib/dataproc_test.py @@ -11,7 +11,7 @@ from googleapiclient import discovery default_credentials, _ = google.auth.default() - default_client = discovery.build('dataproc', 'v1', credentials=default_credentials) + default_client = discovery.build('dataproc', 'v1', cache_discovery=False, credentials=default_credentials) dataproc.set_dataproc_client(default_client) except ImportError: raise unittest.SkipTest('Unable to load google cloud dependencies')