From b8c9051ab699e714645134f4a0876e6f5167b9ba Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Fri, 26 Oct 2018 15:02:30 -0700 Subject: [PATCH] ENH: Add credentials argument to read_gbq and to_gbq. (#231) * ENH: Add credentials argument to read_gbq and to_gbq. Deprecates private_key parameter. * DOC: write files service account example first --- docs/source/changelog.rst | 19 ++- docs/source/howto/authentication.rst | 37 +++++- pandas_gbq/gbq.py | 95 ++++++++++---- tests/system/test_gbq.py | 180 ++++++++++++++------------- 4 files changed, 213 insertions(+), 118 deletions(-) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index dff44ef6..116fa187 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -1,11 +1,26 @@ Changelog ========= -.. _changelog-0.7.1: +.. _changelog-0.8.0: -0.7.1 / unreleased +0.8.0 / unreleased -------------------- +Breaking changes +~~~~~~~~~~~~~~~~ + +- **Deprecate** ``private_key`` parameter to :func:`pandas_gbq.read_gbq` and + :func:`pandas_gbq.to_gbq` in favor of new ``credentials`` argument. Instead, + create a credentials object using + :func:`google.oauth2.service_account.Credentials.from_service_account_info` + or + :func:`google.oauth2.service_account.Credentials.from_service_account_file`. + See the :doc:`authentication how-to guide ` for + examples. (:issue:`161`, :issue:`TODO`) + +Enhancements +~~~~~~~~~~~~ + - Allow newlines in data passed to ``to_gbq``. (:issue:`180`) .. _changelog-0.7.0: diff --git a/docs/source/howto/authentication.rst b/docs/source/howto/authentication.rst index 9f69ed12..7d54467c 100644 --- a/docs/source/howto/authentication.rst +++ b/docs/source/howto/authentication.rst @@ -18,11 +18,38 @@ Create a service account key via the `service account key creation page the Google Cloud Platform Console. Select the JSON key type and download the key file. -To use service account credentials, set the ``private_key`` parameter to one -of: - -* A file path to the JSON file. -* A string containing the JSON file contents. +To use service account credentials, set the ``credentials`` parameter to the result of a call to: + +* :func:`google.oauth2.service_account.Credentials.from_service_account_file`, + which accepts a file path to the JSON file. + + .. code:: python + + credentials = google.oauth2.service_account.Credentials.from_service_account_file( + 'path/to/key.json', + ) + df = pandas_gbq.read_gbq(sql, project_id="YOUR-PROJECT-ID", credentials=credentials) + +* :func:`google.oauth2.service_account.Credentials.from_service_account_info`, + which accepts a dictionary corresponding to the JSON file contents. + + .. code:: python + + credentials = google.oauth2.service_account.Credentials.from_service_account_info( + { + "type": "service_account", + "project_id": "YOUR-PROJECT-ID", + "private_key_id": "6747200734a1f2b9d8d62fc0b9414c5f2461db0e", + "private_key": "-----BEGIN PRIVATE KEY-----\nM...I==\n-----END PRIVATE KEY-----\n", + "client_email": "service-account@YOUR-PROJECT-ID.iam.gserviceaccount.com", + "client_id": "12345678900001", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://accounts.google.com/o/oauth2/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/...iam.gserviceaccount.com" + }, + ) + df = pandas_gbq.read_gbq(sql, project_id="YOUR-PROJECT-ID", credentials=credentials) See the `Getting started with authentication on Google Cloud Platform `_ guide for diff --git a/pandas_gbq/gbq.py b/pandas_gbq/gbq.py index cf57e574..d31defc0 100644 --- a/pandas_gbq/gbq.py +++ b/pandas_gbq/gbq.py @@ -236,6 +236,7 @@ def __init__( dialect="legacy", location=None, try_credentials=None, + credentials=None, ): global context from google.api_core.exceptions import GoogleAPIError @@ -249,11 +250,14 @@ def __init__( self.private_key = private_key self.auth_local_webserver = auth_local_webserver self.dialect = dialect + self.credentials = credentials self.credentials_path = _get_credentials_file() + default_project = None # Load credentials from cache. - self.credentials = context.credentials - default_project = context.project + if not self.credentials: + self.credentials = context.credentials + default_project = context.project # Credentials were explicitly asked for, so don't use the cache. if private_key or reauth or not self.credentials: @@ -563,7 +567,7 @@ def schema_is_subset(self, dataset_id, table_id, schema): def delete_and_recreate_table(self, dataset_id, table_id, table_schema): table = _Table( - self.project_id, dataset_id, private_key=self.private_key + self.project_id, dataset_id, credentials=self.credentials ) table.delete(table_id) table.create(table_id, table_schema) @@ -621,12 +625,13 @@ def read_gbq( index_col=None, col_order=None, reauth=False, - private_key=None, auth_local_webserver=False, dialect=None, location=None, configuration=None, + credentials=None, verbose=None, + private_key=None, ): r"""Load data from Google BigQuery using google-cloud-python @@ -655,10 +660,6 @@ def read_gbq( reauth : boolean, default False Force Google BigQuery to re-authenticate the user. This is useful if multiple accounts are used. - private_key : str, optional - Service account private key in JSON format. Can be file path - or string contents. This is useful for remote server - authentication (eg. Jupyter/IPython notebook on remote host). auth_local_webserver : boolean, default False Use the `local webserver flow`_ instead of the `console flow`_ when getting user credentials. @@ -699,10 +700,28 @@ def read_gbq( For more information see `BigQuery REST API Reference `__. + credentials : google.auth.credentials.Credentials, optional + Credentials for accessing Google APIs. Use this parameter to override + default credentials, such as to use Compute Engine + :class:`google.auth.compute_engine.Credentials` or Service Account + :class:`google.oauth2.service_account.Credentials` directly. + + .. versionadded:: 0.8.0 verbose : None, deprecated Deprecated in Pandas-GBQ 0.4.0. Use the `logging module to adjust verbosity instead `__. + private_key : str, deprecated + Deprecated in pandas-gbq version 0.8.0. Use the ``credentials`` + parameter and + :func:`google.oauth2.service_account.Credentials.from_service_account_info` + or + :func:`google.oauth2.service_account.Credentials.from_service_account_file` + instead. + + Service account private key in JSON format. Can be file path + or string contents. This is useful for remote server + authentication (eg. Jupyter/IPython notebook on remote host). Returns ------- @@ -736,10 +755,11 @@ def read_gbq( connector = GbqConnector( project_id, reauth=reauth, - private_key=private_key, dialect=dialect, auth_local_webserver=auth_local_webserver, location=location, + credentials=credentials, + private_key=private_key, ) schema, rows = connector.run_query(query, configuration=configuration) final_df = _parse_data(schema, rows) @@ -779,12 +799,13 @@ def to_gbq( chunksize=None, reauth=False, if_exists="fail", - private_key=None, auth_local_webserver=False, table_schema=None, location=None, progress_bar=True, + credentials=None, verbose=None, + private_key=None, ): """Write a DataFrame to a Google BigQuery table. @@ -822,10 +843,6 @@ def to_gbq( If table exists, drop it, recreate it, and insert data. ``'append'`` If table exists, insert data. Create if does not exist. - private_key : str, optional - Service account private key in JSON format. Can be file path - or string contents. This is useful for remote server - authentication (eg. Jupyter/IPython notebook on remote host). auth_local_webserver : bool, default False Use the `local webserver flow`_ instead of the `console flow`_ when getting user credentials. @@ -861,10 +878,28 @@ def to_gbq( chunk by chunk. .. versionadded:: 0.5.0 + credentials : google.auth.credentials.Credentials, optional + Credentials for accessing Google APIs. Use this parameter to override + default credentials, such as to use Compute Engine + :class:`google.auth.compute_engine.Credentials` or Service Account + :class:`google.oauth2.service_account.Credentials` directly. + + .. versionadded:: 0.8.0 verbose : bool, deprecated Deprecated in Pandas-GBQ 0.4.0. Use the `logging module to adjust verbosity instead `__. + private_key : str, deprecated + Deprecated in pandas-gbq version 0.8.0. Use the ``credentials`` + parameter and + :func:`google.oauth2.service_account.Credentials.from_service_account_info` + or + :func:`google.oauth2.service_account.Credentials.from_service_account_file` + instead. + + Service account private key in JSON format. Can be file path + or string contents. This is useful for remote server + authentication (eg. Jupyter/IPython notebook on remote host). """ _test_google_api_imports() @@ -889,21 +924,21 @@ def to_gbq( connector = GbqConnector( project_id, reauth=reauth, - private_key=private_key, auth_local_webserver=auth_local_webserver, location=location, # Avoid reads when writing tables. # https://github.com/pydata/pandas-gbq/issues/202 try_credentials=lambda project, creds: creds, + credentials=credentials, + private_key=private_key, ) dataset_id, table_id = destination_table.rsplit(".", 1) table = _Table( project_id, dataset_id, - reauth=reauth, - private_key=private_key, location=location, + credentials=connector.credentials, ) if not table_schema: @@ -980,12 +1015,17 @@ def __init__( project_id, dataset_id, reauth=False, - private_key=None, location=None, + credentials=None, + private_key=None, ): self.dataset_id = dataset_id super(_Table, self).__init__( - project_id, reauth, private_key, location=location + project_id, + reauth, + location=location, + credentials=credentials, + private_key=private_key, ) def exists(self, table_id): @@ -1031,12 +1071,12 @@ def create(self, table_id, schema): "Table {0} already " "exists".format(table_id) ) - if not _Dataset(self.project_id, private_key=self.private_key).exists( + if not _Dataset(self.project_id, credentials=self.credentials).exists( self.dataset_id ): _Dataset( self.project_id, - private_key=self.private_key, + credentials=self.credentials, location=self.location, ).create(self.dataset_id) @@ -1084,10 +1124,19 @@ def delete(self, table_id): class _Dataset(GbqConnector): def __init__( - self, project_id, reauth=False, private_key=None, location=None + self, + project_id, + reauth=False, + location=None, + credentials=None, + private_key=None, ): super(_Dataset, self).__init__( - project_id, reauth, private_key, location=location + project_id, + reauth, + credentials=credentials, + location=location, + private_key=private_key, ) def exists(self, dataset_id): diff --git a/tests/system/test_gbq.py b/tests/system/test_gbq.py index e263f72a..9ae1fbf2 100644 --- a/tests/system/test_gbq.py +++ b/tests/system/test_gbq.py @@ -4,6 +4,7 @@ import uuid from datetime import datetime +import google.oauth2.service_account import numpy as np import pandas.util.testing as tm import pytest @@ -13,6 +14,7 @@ from pandas_gbq import gbq + TABLE_ID = "new_test" @@ -34,13 +36,15 @@ def project(request, project_id): @pytest.fixture() -def credentials(private_key_contents): - return private_key_contents +def credentials(private_key_path): + return google.oauth2.service_account.Credentials.from_service_account_file( + private_key_path + ) @pytest.fixture() def gbq_connector(project, credentials): - return gbq.GbqConnector(project, private_key=credentials) + return gbq.GbqConnector(project, credentials=credentials) @pytest.fixture(scope="module") @@ -97,12 +101,12 @@ def tokyo_table(bigquery_client, tokyo_dataset): @pytest.fixture() def gbq_dataset(project, credentials): - return gbq._Dataset(project, private_key=credentials) + return gbq._Dataset(project, credentials=credentials) @pytest.fixture() def gbq_table(project, credentials, random_dataset_id): - return gbq._Table(project, random_dataset_id, private_key=credentials) + return gbq._Table(project, random_dataset_id, credentials=credentials) def make_mixed_dataframe_v2(test_size): @@ -146,7 +150,7 @@ def test_should_be_able_to_get_results_from_query(self, gbq_connector): def test_should_read(project, credentials): query = 'SELECT "PI" AS valid_string' df = gbq.read_gbq( - query, project_id=project, private_key=credentials, dialect="legacy" + query, project_id=project, credentials=credentials, dialect="legacy" ) tm.assert_frame_equal(df, DataFrame({"valid_string": ["PI"]})) @@ -157,7 +161,7 @@ def setup(self, project, credentials): # - PER-TEST FIXTURES - # put here any instruction you want to be run *BEFORE* *EVERY* test is # executed. - self.gbq_connector = gbq.GbqConnector(project, private_key=credentials) + self.gbq_connector = gbq.GbqConnector(project, credentials=credentials) self.credentials = credentials def test_should_properly_handle_empty_strings(self, project_id): @@ -165,7 +169,7 @@ def test_should_properly_handle_empty_strings(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"empty_string": [""]})) @@ -175,7 +179,7 @@ def test_should_properly_handle_null_strings(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"null_string": [None]})) @@ -185,7 +189,7 @@ def test_should_properly_handle_valid_integers(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"valid_integer": [3]})) @@ -197,7 +201,7 @@ def test_should_properly_handle_nullable_integers(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"nullable_integer": [1, None]})) @@ -207,7 +211,7 @@ def test_should_properly_handle_valid_longs(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"valid_long": [1 << 62]})) @@ -219,7 +223,7 @@ def test_should_properly_handle_nullable_longs(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal( @@ -231,7 +235,7 @@ def test_should_properly_handle_null_integers(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"null_integer": [None]})) @@ -243,7 +247,7 @@ def test_should_properly_handle_valid_floats(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"valid_float": [pi]})) @@ -257,7 +261,7 @@ def test_should_properly_handle_nullable_floats(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"nullable_float": [pi, None]})) @@ -269,7 +273,7 @@ def test_should_properly_handle_valid_doubles(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal( @@ -285,7 +289,7 @@ def test_should_properly_handle_nullable_doubles(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal( @@ -297,7 +301,7 @@ def test_should_properly_handle_null_floats(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"null_float": [np.nan]})) @@ -307,7 +311,7 @@ def test_should_properly_handle_timestamp_unix_epoch(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal( @@ -322,7 +326,7 @@ def test_should_properly_handle_arbitrary_timestamp(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal( @@ -359,7 +363,7 @@ def test_return_correct_types(self, project_id, expression, type_): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="standard", ) assert df["_"].dtype == type_ @@ -369,7 +373,7 @@ def test_should_properly_handle_null_timestamp(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"null_timestamp": [NaT]})) @@ -379,7 +383,7 @@ def test_should_properly_handle_null_boolean(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, DataFrame({"null_boolean": [None]})) @@ -391,7 +395,7 @@ def test_should_properly_handle_nullable_booleans(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal( @@ -411,7 +415,7 @@ def test_unicode_string_conversion_and_normalization(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) tm.assert_frame_equal(df, correct_test_datatype) @@ -422,7 +426,7 @@ def test_index_column(self, project_id): query, project_id=project_id, index_col="string_1", - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) correct_frame = DataFrame( @@ -437,7 +441,7 @@ def test_column_order(self, project_id): query, project_id=project_id, col_order=col_order, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) correct_frame = DataFrame( @@ -455,7 +459,7 @@ def test_read_gbq_raises_invalid_column_order(self, project_id): query, project_id=project_id, col_order=col_order, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -467,7 +471,7 @@ def test_column_order_plus_index(self, project_id): project_id=project_id, index_col="string_1", col_order=col_order, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) correct_frame = DataFrame( @@ -488,7 +492,7 @@ def test_read_gbq_raises_invalid_index_column(self, project_id): project_id=project_id, index_col="string_bbb", col_order=col_order, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -497,7 +501,7 @@ def test_malformed_query(self, project_id): gbq.read_gbq( "SELCET * FORM [publicdata:samples.shakespeare]", project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -506,7 +510,7 @@ def test_bad_project_id(self): gbq.read_gbq( "SELCET * FROM [publicdata:samples.shakespeare]", project_id="not-my-project", - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -515,7 +519,7 @@ def test_bad_table_name(self, project_id): gbq.read_gbq( "SELECT * FROM [publicdata:samples.nope]", project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -527,7 +531,7 @@ def test_download_dataset_larger_than_200k_rows(self, project_id): "SELECT id FROM [publicdata:samples.wikipedia] " "GROUP EACH BY id ORDER BY id ASC LIMIT {0}".format(test_size), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) assert len(df.drop_duplicates()) == test_size @@ -540,7 +544,7 @@ def test_zero_rows(self, project_id): "FROM [publicdata:samples.wikipedia] " "WHERE timestamp=-9999999", project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) page_array = np.zeros( @@ -561,7 +565,7 @@ def test_one_row_one_column(self, project_id): df = gbq.read_gbq( "SELECT 3 as v", project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="standard", ) expected_result = DataFrame(dict(v=[3])) @@ -577,7 +581,7 @@ def test_legacy_sql(self, project_id): legacy_sql, project_id=project_id, dialect="standard", - private_key=self.credentials, + credentials=self.credentials, ) # Test that a legacy sql statement succeeds when @@ -586,7 +590,7 @@ def test_legacy_sql(self, project_id): legacy_sql, project_id=project_id, dialect="legacy", - private_key=self.credentials, + credentials=self.credentials, ) assert len(df.drop_duplicates()) == 10 @@ -602,7 +606,7 @@ def test_standard_sql(self, project_id): gbq.read_gbq( standard_sql, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -612,7 +616,7 @@ def test_standard_sql(self, project_id): standard_sql, project_id=project_id, dialect="standard", - private_key=self.credentials, + credentials=self.credentials, ) assert len(df.drop_duplicates()) == 10 @@ -642,7 +646,7 @@ def test_query_with_parameters(self, project_id): gbq.read_gbq( sql_statement, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -651,7 +655,7 @@ def test_query_with_parameters(self, project_id): df = gbq.read_gbq( sql_statement, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, configuration=config, dialect="legacy", ) @@ -667,7 +671,7 @@ def test_query_inside_configuration(self, project_id): gbq.read_gbq( query_no_use, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, configuration=config, dialect="legacy", ) @@ -675,7 +679,7 @@ def test_query_inside_configuration(self, project_id): df = gbq.read_gbq( None, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, configuration=config, dialect="legacy", ) @@ -703,7 +707,7 @@ def test_configuration_without_query(self, project_id): gbq.read_gbq( sql_statement, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, configuration=config, dialect="legacy", ) @@ -721,7 +725,7 @@ def test_configuration_raises_value_error_with_multiple_config( gbq.read_gbq( sql_statement, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, configuration=config, dialect="legacy", ) @@ -734,7 +738,7 @@ def test_timeout_configuration(self, project_id): gbq.read_gbq( sql_statement, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, configuration=config, dialect="legacy", ) @@ -760,7 +764,7 @@ def test_struct(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="standard", ) expected = DataFrame( @@ -774,7 +778,7 @@ def test_array(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="standard", ) tm.assert_frame_equal( @@ -794,7 +798,7 @@ def test_array_length_zero(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="standard", ) expected = DataFrame( @@ -818,7 +822,7 @@ def test_array_agg(self, project_id): df = gbq.read_gbq( query, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="standard", ) tm.assert_frame_equal( @@ -860,7 +864,7 @@ def setup(self, project, credentials, random_dataset_id): # put here any instruction you want to be run *BEFORE* *EVERY* test is # executed. self.table = gbq._Table( - project, random_dataset_id, private_key=credentials + project, random_dataset_id, credentials=credentials ) self.destination_table = "{}.{}".format(random_dataset_id, TABLE_ID) self.credentials = credentials @@ -875,7 +879,7 @@ def test_upload_data(self, project_id): self.destination_table + test_id, project_id, chunksize=10000, - private_key=self.credentials, + credentials=self.credentials, ) result = gbq.read_gbq( @@ -883,7 +887,7 @@ def test_upload_data(self, project_id): self.destination_table + test_id ), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) assert result["num_rows"][0] == test_size @@ -900,7 +904,7 @@ def test_upload_data_if_table_exists_fail(self, project_id): df, self.destination_table + test_id, project_id, - private_key=self.credentials, + credentials=self.credentials, ) # Test the if_exists parameter with value 'fail' @@ -910,7 +914,7 @@ def test_upload_data_if_table_exists_fail(self, project_id): self.destination_table + test_id, project_id, if_exists="fail", - private_key=self.credentials, + credentials=self.credentials, ) def test_upload_data_if_table_exists_append(self, project_id): @@ -925,7 +929,7 @@ def test_upload_data_if_table_exists_append(self, project_id): self.destination_table + test_id, project_id, chunksize=10000, - private_key=self.credentials, + credentials=self.credentials, ) # Test the if_exists parameter with value 'append' @@ -934,7 +938,7 @@ def test_upload_data_if_table_exists_append(self, project_id): self.destination_table + test_id, project_id, if_exists="append", - private_key=self.credentials, + credentials=self.credentials, ) result = gbq.read_gbq( @@ -942,7 +946,7 @@ def test_upload_data_if_table_exists_append(self, project_id): self.destination_table + test_id ), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) assert result["num_rows"][0] == test_size * 2 @@ -954,7 +958,7 @@ def test_upload_data_if_table_exists_append(self, project_id): self.destination_table + test_id, project_id, if_exists="append", - private_key=self.credentials, + credentials=self.credentials, ) def test_upload_subset_columns_if_table_exists_append(self, project_id): @@ -971,7 +975,7 @@ def test_upload_subset_columns_if_table_exists_append(self, project_id): self.destination_table + test_id, project_id, chunksize=10000, - private_key=self.credentials, + credentials=self.credentials, ) # Test the if_exists parameter with value 'append' @@ -980,7 +984,7 @@ def test_upload_subset_columns_if_table_exists_append(self, project_id): self.destination_table + test_id, project_id, if_exists="append", - private_key=self.credentials, + credentials=self.credentials, ) result = gbq.read_gbq( @@ -988,7 +992,7 @@ def test_upload_subset_columns_if_table_exists_append(self, project_id): self.destination_table + test_id ), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) assert result["num_rows"][0] == test_size * 2 @@ -1005,7 +1009,7 @@ def test_upload_data_if_table_exists_replace(self, project_id): self.destination_table + test_id, project_id, chunksize=10000, - private_key=self.credentials, + credentials=self.credentials, ) # Test the if_exists parameter with the value 'replace'. @@ -1014,7 +1018,7 @@ def test_upload_data_if_table_exists_replace(self, project_id): self.destination_table + test_id, project_id, if_exists="replace", - private_key=self.credentials, + credentials=self.credentials, ) result = gbq.read_gbq( @@ -1022,7 +1026,7 @@ def test_upload_data_if_table_exists_replace(self, project_id): self.destination_table + test_id ), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) assert result["num_rows"][0] == 5 @@ -1039,7 +1043,7 @@ def test_upload_data_if_table_exists_raises_value_error(self, project_id): self.destination_table + test_id, project_id, if_exists="xxxxx", - private_key=self.credentials, + credentials=self.credentials, ) def test_google_upload_errors_should_raise_exception(self, project_id): @@ -1063,7 +1067,7 @@ def test_google_upload_errors_should_raise_exception(self, project_id): bad_df, self.destination_table + test_id, project_id, - private_key=self.credentials, + credentials=self.credentials, ) def test_upload_chinese_unicode_data(self, project_id): @@ -1078,14 +1082,14 @@ def test_upload_chinese_unicode_data(self, project_id): df, self.destination_table + test_id, project_id, - private_key=self.credentials, + credentials=self.credentials, chunksize=10000, ) result_df = gbq.read_gbq( "SELECT * FROM {0}".format(self.destination_table + test_id), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -1118,14 +1122,14 @@ def test_upload_other_unicode_data(self, project_id): df, self.destination_table + test_id, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, chunksize=10000, ) result_df = gbq.read_gbq( "SELECT * FROM {0}".format(self.destination_table + test_id), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -1155,13 +1159,13 @@ def test_upload_mixed_float_and_int(self, project_id): df, self.destination_table + test_id, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, ) result_df = gbq.read_gbq( "SELECT * FROM {0}".format(self.destination_table + test_id), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -1176,13 +1180,13 @@ def test_upload_data_with_newlines(self, project_id): df, self.destination_table + test_id, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, ) result_df = gbq.read_gbq( "SELECT * FROM {0}".format(self.destination_table + test_id), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -1207,7 +1211,7 @@ def test_upload_data_flexible_column_order(self, project_id): self.destination_table + test_id, project_id, chunksize=10000, - private_key=self.credentials, + credentials=self.credentials, ) df_columns_reversed = df[df.columns[::-1]] @@ -1217,7 +1221,7 @@ def test_upload_data_flexible_column_order(self, project_id): self.destination_table + test_id, project_id, if_exists="append", - private_key=self.credentials, + credentials=self.credentials, ) def test_upload_data_with_valid_user_schema(self, project_id): @@ -1236,7 +1240,7 @@ def test_upload_data_with_valid_user_schema(self, project_id): df, destination_table, project_id, - private_key=self.credentials, + credentials=self.credentials, table_schema=test_schema, ) dataset, table = destination_table.split(".") @@ -1261,7 +1265,7 @@ def test_upload_data_with_invalid_user_schema_raises_error( df, destination_table, project_id, - private_key=self.credentials, + credentials=self.credentials, table_schema=test_schema, ) @@ -1281,7 +1285,7 @@ def test_upload_data_with_missing_schema_fields_raises_error( df, destination_table, project_id, - private_key=self.credentials, + credentials=self.credentials, table_schema=test_schema, ) @@ -1299,13 +1303,13 @@ def test_upload_data_with_timestamp(self, project_id): df, self.destination_table + test_id, project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, ) result_df = gbq.read_gbq( "SELECT * FROM {0}".format(self.destination_table + test_id), project_id=project_id, - private_key=self.credentials, + credentials=self.credentials, dialect="legacy", ) @@ -1331,7 +1335,7 @@ def test_upload_data_with_different_df_and_user_schema(self, project_id): df, destination_table, project_id, - private_key=self.credentials, + credentials=self.credentials, table_schema=test_schema, ) dataset, table = destination_table.split(".") @@ -1351,7 +1355,7 @@ def test_upload_data_tokyo( df, tokyo_destination, project_id, - private_key=self.credentials, + credentials=self.credentials, location="asia-northeast1", ) @@ -1375,7 +1379,7 @@ def test_upload_data_tokyo_non_existing_dataset( df, non_existing_tokyo_destination, project_id, - private_key=self.credentials, + credentials=self.credentials, location="asia-northeast1", ) @@ -1455,7 +1459,7 @@ def test_create_table_data_dataset_does_not_exist( ): table_id = "test_create_table_data_dataset_does_not_exist" table_with_new_dataset = gbq._Table( - project, random_dataset_id, private_key=credentials + project, random_dataset_id, credentials=credentials ) df = make_mixed_dataframe_v2(10) table_with_new_dataset.create(table_id, gbq._generate_bq_schema(df))