From 309467758feb2100665390b291a4cf4759a534ce Mon Sep 17 00:00:00 2001 From: Josema Camacho Date: Fri, 4 Sep 2020 13:46:40 +0200 Subject: [PATCH 1/5] GCloud credentials from DO --- cartoframes/auth/credentials.py | 29 ++- cartoframes/data/clients/bigquery_client.py | 207 ------------------ cartoframes/data/observatory/catalog/utils.py | 2 +- tests/e2e/data/client/test_bigquery_client.py | 102 --------- tests/unit/auth/test_credentials.py | 17 +- .../unit/data/client/test_bigquery_client.py | 80 ------- tests/unit/data/observatory/catalog/mocks.py | 18 -- 7 files changed, 35 insertions(+), 420 deletions(-) delete mode 100644 cartoframes/data/clients/bigquery_client.py delete mode 100644 tests/e2e/data/client/test_bigquery_client.py delete mode 100644 tests/unit/data/client/test_bigquery_client.py delete mode 100644 tests/unit/data/observatory/catalog/mocks.py diff --git a/cartoframes/auth/credentials.py b/cartoframes/auth/credentials.py index c02cc76d0..eea87fcb8 100644 --- a/cartoframes/auth/credentials.py +++ b/cartoframes/auth/credentials.py @@ -3,13 +3,13 @@ import os from urllib.parse import urlparse + from carto.auth import APIKeyAuthClient from carto.do_token import DoTokenManager from .. import __version__ from ..utils.logger import log -from ..utils.utils import is_valid_str, check_do_enabled, save_in_config, \ - read_from_config, default_config_path +from ..utils.utils import is_valid_str, check_do_enabled, save_in_config, read_from_config, default_config_path from warnings import filterwarnings filterwarnings('ignore', category=FutureWarning, module='carto') @@ -64,6 +64,7 @@ def __init__(self, username=None, api_key='default_public', base_url=None, sessi self._user_id = None self._api_key_auth_client = None self._allow_non_secure = allow_non_secure + self._do_credentials = None self._norm_credentials() @@ -256,11 +257,15 @@ def delete(cls, config_file=None): except OSError: log.warning('No credential file found at {}.'.format(path_to_remove)) - @check_do_enabled - def get_do_credentials(self): - """Returns the Data Observatory v2 credentials""" - do_token_manager = DoTokenManager(self.get_api_key_auth_client()) - return do_token_manager.get() + def is_instant_licensing_active(self): + """Returns if the user has instant licensing activated for the Data Observatory v2""" + do_credentials = self._get_do_credentials() + return do_credentials.instant_licensing + + def get_gcloud_auth_info(self): + """Returns the Data Observatory v2 Google Cloud token and project""" + do_credentials = self._get_do_credentials() + return do_credentials.access_token, do_credentials.bq_project def get_api_key_auth_client(self): if not self._api_key_auth_client: @@ -274,6 +279,16 @@ def get_api_key_auth_client(self): return self._api_key_auth_client + @check_do_enabled + def _get_do_credentials(self): + """Returns the Data Observatory v2 credentials""" + if self._do_credentials: + return self._do_credentials + + do_token_manager = DoTokenManager(self.get_api_key_auth_client()) + self._do_credentials = do_token_manager.get() + return self._do_credentials + def _norm_credentials(self): """Standardize credentials""" if self._base_url: diff --git a/cartoframes/data/clients/bigquery_client.py b/cartoframes/data/clients/bigquery_client.py deleted file mode 100644 index f9710432b..000000000 --- a/cartoframes/data/clients/bigquery_client.py +++ /dev/null @@ -1,207 +0,0 @@ -import os -import csv -import tqdm -import pandas as pd - -from google.auth.exceptions import RefreshError -from google.cloud import bigquery, storage, bigquery_storage_v1beta1 as bigquery_storage -from google.oauth2.credentials import Credentials as GoogleCredentials -from google.api_core.exceptions import DeadlineExceeded - -from ...auth import get_default_credentials -from ...utils.logger import log -from ...utils.utils import timelogger, is_ipython_notebook -from ...exceptions import DOError - -_GCS_CHUNK_SIZE = 25 * 1024 * 1024 # 25MB. This must be a multiple of 256 KB per the API specification. -_BQS_TIMEOUT = 2 * 3600 # 2 hours in seconds - - -def refresh_clients(func): - def wrapper(self, *args, **kwargs): - try: - return func(self, *args, **kwargs) - except RefreshError: - self._init_clients() - try: - return func(self, *args, **kwargs) - except RefreshError: - raise DOError('Something went wrong accessing data. ' - 'Please, try again in a few seconds or contact support for help.') - return wrapper - - -class BigQueryClient: - - def __init__(self, credentials): - self._credentials = credentials or get_default_credentials() - self.bq_client = None - self.gcs_client = None - self.bq_storage_client = None - - self._gcp_execution_project = None - self.bq_public_project = None - self.bq_project = None - self.bq_dataset = None - self.instant_licensing = None - self._gcs_bucket = None - - self._init_clients() - - def _init_clients(self): - do_credentials = self._credentials.get_do_credentials() - google_credentials = GoogleCredentials(do_credentials.access_token) - - self.bq_client = bigquery.Client( - project=do_credentials.gcp_execution_project, - credentials=google_credentials - ) - - self.gcs_client = storage.Client( - project=do_credentials.bq_project, - credentials=google_credentials - ) - - self.bq_storage_client = bigquery_storage.BigQueryStorageClient( - credentials=google_credentials - ) - - self._gcp_execution_project = do_credentials.gcp_execution_project - self.bq_public_project = do_credentials.bq_public_project - self.bq_project = do_credentials.bq_project - self.bq_dataset = do_credentials.bq_dataset - self.instant_licensing = do_credentials.instant_licensing - self._gcs_bucket = do_credentials.gcs_bucket - - @refresh_clients - def query(self, query, **kwargs): - return self.bq_client.query(query, **kwargs) - - def upload_dataframe(self, dataframe, schema, tablename): - self._upload_dataframe_to_GCS(dataframe, tablename) - self._import_from_GCS_to_BQ(schema, tablename) - - @timelogger - def download_to_file(self, job, file_path, fail_if_exists=False, column_names=None, progress_bar=True): - if fail_if_exists and os.path.isfile(file_path): - raise OSError('The file `{}` already exists.'.format(file_path)) - - try: - rows = self._download_by_bq_storage_api(job) - except Exception: - log.debug('Cannot download using BigQuery Storage API, fallback to standard') - rows = _get_job_result(job, 'Error downloading data') - - try: - _rows_to_file(rows, file_path, column_names, progress_bar) - except DeadlineExceeded: - log.debug('Cannot download using BigQuery Storage API, fallback to standard') - rows = _get_job_result(job, 'Error downloading data') - _rows_to_file(rows, file_path, column_names, progress_bar) - - @timelogger - def download_to_dataframe(self, job): - try: - rows = self._download_by_bq_storage_api(job) - data = list(rows) - return pd.DataFrame(data) - except Exception: - log.debug('Cannot download using BigQuery Storage API, fallback to standard') - - try: - return job.to_dataframe() - except Exception: - if job.errors: - log.error([error['message'] for error in job.errors if 'message' in error]) - - raise DOError('Error downloading data') - - def _download_by_bq_storage_api(self, job, timeout=_BQS_TIMEOUT): - table_ref = job.destination.to_bqstorage() - - parent = 'projects/{}'.format(self._gcp_execution_project) - session = self.bq_storage_client.create_read_session( - table_ref, - parent, - requested_streams=1, - format_=bigquery_storage.enums.DataFormat.AVRO, - # We use a LIQUID strategy because we only read from a - # single stream. Consider BALANCED if requested_streams > 1 - sharding_strategy=(bigquery_storage.enums.ShardingStrategy.LIQUID) - ) - - reader = self.bq_storage_client.read_rows( - bigquery_storage.types.StreamPosition(stream=session.streams[0]), - timeout=timeout - ) - - return reader.rows(session) - - @refresh_clients - @timelogger - def _upload_dataframe_to_GCS(self, dataframe, tablename): - log.debug('Uploading to GCS') - bucket = self.gcs_client.get_bucket(self._gcs_bucket) - blob = bucket.blob(tablename, chunk_size=_GCS_CHUNK_SIZE) - dataframe.to_csv(tablename, index=False, header=False) - try: - blob.upload_from_filename(tablename) - finally: - os.remove(tablename) - - @refresh_clients - @timelogger - def _import_from_GCS_to_BQ(self, schema, tablename): - log.debug('Importing to BQ from GCS') - - dataset_ref = self.bq_client.dataset(self.bq_dataset, project=self.bq_project) - table_ref = dataset_ref.table(tablename) - schema_wrapped = [bigquery.SchemaField(column, dtype) for column, dtype in schema.items()] - - job_config = bigquery.LoadJobConfig() - job_config.schema = schema_wrapped - job_config.source_format = bigquery.SourceFormat.CSV - uri = 'gs://{bucket}/{tablename}'.format(bucket=self._gcs_bucket, tablename=tablename) - - job = self.bq_client.load_table_from_uri( - uri, table_ref, job_config=job_config - ) - - _get_job_result(job, 'Error uploading data') - - def get_table_column_names(self, project, dataset, table): - table_info = self._get_table(project, dataset, table) - return [field.name for field in table_info.schema] - - @refresh_clients - def _get_table(self, project, dataset, table): - full_table_name = '{}.{}.{}'.format(project, dataset, table) - return self.bq_client.get_table(full_table_name) - - -def _rows_to_file(rows, file_path, column_names=None, progress_bar=True): - show_progress_bar = progress_bar and is_ipython_notebook() - - if show_progress_bar: - pb = tqdm.tqdm_notebook(total=rows.total_rows) - - with open(file_path, 'w') as csvfile: - csvwriter = csv.writer(csvfile) - - if column_names: - csvwriter.writerow(column_names) - - for row in rows: - csvwriter.writerow(row.values()) - if show_progress_bar: - pb.update(1) - - -def _get_job_result(job, error_message): - try: - return job.result() - except Exception: - if job.errors: - log.error([error['message'] for error in job.errors if 'message' in error]) - - raise DOError(error_message) diff --git a/cartoframes/data/observatory/catalog/utils.py b/cartoframes/data/observatory/catalog/utils.py index d0a2edfb7..a0f3a589a 100644 --- a/cartoframes/data/observatory/catalog/utils.py +++ b/cartoframes/data/observatory/catalog/utils.py @@ -15,7 +15,7 @@ def display_existing_subscription_message(entity_id, entity_type): def display_subscription_form(entity_id, entity_type, credentials): info = fetch_subscription_info(entity_id, entity_type, credentials) - instant_licensing = credentials.get_do_credentials().instant_licensing + instant_licensing = credentials.is_instant_licensing_active() if is_ipython_notebook(): _display_subscription_form_notebook(entity_id, entity_type, info, instant_licensing, credentials) diff --git a/tests/e2e/data/client/test_bigquery_client.py b/tests/e2e/data/client/test_bigquery_client.py deleted file mode 100644 index 4b1b308e4..000000000 --- a/tests/e2e/data/client/test_bigquery_client.py +++ /dev/null @@ -1,102 +0,0 @@ -import os -import json -import pytest -import unittest - -from google.auth.exceptions import RefreshError -from google.cloud import bigquery - -from cartoframes.auth import Credentials -from cartoframes.exceptions import DOError -from cartoframes.data.clients.bigquery_client import BigQueryClient - - -_WORKING_PROJECT = 'carto-do-customers' - - -class RefreshTokenChecker(object): - def __init__(self, response, raise_after=1): - self.number_of_calls = 0 - self.response = response - self.raise_after = raise_after - - def query_raiser(self, query, **kwargs): - self.number_of_calls += 1 - if self.number_of_calls < self.raise_after: - return self.response - else: - raise RefreshError() - - -class ResponseMock(list): - def __init__(self, data, **kwargs): - super(ResponseMock, self).__init__(data, **kwargs) - self.total_rows = len(data) - - -class QueryJobMock(object): - def __init__(self, response): - self.response = response - - def result(self): - return ResponseMock(self.response) - - -class TestBigQueryClient(unittest.TestCase): - def setUp(self): - if (os.environ.get('APIKEY') is None or os.environ.get('USERNAME') is None): - creds = json.loads(open('tests/e2e/secret.json').read()) - self.apikey = creds['APIKEY'] - self.username = creds['USERNAME'] - else: - self.apikey = os.environ['APIKEY'] - self.username = os.environ['USERNAME'] - - self.credentials = Credentials(self.username, self.apikey) - self.file_path = '/tmp/test_download.csv' - - def tearDown(self): - if os.path.isfile(self.file_path): - os.remove(self.file_path) - - def test_instantiation(self): - bq_client = BigQueryClient(self.credentials) - assert isinstance(bq_client, BigQueryClient) - - def test_refresh_token_raises_cartoexception(self): - refresh_token_checker = RefreshTokenChecker('', 0) - original_query_method = bigquery.Client.query - bigquery.Client.query = refresh_token_checker.query_raiser - - bq_client = BigQueryClient(self.credentials) - with pytest.raises(DOError): - bq_client.query('select * from') - - bigquery.Client.query = original_query_method - - def test_refresh_token(self): - expected_response = 'ok' - refresh_token_checker = RefreshTokenChecker(expected_response, 2) - original_query_method = bigquery.Client.query - bigquery.Client.query = refresh_token_checker.query_raiser - - bq_client = BigQueryClient(self.credentials) - response = bq_client.query('select * from') - assert response == expected_response - - bigquery.Client.query = original_query_method - - def test_download_using_if_exists(self): - project = _WORKING_PROJECT - dataset = 'fake_dataset' - table = 'fake_table' - file_path = self.file_path - - bq_client = BigQueryClient(self.credentials) - - query = 'SELECT * FROM `{}.{}.{}`'.format(project, dataset, table) - job = bq_client.query(query) - - with open(file_path, 'w'): - with self.assertRaises(OSError): - bq_client.download_to_file(job, file_path, fail_if_exists=True, progress_bar=False) diff --git a/tests/unit/auth/test_credentials.py b/tests/unit/auth/test_credentials.py index 06b7901ae..9d62c8ece 100644 --- a/tests/unit/auth/test_credentials.py +++ b/tests/unit/auth/test_credentials.py @@ -107,19 +107,26 @@ def test_get_api_key_auth_client(self): credentials.get_api_key_auth_client() assert credentials._api_key_auth_client is not None - def test_get_do_credentials(self, mocker): + def test_do_credentials(self, mocker): access_token = '1234' + project = 'project' + instant_licensing = True - class Token: + class DOCredentials: def __init__(self): self.access_token = access_token + self.bq_project = project + self.instant_licensing = instant_licensing - mocker.patch('carto.do_token.DoTokenManager.get', return_value=Token()) + mocker.patch('carto.do_token.DoTokenManager.get', return_value=DOCredentials()) credentials = Credentials(self.username, self.api_key) - do_credentials = credentials.get_do_credentials() + access_token_from_do, project_from_do = credentials.get_gcloud_auth_info() + instant_licensing_from_do = credentials.is_instant_licensing_active() - assert do_credentials.access_token == access_token + assert access_token_from_do == access_token + assert project_from_do == project + assert instant_licensing_from_do == instant_licensing class TestCredentialsFromFile: diff --git a/tests/unit/data/client/test_bigquery_client.py b/tests/unit/data/client/test_bigquery_client.py deleted file mode 100644 index b27783179..000000000 --- a/tests/unit/data/client/test_bigquery_client.py +++ /dev/null @@ -1,80 +0,0 @@ -import os -import csv -import pandas as pd - -from unittest.mock import Mock, patch - -from cartoframes.auth import Credentials -from cartoframes.data.clients.bigquery_client import BigQueryClient - - -class ResponseMock(list): - def __init__(self, data, **kwargs): - super(ResponseMock, self).__init__(data, **kwargs) - self.total_rows = len(data) - - -class QueryJobMock(object): - def __init__(self, response): - self.response = response - - def result(self): - return ResponseMock(self.response) - - -class TestBigQueryClient(object): - def setup_method(self): - self.original_init_clients = BigQueryClient._init_clients - BigQueryClient._init_clients = Mock(return_value=(True, True, True)) - self.username = 'username' - self.apikey = 'apikey' - self.credentials = Credentials(self.username, self.apikey) - self.file_path = '/tmp/test_download.csv' - - def teardown_method(self): - self.credentials = None - BigQueryClient._init_clients = self.original_init_clients - - if os.path.isfile(self.file_path): - os.remove(self.file_path) - - @patch.object(BigQueryClient, 'get_table_column_names') - @patch.object(BigQueryClient, '_download_by_bq_storage_api') - def test_download_to_file_full(self, download_mock, column_names_mock): - data = [{'0': 'word', '1': 'word word'}] - columns = ['column1', 'column2'] - - column_names_mock.return_value = Mock(return_value=columns) - download_mock.return_value = data - - file_path = self.file_path - - bq_client = BigQueryClient(self.credentials) - job = QueryJobMock(data) - bq_client.download_to_file(job, file_path, column_names=columns, progress_bar=False) - - rows = [] - with open(file_path) as csvfile: - csvreader = csv.reader(csvfile) - rows.append(next(csvreader)) - rows.append(next(csvreader)) - - assert rows[0] == columns - assert rows[1] == list(data[0].values()) - - @patch.object(BigQueryClient, 'get_table_column_names') - @patch.object(BigQueryClient, '_download_by_bq_storage_api') - def test_download_to_dataframe_full(self, download_mock, column_names_mock): - data = [{'column1': 'word', 'column2': 'word word'}] - columns = ['column1', 'column2'] - - column_names_mock.return_value = Mock(return_value=columns) - download_mock.return_value = data - - expected_df = pd.DataFrame(data, columns=columns) - - bq_client = BigQueryClient(self.credentials) - job = QueryJobMock(data) - df = bq_client.download_to_dataframe(job) - - assert df.equals(expected_df) diff --git a/tests/unit/data/observatory/catalog/mocks.py b/tests/unit/data/observatory/catalog/mocks.py deleted file mode 100644 index e561086d4..000000000 --- a/tests/unit/data/observatory/catalog/mocks.py +++ /dev/null @@ -1,18 +0,0 @@ -class BigQueryClientMock(object): - def __init__(self, exception=None): - self.exception = exception - - self.bq_public_project = 'public_data_project' - self.bq_project = 'user_data_project' - self.bq_dataset = 'username' - self._gcs_bucket = 'bucket_name' - - def query(self, _1): - return True - - def download_to_file(self, _1, _2, column_names=None): - if isinstance(self.exception, Exception): - raise self.exception - - def get_table_column_names(self, _1, _2, _3): - return True From 8b3c96b33456e8f446bf549aba94267c3146aa94 Mon Sep 17 00:00:00 2001 From: Josema Camacho Date: Mon, 7 Sep 2020 12:55:42 +0200 Subject: [PATCH 2/5] Modified GCP credenials function name and return order --- cartoframes/auth/credentials.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cartoframes/auth/credentials.py b/cartoframes/auth/credentials.py index eea87fcb8..d0334e92b 100644 --- a/cartoframes/auth/credentials.py +++ b/cartoframes/auth/credentials.py @@ -262,10 +262,10 @@ def is_instant_licensing_active(self): do_credentials = self._get_do_credentials() return do_credentials.instant_licensing - def get_gcloud_auth_info(self): + def get_gcp_auth_info(self): """Returns the Data Observatory v2 Google Cloud token and project""" do_credentials = self._get_do_credentials() - return do_credentials.access_token, do_credentials.bq_project + return do_credentials.bq_project, do_credentials.access_token def get_api_key_auth_client(self): if not self._api_key_auth_client: From e2bb016cf9899e360a339a19f132582304c317fb Mon Sep 17 00:00:00 2001 From: Josema Camacho Date: Mon, 7 Sep 2020 17:08:55 +0200 Subject: [PATCH 3/5] Added example --- cartoframes/auth/credentials.py | 13 +- docs/developer-center/examples/examples.json | 5 + .../google_cloud_platform_credentials.ipynb | 176 ++++++++++++++++++ 3 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 examples/data_observatory/google_cloud_platform_credentials.ipynb diff --git a/cartoframes/auth/credentials.py b/cartoframes/auth/credentials.py index d0334e92b..117aa0c71 100644 --- a/cartoframes/auth/credentials.py +++ b/cartoframes/auth/credentials.py @@ -258,12 +258,21 @@ def delete(cls, config_file=None): log.warning('No credential file found at {}.'.format(path_to_remove)) def is_instant_licensing_active(self): - """Returns if the user has instant licensing activated for the Data Observatory v2""" + """Returns if the user has instant licensing activated for the Data Observatory v2.""" do_credentials = self._get_do_credentials() return do_credentials.instant_licensing def get_gcp_auth_info(self): - """Returns the Data Observatory v2 Google Cloud token and project""" + """Returns the Data Observatory v2 Google Cloud Platform project and token. + + Example: + >>> from cartoframes.auth import Credentials + >>> from google.oauth2.credentials import Credentials as GCPCredentials + >>> creds = Credentials(username='johnsmith', api_key='abcdefg') + >>> gcp_project, gcp_token = creds.get_gcp_auth_info() + >>> gcp_credentials = GCPCredentials(gcp_token) + + """ do_credentials = self._get_do_credentials() return do_credentials.bq_project, do_credentials.access_token diff --git a/docs/developer-center/examples/examples.json b/docs/developer-center/examples/examples.json index 1c8bbe334..06b2eea9f 100644 --- a/docs/developer-center/examples/examples.json +++ b/docs/developer-center/examples/examples.json @@ -380,6 +380,11 @@ "desc": "Basic steps to subscribe to a premium dataset in the Data Observatory to start enriching your data", "file": "enrichment_subscription_workflow", "path": "data_observatory" + }, { + "title": "Google Cloud Platform credentials", + "desc": "Get and use your Data Observatory's Google Cloud Platform credentials", + "file": "google_cloud_platform_credentials", + "path": "data_observatory" }] }, { "title": "Publish and share", diff --git a/examples/data_observatory/google_cloud_platform_credentials.ipynb b/examples/data_observatory/google_cloud_platform_credentials.ipynb new file mode 100644 index 000000000..d9a138c4e --- /dev/null +++ b/examples/data_observatory/google_cloud_platform_credentials.ipynb @@ -0,0 +1,176 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Google Cloud Platform credentials\n", + "\n", + "This example illustrates how to create a Google Cloud Platform credentials object from your Data Observatory enabled CARTO account." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from cartoframes.auth import Credentials\n", + "from cartoframes.data.observatory import Dataset\n", + "\n", + "from google.cloud import bigquery, storage, bigquery_storage_v1beta1 as bigquery_storage\n", + "from google.oauth2.credentials import Credentials as GCPCredentials" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "creds = Credentials('creds.json')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "gcp_project, gcp_token = creds.get_gcp_auth_info()\n", + "gcp_credentials = GCPCredentials(gcp_token)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "bq_client = bigquery.Client(project=gcp_project, credentials=gcp_credentials)\n", + "gcs_client = storage.Client(project=gcp_project, credentials=gcp_credentials)\n", + "bqs_client = bigquery_storage.BigQueryStorageClient(credentials=gcp_project)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "dataset = Dataset.get('acs_sociodemogr_8c2655e0')" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset.is_public_data" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'carto-do-public-data.usa_acs.demographics_sociodemographics_usa_county_2015_5yrs_20132017'" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dataset.id" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "query_job = bq_client.query('SELECT * FROM `{}` LIMIT 1'.format(dataset.id))\n", + "results = query_job.result()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[SchemaField('geoid', 'STRING', 'NULLABLE', None, ()),\n", + " SchemaField('do_date', 'DATE', 'NULLABLE', None, ()),\n", + " SchemaField('total_pop', 'FLOAT', 'NULLABLE', None, ())]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results.schema[0:3]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "18025 10598.0\n" + ] + } + ], + "source": [ + "for row in results:\n", + " print(row['geoid'], row['total_pop'])" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.5.3 64-bit ('cartoframes': venv)", + "language": "python", + "name": "python35364bitcartoframesvenveab847f182df42dfb14e0580b7bdb4bc" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From ab56ed5a9fe60827f17461b95f02d2f647ca183e Mon Sep 17 00:00:00 2001 From: Josema Camacho Date: Tue, 8 Sep 2020 09:38:49 +0200 Subject: [PATCH 4/5] Fixed gcp auth info credentials test --- tests/unit/auth/test_credentials.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/auth/test_credentials.py b/tests/unit/auth/test_credentials.py index 9d62c8ece..bf7116107 100644 --- a/tests/unit/auth/test_credentials.py +++ b/tests/unit/auth/test_credentials.py @@ -121,7 +121,7 @@ def __init__(self): mocker.patch('carto.do_token.DoTokenManager.get', return_value=DOCredentials()) credentials = Credentials(self.username, self.api_key) - access_token_from_do, project_from_do = credentials.get_gcloud_auth_info() + access_token_from_do, project_from_do = credentials.get_gcp_auth_info() instant_licensing_from_do = credentials.is_instant_licensing_active() assert access_token_from_do == access_token From 69183fd8313cf0ca97064a240f4aa2bb7a2e006c Mon Sep 17 00:00:00 2001 From: Josema Camacho Date: Tue, 8 Sep 2020 09:41:32 +0200 Subject: [PATCH 5/5] get_gcp_auth_info return order --- tests/unit/auth/test_credentials.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/auth/test_credentials.py b/tests/unit/auth/test_credentials.py index bf7116107..f3724cffd 100644 --- a/tests/unit/auth/test_credentials.py +++ b/tests/unit/auth/test_credentials.py @@ -121,7 +121,7 @@ def __init__(self): mocker.patch('carto.do_token.DoTokenManager.get', return_value=DOCredentials()) credentials = Credentials(self.username, self.api_key) - access_token_from_do, project_from_do = credentials.get_gcp_auth_info() + project_from_do, access_token_from_do = credentials.get_gcp_auth_info() instant_licensing_from_do = credentials.is_instant_licensing_active() assert access_token_from_do == access_token