josema/ch88278/provide_google_creds_in_cartoframes

cartoframes/auth/credentials.py

@@ -3,13 +3,13 @@
 import os
 
 from urllib.parse import urlparse
+
 from carto.auth import APIKeyAuthClient
 from carto.do_token import DoTokenManager
 
 from .. import __version__
 from ..utils.logger import log
-from ..utils.utils import is_valid_str, check_do_enabled, save_in_config, \
-                          read_from_config, default_config_path
+from ..utils.utils import is_valid_str, check_do_enabled, save_in_config, read_from_config, default_config_path
 
 from warnings import filterwarnings
 filterwarnings('ignore', category=FutureWarning, module='carto')
@@ -64,6 +64,7 @@ def __init__(self, username=None, api_key='default_public', base_url=None, sessi
         self._user_id = None
         self._api_key_auth_client = None
         self._allow_non_secure = allow_non_secure
+        self._do_credentials = None
 
         self._norm_credentials()
 
@@ -256,11 +257,24 @@ def delete(cls, config_file=None):
         except OSError:
             log.warning('No credential file found at {}.'.format(path_to_remove))
 
-    @check_do_enabled
-    def get_do_credentials(self):
-        """Returns the Data Observatory v2 credentials"""
-        do_token_manager = DoTokenManager(self.get_api_key_auth_client())
-        return do_token_manager.get()
+    def is_instant_licensing_active(self):
+        """Returns if the user has instant licensing activated for the Data Observatory v2."""
+        do_credentials = self._get_do_credentials()
+        return do_credentials.instant_licensing
+
+    def get_gcp_auth_info(self):
+        """Returns the Data Observatory v2 Google Cloud Platform project and token.
+
+        Example:
+            >>> from cartoframes.auth import Credentials
+            >>> from google.oauth2.credentials import Credentials as GCPCredentials
+            >>> creds = Credentials(username='johnsmith', api_key='abcdefg')
+            >>> gcp_project, gcp_token = creds.get_gcp_auth_info()
+            >>> gcp_credentials = GCPCredentials(gcp_token)
+
+        """
+        do_credentials = self._get_do_credentials()
+        return do_credentials.bq_project, do_credentials.access_token
 
     def get_api_key_auth_client(self):
         if not self._api_key_auth_client:
@@ -274,6 +288,16 @@ def get_api_key_auth_client(self):
 
         return self._api_key_auth_client
 
+    @check_do_enabled
+    def _get_do_credentials(self):
+        """Returns the Data Observatory v2 credentials"""
+        if self._do_credentials:
+            return self._do_credentials
+
+        do_token_manager = DoTokenManager(self.get_api_key_auth_client())
+        self._do_credentials = do_token_manager.get()
+        return self._do_credentials
+
     def _norm_credentials(self):
         """Standardize credentials"""
         if self._base_url:

cartoframes/data/observatory/catalog/utils.py

@@ -15,7 +15,7 @@ def display_existing_subscription_message(entity_id, entity_type):
 
 def display_subscription_form(entity_id, entity_type, credentials):
     info = fetch_subscription_info(entity_id, entity_type, credentials)
-    instant_licensing = credentials.get_do_credentials().instant_licensing
+    instant_licensing = credentials.is_instant_licensing_active()
 
     if is_ipython_notebook():
         _display_subscription_form_notebook(entity_id, entity_type, info, instant_licensing, credentials)

docs/examples/examples.json

@@ -376,6 +376,11 @@
             "desc": "Basic steps to subscribe to a premium dataset in the Data Observatory to start enriching your data",
             "file": "enrichment_subscription_workflow",
             "path": "data_observatory"
+        }, {
+            "title": "Google Cloud Platform credentials",
+            "desc": "Get and use your Data Observatory's Google Cloud Platform credentials",
+            "file": "google_cloud_platform_credentials",
+            "path": "data_observatory"
         }]
     }, {
         "title": "Publish and share",

examples/data_observatory/google_cloud_platform_credentials.ipynb

@@ -0,0 +1,176 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Google Cloud Platform credentials\n",
+    "\n",
+    "This example illustrates how to create a Google Cloud Platform credentials object from your Data Observatory enabled CARTO account."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from cartoframes.auth import Credentials\n",
+    "from cartoframes.data.observatory import Dataset\n",
+    "\n",
+    "from google.cloud import bigquery, storage, bigquery_storage_v1beta1 as bigquery_storage\n",
+    "from google.oauth2.credentials import Credentials as GCPCredentials"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "creds = Credentials('creds.json')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gcp_project, gcp_token = creds.get_gcp_auth_info()\n",
+    "gcp_credentials = GCPCredentials(gcp_token)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "bq_client = bigquery.Client(project=gcp_project, credentials=gcp_credentials)\n",
+    "gcs_client = storage.Client(project=gcp_project, credentials=gcp_credentials)\n",
+    "bqs_client = bigquery_storage.BigQueryStorageClient(credentials=gcp_project)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset = Dataset.get('acs_sociodemogr_8c2655e0')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset.is_public_data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'carto-do-public-data.usa_acs.demographics_sociodemographics_usa_county_2015_5yrs_20132017'"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset.id"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query_job = bq_client.query('SELECT * FROM `{}` LIMIT 1'.format(dataset.id))\n",
+    "results = query_job.result()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[SchemaField('geoid', 'STRING', 'NULLABLE', None, ()),\n",
+       " SchemaField('do_date', 'DATE', 'NULLABLE', None, ()),\n",
+       " SchemaField('total_pop', 'FLOAT', 'NULLABLE', None, ())]"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "results.schema[0:3]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "18025 10598.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "for row in results:\n",
+    "    print(row['geoid'], row['total_pop'])"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.5.3 64-bit ('cartoframes': venv)",
+   "language": "python",
+   "name": "python35364bitcartoframesvenveab847f182df42dfb14e0580b7bdb4bc"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.5.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

tests/unit/auth/test_credentials.py

@@ -107,19 +107,26 @@ def test_get_api_key_auth_client(self):
         credentials.get_api_key_auth_client()
         assert credentials._api_key_auth_client is not None
 
-    def test_get_do_credentials(self, mocker):
+    def test_do_credentials(self, mocker):
         access_token = '1234'
+        project = 'project'
+        instant_licensing = True
 
-        class Token:
+        class DOCredentials:
             def __init__(self):
                 self.access_token = access_token
+                self.bq_project = project
+                self.instant_licensing = instant_licensing
 
-        mocker.patch('carto.do_token.DoTokenManager.get', return_value=Token())
+        mocker.patch('carto.do_token.DoTokenManager.get', return_value=DOCredentials())
 
         credentials = Credentials(self.username, self.api_key)
-        do_credentials = credentials.get_do_credentials()
+        project_from_do, access_token_from_do = credentials.get_gcp_auth_info()
+        instant_licensing_from_do = credentials.is_instant_licensing_active()
 
-        assert do_credentials.access_token == access_token
+        assert access_token_from_do == access_token
+        assert project_from_do == project
+        assert instant_licensing_from_do == instant_licensing
 
 
 class TestCredentialsFromFile: