Setting default credentials for getting datasets in staging

cartoframes/auth/defaults.py

@@ -2,6 +2,7 @@
 from ..utils.utils import is_url, is_json_filepath
 
 _default_credentials = None
+_default_do_credentials = None
 
 
 def set_default_credentials(
@@ -85,30 +86,10 @@ def set_default_credentials(
 
     """
     global _default_credentials
-
-    _base_url = base_url if first is None else first
-    _username = username if first is None else first
-    _filepath = filepath if first is None else first
-    _api_key = (api_key if second is None else second) or 'default_public'
-    _credentials = credentials if first is None else first
-
-    if isinstance(_credentials, Credentials):
-        _default_credentials = _credentials
-
-    elif isinstance(_filepath, str) and is_json_filepath(_filepath):
-        _default_credentials = Credentials.from_file(_filepath)
-
-    elif isinstance(_base_url or _username, str) and isinstance(_api_key, str):
-        if _base_url and is_url(_base_url):
-            _default_credentials = Credentials(base_url=_base_url, api_key=_api_key, allow_non_secure=allow_non_secure)
-        else:
-            _default_credentials = Credentials(username=_username, api_key=_api_key, allow_non_secure=allow_non_secure)
-
-    else:
-        _default_credentials = Credentials.from_file()
-
-    if session:
-        _default_credentials.session = session
+    _default_credentials = _set_credentials(
+        first, second, credentials, filepath, username,
+        base_url, api_key, session, allow_non_secure
+    )
 
 
 def get_default_credentials():
@@ -139,3 +120,54 @@ def unset_default_credentials():
     """
     global _default_credentials
     _default_credentials = None
+
+
+def set_default_do_credentials(
+        first=None, second=None, credentials=None, filepath=None,
+        username=None, base_url=None, api_key=None, session=None, allow_non_secure=False):
+
+    global _default_do_credentials
+    _default_do_credentials = _set_credentials(
+        first, second, credentials, filepath, username,
+        base_url, api_key, session, allow_non_secure
+    )
+
+
+def get_default_do_credentials():
+    return _default_do_credentials
+
+
+def unset_default_do_credentials():
+    global _default_do_credentials
+    _default_do_credentials = None
+
+
+def _set_credentials(
+        first=None, second=None, credentials=None, filepath=None,
+        username=None, base_url=None, api_key=None, session=None, allow_non_secure=False):
+
+    _base_url = base_url if first is None else first
+    _username = username if first is None else first
+    _filepath = filepath if first is None else first
+    _api_key = (api_key if second is None else second) or 'default_public'
+    _credentials = credentials if first is None else first
+
+    if isinstance(_credentials, Credentials):
+        pass
+
+    elif isinstance(_filepath, str) and is_json_filepath(_filepath):
+        _credentials = Credentials.from_file(_filepath)
+
+    elif isinstance(_base_url or _username, str) and isinstance(_api_key, str):
+        if _base_url and is_url(_base_url):
+            _credentials = Credentials(base_url=_base_url, api_key=_api_key, allow_non_secure=allow_non_secure)
+        else:
+            _credentials = Credentials(username=_username, api_key=_api_key, allow_non_secure=allow_non_secure)
+
+    else:
+        _credentials = Credentials.from_file()
+
+    if session:
+        _credentials.session = session
+
+    return _credentials

cartoframes/data/observatory/catalog/dataset.py

@@ -343,7 +343,7 @@ def get_all(cls, filters=None, credentials=None):
         return cls._entity_repo.get_all(filters, credentials)
 
     @classmethod
-    def get_datasets_spatial_filtered(cls, filter_dataset):
+    def get_datasets_spatial_filtered(cls, filter_dataset, credentials=None):
         user_gdf = cls._get_user_geodataframe(filter_dataset)
 
         # TODO: check if the dataframe has a geometry column if not exception
@@ -352,8 +352,13 @@ def get_datasets_spatial_filtered(cls, filter_dataset):
         catalog_geographies_gdf = get_geography_repo().get_geographies_gdf()
         matched_geographies_ids = cls._join_geographies_geodataframes(catalog_geographies_gdf, user_gdf)
 
+        if credentials is not None:
+            check_credentials(credentials)
+
         # Get Dataset objects
-        return get_dataset_repo().get_all({'geography_id': list(matched_geographies_ids)})
+        return get_dataset_repo().get_all(
+            {'geography_id': list(matched_geographies_ids)}, credentials
+        )
 
     @staticmethod
     def _get_user_geodataframe(filter_dataset):

cartoframes/data/observatory/catalog/repository/repo_client.py

@@ -8,37 +8,55 @@
 class RepoClient:
 
     def __init__(self):
-        self._do_dataset = None
-        default_credentials = defaults.get_default_credentials() or Credentials(DEFAULT_USER)
+        default_credentials = Credentials(DEFAULT_USER)
         default_auth_client = default_credentials.get_api_key_auth_client()
         self._default_do_dataset = DODataset(auth_client=default_auth_client)
+        self._user_do_dataset = None
+        self._external_do_dataset = None
 
     def set_user_credentials(self, credentials):
         if credentials is not None:
             auth_client = credentials.get_api_key_auth_client()
-            self._do_dataset = DODataset(auth_client=auth_client)
+            self._user_do_dataset = DODataset(auth_client=auth_client)
         else:
-            self._do_dataset = None
+            self._user_do_dataset = None
 
     def reset_user_credentials(self):
-        self._do_dataset = None
+        self._user_do_dataset = None
+
+    def set_external_credentials(self):
+        # This must be checked every time to allow the definition of
+        # "default_do_credentials" at any point in the code because
+        # every repo uses a singleton instance of this client
+        external_credentials = defaults.get_default_do_credentials()
+        if external_credentials is not None:
+            external_auth_client = external_credentials.get_api_key_auth_client()
+            self._external_do_dataset = DODataset(auth_client=external_auth_client)
+        else:
+            self._external_do_dataset = None
 
     def get_countries(self, filters=None):
+        self.set_external_credentials()
         return self._get_entity('countries', filters)
 
     def get_categories(self, filters=None):
+        self.set_external_credentials()
         return self._get_entity('categories', filters)
 
     def get_providers(self, filters=None):
+        self.set_external_credentials()
         return self._get_entity('providers', filters)
 
     def get_datasets(self, filters=None):
+        self.set_external_credentials()
         return self._get_entity('datasets', filters, use_slug=True)
 
     def get_geographies(self, filters=None):
+        self.set_external_credentials()
         return self._get_entity('geographies', filters, use_slug=True)
 
     def get_variables(self, filters=None):
+        self.set_external_credentials()
         filter_id = self._get_filter_id(filters, use_slug=True)
         if filter_id:
             return self._fetch_entity_id('variables', filter_id)
@@ -47,6 +65,7 @@ def get_variables(self, filters=None):
             return self._fetch_entity(entity, filters)
 
     def get_variables_groups(self, filters=None):
+        self.set_external_credentials()
         filter_id = self._get_filter_id(filters, use_slug=True)
         if filter_id:
             return self._fetch_entity_id('variables_groups', filter_id)
@@ -75,7 +94,5 @@ def _fetch_entity_id(self, entity, filter_id):
             return self._fetch_entity('{0}/{1}'.format(entity, filter_id))
 
     def _fetch_entity(self, entity, filters=None):
-        if self._do_dataset:
-            return self._do_dataset.metadata(entity, filters)
-        else:
-            return self._default_do_dataset.metadata(entity, filters)
+        do_dataset = self._user_do_dataset or self._external_do_dataset or self._default_do_dataset
+        return do_dataset.metadata(entity, filters)

docs/developers/documentation.rst

@@ -147,4 +147,19 @@ Custom
 - DOError.
     - CatalogError.
     - EnrichmentError.
-- PublishError.
+- PublishError.
+
+
+Development with Staging
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Before releasing to production we need to test everything in staging. In order to do that, we need to configure CARTOframes to point to staging.
+There is a set of internal functions to configure the default DO credentials used by the DO Catalog.
+
+.. code::
+    from cartoframes.auth.defaults import set_default_do_credentials
+
+    set_default_do_credentials(username='USER', base_url='https://ORG.carto-staging.com')
+
+    # After that, every request to the DO Catalog will be done with the provided credentials
+    # instead of the default ones for production (user 'do-metadata').

tests/unit/data/observatory/catalog/test_catalog.py

@@ -198,7 +198,8 @@ def test_all_filters(self, mocked_datasets):
             COUNTRY_FILTER: 'usa',
             CATEGORY_FILTER: 'demographics',
             PUBLIC_FILTER: 'true',
-            GEOGRAPHY_FILTER: 'carto-do-public-data.tiger.geography_esp_census_2019'})
+            GEOGRAPHY_FILTER: 'carto-do-public-data.tiger.geography_esp_census_2019'
+        })
 
         assert datasets == test_datasets