diff --git a/cartoframes/auth/defaults.py b/cartoframes/auth/defaults.py index 1fcb78131..9f71930b3 100644 --- a/cartoframes/auth/defaults.py +++ b/cartoframes/auth/defaults.py @@ -2,6 +2,7 @@ from ..utils.utils import is_url, is_json_filepath _default_credentials = None +_default_do_credentials = None def set_default_credentials( @@ -85,30 +86,10 @@ def set_default_credentials( """ global _default_credentials - - _base_url = base_url if first is None else first - _username = username if first is None else first - _filepath = filepath if first is None else first - _api_key = (api_key if second is None else second) or 'default_public' - _credentials = credentials if first is None else first - - if isinstance(_credentials, Credentials): - _default_credentials = _credentials - - elif isinstance(_filepath, str) and is_json_filepath(_filepath): - _default_credentials = Credentials.from_file(_filepath) - - elif isinstance(_base_url or _username, str) and isinstance(_api_key, str): - if _base_url and is_url(_base_url): - _default_credentials = Credentials(base_url=_base_url, api_key=_api_key, allow_non_secure=allow_non_secure) - else: - _default_credentials = Credentials(username=_username, api_key=_api_key, allow_non_secure=allow_non_secure) - - else: - _default_credentials = Credentials.from_file() - - if session: - _default_credentials.session = session + _default_credentials = _set_credentials( + first, second, credentials, filepath, username, + base_url, api_key, session, allow_non_secure + ) def get_default_credentials(): @@ -139,3 +120,54 @@ def unset_default_credentials(): """ global _default_credentials _default_credentials = None + + +def set_default_do_credentials( + first=None, second=None, credentials=None, filepath=None, + username=None, base_url=None, api_key=None, session=None, allow_non_secure=False): + + global _default_do_credentials + _default_do_credentials = _set_credentials( + first, second, credentials, filepath, username, + base_url, api_key, session, allow_non_secure + ) + + +def get_default_do_credentials(): + return _default_do_credentials + + +def unset_default_do_credentials(): + global _default_do_credentials + _default_do_credentials = None + + +def _set_credentials( + first=None, second=None, credentials=None, filepath=None, + username=None, base_url=None, api_key=None, session=None, allow_non_secure=False): + + _base_url = base_url if first is None else first + _username = username if first is None else first + _filepath = filepath if first is None else first + _api_key = (api_key if second is None else second) or 'default_public' + _credentials = credentials if first is None else first + + if isinstance(_credentials, Credentials): + pass + + elif isinstance(_filepath, str) and is_json_filepath(_filepath): + _credentials = Credentials.from_file(_filepath) + + elif isinstance(_base_url or _username, str) and isinstance(_api_key, str): + if _base_url and is_url(_base_url): + _credentials = Credentials(base_url=_base_url, api_key=_api_key, allow_non_secure=allow_non_secure) + else: + _credentials = Credentials(username=_username, api_key=_api_key, allow_non_secure=allow_non_secure) + + else: + _credentials = Credentials.from_file() + + if session: + _credentials.session = session + + return _credentials diff --git a/cartoframes/data/observatory/catalog/dataset.py b/cartoframes/data/observatory/catalog/dataset.py index 915229235..1f9324faf 100644 --- a/cartoframes/data/observatory/catalog/dataset.py +++ b/cartoframes/data/observatory/catalog/dataset.py @@ -343,7 +343,7 @@ def get_all(cls, filters=None, credentials=None): return cls._entity_repo.get_all(filters, credentials) @classmethod - def get_datasets_spatial_filtered(cls, filter_dataset): + def get_datasets_spatial_filtered(cls, filter_dataset, credentials=None): user_gdf = cls._get_user_geodataframe(filter_dataset) # TODO: check if the dataframe has a geometry column if not exception @@ -352,8 +352,13 @@ def get_datasets_spatial_filtered(cls, filter_dataset): catalog_geographies_gdf = get_geography_repo().get_geographies_gdf() matched_geographies_ids = cls._join_geographies_geodataframes(catalog_geographies_gdf, user_gdf) + if credentials is not None: + check_credentials(credentials) + # Get Dataset objects - return get_dataset_repo().get_all({'geography_id': list(matched_geographies_ids)}) + return get_dataset_repo().get_all( + {'geography_id': list(matched_geographies_ids)}, credentials + ) @staticmethod def _get_user_geodataframe(filter_dataset): diff --git a/cartoframes/data/observatory/catalog/repository/repo_client.py b/cartoframes/data/observatory/catalog/repository/repo_client.py index 6efb4b888..b828b4c91 100644 --- a/cartoframes/data/observatory/catalog/repository/repo_client.py +++ b/cartoframes/data/observatory/catalog/repository/repo_client.py @@ -8,37 +8,55 @@ class RepoClient: def __init__(self): - self._do_dataset = None - default_credentials = defaults.get_default_credentials() or Credentials(DEFAULT_USER) + default_credentials = Credentials(DEFAULT_USER) default_auth_client = default_credentials.get_api_key_auth_client() self._default_do_dataset = DODataset(auth_client=default_auth_client) + self._user_do_dataset = None + self._external_do_dataset = None def set_user_credentials(self, credentials): if credentials is not None: auth_client = credentials.get_api_key_auth_client() - self._do_dataset = DODataset(auth_client=auth_client) + self._user_do_dataset = DODataset(auth_client=auth_client) else: - self._do_dataset = None + self._user_do_dataset = None def reset_user_credentials(self): - self._do_dataset = None + self._user_do_dataset = None + + def set_external_credentials(self): + # This must be checked every time to allow the definition of + # "default_do_credentials" at any point in the code because + # every repo uses a singleton instance of this client + external_credentials = defaults.get_default_do_credentials() + if external_credentials is not None: + external_auth_client = external_credentials.get_api_key_auth_client() + self._external_do_dataset = DODataset(auth_client=external_auth_client) + else: + self._external_do_dataset = None def get_countries(self, filters=None): + self.set_external_credentials() return self._get_entity('countries', filters) def get_categories(self, filters=None): + self.set_external_credentials() return self._get_entity('categories', filters) def get_providers(self, filters=None): + self.set_external_credentials() return self._get_entity('providers', filters) def get_datasets(self, filters=None): + self.set_external_credentials() return self._get_entity('datasets', filters, use_slug=True) def get_geographies(self, filters=None): + self.set_external_credentials() return self._get_entity('geographies', filters, use_slug=True) def get_variables(self, filters=None): + self.set_external_credentials() filter_id = self._get_filter_id(filters, use_slug=True) if filter_id: return self._fetch_entity_id('variables', filter_id) @@ -47,6 +65,7 @@ def get_variables(self, filters=None): return self._fetch_entity(entity, filters) def get_variables_groups(self, filters=None): + self.set_external_credentials() filter_id = self._get_filter_id(filters, use_slug=True) if filter_id: return self._fetch_entity_id('variables_groups', filter_id) @@ -75,7 +94,5 @@ def _fetch_entity_id(self, entity, filter_id): return self._fetch_entity('{0}/{1}'.format(entity, filter_id)) def _fetch_entity(self, entity, filters=None): - if self._do_dataset: - return self._do_dataset.metadata(entity, filters) - else: - return self._default_do_dataset.metadata(entity, filters) + do_dataset = self._user_do_dataset or self._external_do_dataset or self._default_do_dataset + return do_dataset.metadata(entity, filters) diff --git a/docs/developers/documentation.rst b/docs/developers/documentation.rst index ddd730e83..69e44b3a4 100644 --- a/docs/developers/documentation.rst +++ b/docs/developers/documentation.rst @@ -147,4 +147,19 @@ Custom - DOError. - CatalogError. - EnrichmentError. -- PublishError. \ No newline at end of file +- PublishError. + + +Development with Staging +~~~~~~~~~~~~~~~~~~~~~~~~ + +Before releasing to production we need to test everything in staging. In order to do that, we need to configure CARTOframes to point to staging. +There is a set of internal functions to configure the default DO credentials used by the DO Catalog. + +.. code:: + from cartoframes.auth.defaults import set_default_do_credentials + + set_default_do_credentials(username='USER', base_url='https://ORG.carto-staging.com') + + # After that, every request to the DO Catalog will be done with the provided credentials + # instead of the default ones for production (user 'do-metadata'). \ No newline at end of file diff --git a/tests/unit/data/observatory/catalog/test_catalog.py b/tests/unit/data/observatory/catalog/test_catalog.py index 6fca9a74e..8d0b78037 100644 --- a/tests/unit/data/observatory/catalog/test_catalog.py +++ b/tests/unit/data/observatory/catalog/test_catalog.py @@ -198,7 +198,8 @@ def test_all_filters(self, mocked_datasets): COUNTRY_FILTER: 'usa', CATEGORY_FILTER: 'demographics', PUBLIC_FILTER: 'true', - GEOGRAPHY_FILTER: 'carto-do-public-data.tiger.geography_esp_census_2019'}) + GEOGRAPHY_FILTER: 'carto-do-public-data.tiger.geography_esp_census_2019' + }) assert datasets == test_datasets