From 023a71d9f8d00c12b7170bbe1973769c0ec7926b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Arroyo=20Torrens?= Date: Tue, 28 Apr 2020 08:38:56 +0200 Subject: [PATCH 1/4] Add Catalog.providers --- .../data/observatory/catalog/catalog.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/cartoframes/data/observatory/catalog/catalog.py b/cartoframes/data/observatory/catalog/catalog.py index 8f4086b0a..f4ef93cbe 100644 --- a/cartoframes/data/observatory/catalog/catalog.py +++ b/cartoframes/data/observatory/catalog/catalog.py @@ -1,7 +1,8 @@ -from .dataset import Dataset from .entity import is_slug_value -from .category import Category from .country import Country +from .category import Category +from .provider import Provider +from .dataset import Dataset from .geography import Geography from .subscriptions import Subscriptions from .repository.constants import COUNTRY_FILTER, CATEGORY_FILTER, GEOGRAPHY_FILTER, PROVIDER_FILTER @@ -143,6 +144,19 @@ def categories(self): """ return Category.get_all(self.filters) + @property + def providers(self): + """Get all the providers in the Catalog. + + Returns: + :py:class:`CatalogList ` + + Raises: + CatalogError: if there's a problem when connecting to the catalog or no datasets are found. + + """ + return Provider.get_all(self.filters) + @property def datasets(self): """Get all the datasets in the Catalog. From 1e51a374704b4edc85ba9e4ad26c9a88bd6c9568 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Arroyo=20Torrens?= Date: Tue, 28 Apr 2020 08:46:47 +0200 Subject: [PATCH 2/4] Add test --- .../data/observatory/catalog/test_catalog.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/unit/data/observatory/catalog/test_catalog.py b/tests/unit/data/observatory/catalog/test_catalog.py index 306354f04..f104df814 100644 --- a/tests/unit/data/observatory/catalog/test_catalog.py +++ b/tests/unit/data/observatory/catalog/test_catalog.py @@ -7,6 +7,7 @@ from cartoframes.data.observatory.catalog.geography import Geography from cartoframes.data.observatory.catalog.country import Country from cartoframes.data.observatory.catalog.category import Category +from cartoframes.data.observatory.catalog.provider import Provider from cartoframes.data.observatory.catalog.catalog import Catalog from cartoframes.data.observatory.catalog.subscriptions import Subscriptions from cartoframes.data.observatory.catalog.repository.geography_repo import GeographyRepository @@ -15,7 +16,8 @@ ) from .examples import ( test_country2, test_country1, test_category1, test_category2, test_dataset1, test_dataset2, - test_geographies, test_datasets, test_categories, test_countries, test_geography1, test_geography2 + test_geographies, test_datasets, test_categories, test_countries, test_geography1, test_geography2, + test_provider1, test_provider2 ) @@ -47,6 +49,19 @@ def test_categories(self, mocked_categories): # Then assert categories == expected_categories + @patch.object(Provider, 'get_all') + def test_providers(self, mocked_providers): + # Given + expected_providers = [test_provider1, test_provider2] + mocked_providers.return_value = expected_providers + catalog = Catalog() + + # When + providers = catalog.providers + + # Then + assert providers == expected_providers + @patch.object(Dataset, 'get_all') def test_datasets(self, mocked_datasets): # Given From 20a8ab0b1d8417fb37bc525349fd31641ff8b7ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Arroyo=20Torrens?= Date: Thu, 30 Apr 2020 11:50:07 +0200 Subject: [PATCH 3/4] Update allowed filters in the catalog --- .../data/observatory/catalog/repository/category_repo.py | 4 ++-- .../data/observatory/catalog/repository/country_repo.py | 4 ++-- .../data/observatory/catalog/repository/dataset_repo.py | 4 ++-- .../data/observatory/catalog/repository/geography_repo.py | 4 ++-- .../data/observatory/catalog/repository/provider_repo.py | 4 +++- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/cartoframes/data/observatory/catalog/repository/category_repo.py b/cartoframes/data/observatory/catalog/repository/category_repo.py index 624f97a51..0a011400c 100644 --- a/cartoframes/data/observatory/catalog/repository/category_repo.py +++ b/cartoframes/data/observatory/catalog/repository/category_repo.py @@ -1,9 +1,9 @@ -from .constants import COUNTRY_FILTER +from .constants import COUNTRY_FILTER, PROVIDER_FILTER from .entity_repo import EntityRepository _CATEGORY_ID_FIELD = 'id' -_ALLOWED_FILTERS = [COUNTRY_FILTER] +_ALLOWED_FILTERS = [COUNTRY_FILTER, PROVIDER_FILTER] def get_category_repo(): diff --git a/cartoframes/data/observatory/catalog/repository/country_repo.py b/cartoframes/data/observatory/catalog/repository/country_repo.py index 84d885cbb..1717f4890 100644 --- a/cartoframes/data/observatory/catalog/repository/country_repo.py +++ b/cartoframes/data/observatory/catalog/repository/country_repo.py @@ -1,9 +1,9 @@ -from .constants import CATEGORY_FILTER +from .constants import CATEGORY_FILTER, PROVIDER_FILTER from .entity_repo import EntityRepository _COUNTRY_ID_FIELD = 'id' -_ALLOWED_FILTERS = [CATEGORY_FILTER] +_ALLOWED_FILTERS = [CATEGORY_FILTER, PROVIDER_FILTER] def get_country_repo(): diff --git a/cartoframes/data/observatory/catalog/repository/dataset_repo.py b/cartoframes/data/observatory/catalog/repository/dataset_repo.py index b50bc1ba1..b74143f89 100644 --- a/cartoframes/data/observatory/catalog/repository/dataset_repo.py +++ b/cartoframes/data/observatory/catalog/repository/dataset_repo.py @@ -1,11 +1,11 @@ -from .constants import CATEGORY_FILTER, COUNTRY_FILTER, GEOGRAPHY_FILTER, PROVIDER_FILTER +from .constants import CATEGORY_FILTER, COUNTRY_FILTER, PROVIDER_FILTER, GEOGRAPHY_FILTER from .entity_repo import EntityRepository DATASET_TYPE = 'dataset' _DATASET_ID_FIELD = 'id' _DATASET_SLUG_FIELD = 'slug' -_ALLOWED_FILTERS = [CATEGORY_FILTER, COUNTRY_FILTER, GEOGRAPHY_FILTER, PROVIDER_FILTER] +_ALLOWED_FILTERS = [CATEGORY_FILTER, COUNTRY_FILTER, PROVIDER_FILTER, GEOGRAPHY_FILTER] def get_dataset_repo(): diff --git a/cartoframes/data/observatory/catalog/repository/geography_repo.py b/cartoframes/data/observatory/catalog/repository/geography_repo.py index 85da8d87a..cd4cb04b6 100644 --- a/cartoframes/data/observatory/catalog/repository/geography_repo.py +++ b/cartoframes/data/observatory/catalog/repository/geography_repo.py @@ -1,14 +1,14 @@ from geopandas import GeoDataFrame from .....utils.geom_utils import set_geometry -from .constants import COUNTRY_FILTER, CATEGORY_FILTER, PROVIDER_FILTER +from .constants import CATEGORY_FILTER, COUNTRY_FILTER, PROVIDER_FILTER from .entity_repo import EntityRepository GEOGRAPHY_TYPE = 'geography' _GEOGRAPHY_ID_FIELD = 'id' _GEOGRAPHY_SLUG_FIELD = 'slug' -_ALLOWED_FILTERS = [COUNTRY_FILTER, CATEGORY_FILTER, PROVIDER_FILTER] +_ALLOWED_FILTERS = [CATEGORY_FILTER, COUNTRY_FILTER, PROVIDER_FILTER] def get_geography_repo(): diff --git a/cartoframes/data/observatory/catalog/repository/provider_repo.py b/cartoframes/data/observatory/catalog/repository/provider_repo.py index 722165860..acdc9bcba 100644 --- a/cartoframes/data/observatory/catalog/repository/provider_repo.py +++ b/cartoframes/data/observatory/catalog/repository/provider_repo.py @@ -1,7 +1,9 @@ +from .constants import CATEGORY_FILTER, COUNTRY_FILTER from .entity_repo import EntityRepository _PROVIDER_ID_FIELD = 'id' +_ALLOWED_FILTERS = [CATEGORY_FILTER, COUNTRY_FILTER] def get_provider_repo(): @@ -11,7 +13,7 @@ def get_provider_repo(): class ProviderRepository(EntityRepository): def __init__(self): - super(ProviderRepository, self).__init__(_PROVIDER_ID_FIELD, []) + super(ProviderRepository, self).__init__(_PROVIDER_ID_FIELD, _ALLOWED_FILTERS) @classmethod def _get_entity_class(cls): From 091e672ecc8f5848aae9762f7a152739d392b4b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Arroyo=20Torrens?= Date: Thu, 30 Apr 2020 11:55:28 +0200 Subject: [PATCH 4/4] Add catalog repo unit tests --- .../catalog/repository/test_category_repo.py | 3 +- .../catalog/repository/test_country_repo.py | 3 +- .../catalog/repository/test_provider_repo.py | 29 +++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/tests/unit/data/observatory/catalog/repository/test_category_repo.py b/tests/unit/data/observatory/catalog/repository/test_category_repo.py index 6c7bae285..5a3a138c7 100644 --- a/tests/unit/data/observatory/catalog/repository/test_category_repo.py +++ b/tests/unit/data/observatory/catalog/repository/test_category_repo.py @@ -63,7 +63,8 @@ def test_get_all_only_uses_allowed_filters(self, mocked_repo): # Then mocked_repo.assert_called_once_with({ - COUNTRY_FILTER: 'usa' + COUNTRY_FILTER: 'usa', + PROVIDER_FILTER: 'open_data' }) assert categories == test_categories diff --git a/tests/unit/data/observatory/catalog/repository/test_country_repo.py b/tests/unit/data/observatory/catalog/repository/test_country_repo.py index f6adf2d37..7fd71e2cf 100644 --- a/tests/unit/data/observatory/catalog/repository/test_country_repo.py +++ b/tests/unit/data/observatory/catalog/repository/test_country_repo.py @@ -62,7 +62,8 @@ def test_get_all_only_uses_allowed_filters(self, mocked_repo): # Then mocked_repo.assert_called_once_with({ - CATEGORY_FILTER: 'demographics' + CATEGORY_FILTER: 'demographics', + PROVIDER_FILTER: 'open_data' }) assert countries == test_countries diff --git a/tests/unit/data/observatory/catalog/repository/test_provider_repo.py b/tests/unit/data/observatory/catalog/repository/test_provider_repo.py index 828c8c655..709d6aa3d 100644 --- a/tests/unit/data/observatory/catalog/repository/test_provider_repo.py +++ b/tests/unit/data/observatory/catalog/repository/test_provider_repo.py @@ -7,6 +7,10 @@ from cartoframes.data.observatory.catalog.provider import Provider from cartoframes.data.observatory.catalog.repository.provider_repo import ProviderRepository from cartoframes.data.observatory.catalog.repository.repo_client import RepoClient +from cartoframes.data.observatory.catalog.repository.constants import ( + CATEGORY_FILTER, COUNTRY_FILTER, DATASET_FILTER, GEOGRAPHY_FILTER, VARIABLE_FILTER, + VARIABLE_GROUP_FILTER +) from ..examples import test_provider1, test_providers, db_provider1, db_provider2 @@ -39,6 +43,31 @@ def test_get_all_when_empty(self, mocked_repo): mocked_repo.assert_called_once_with(None) assert providers is None + @patch.object(RepoClient, 'get_providers') + def test_get_all_only_uses_allowed_filters(self, mocked_repo): + # Given + mocked_repo.return_value = [db_provider1, db_provider2] + repo = ProviderRepository() + filters = { + COUNTRY_FILTER: 'usa', + DATASET_FILTER: 'carto-do.project.census2011', + CATEGORY_FILTER: 'demographics', + VARIABLE_FILTER: 'population', + GEOGRAPHY_FILTER: 'census-geo', + VARIABLE_GROUP_FILTER: 'var-group', + 'fake_field_id': 'fake_value' + } + + # When + providers = repo.get_all(filters) + + # Then + mocked_repo.assert_called_once_with({ + COUNTRY_FILTER: 'usa', + CATEGORY_FILTER: 'demographics' + }) + assert providers == test_providers + @patch.object(RepoClient, 'get_providers') def test_get_by_id(self, mocked_repo): # Given