Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Filter only product entities in subscriptions #1723

Merged
merged 2 commits into from
Mar 11, 2021
Merged

Filter only product entities in subscriptions #1723

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e9fde02
Filter product datasets/geographies with metadata entities endpoint
Jesus89 Mar 10, 2021
bfaaf08
Update tests
Jesus89 Mar 10, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cartoframes/data/observatory/catalog/repository/dataset_repo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ def get_all(self, filters=None, credentials=None):

# Using user credentials to fetch entities
self.client.set_user_credentials(credentials)
filters = self.client.get_entities_filters(filters)
entities = self._get_filtered_entities(filters)
self.client.reset_user_credentials()
return entities
Expand Down
1 change: 1 addition & 0 deletions cartoframes/data/observatory/catalog/repository/geography_repo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ def get_all(self, filters=None, credentials=None):

# Using user credentials to fetch entities
self.client.set_user_credentials(credentials)
filters = self.client.get_entities_filters(filters)
entities = self._get_filtered_entities(filters)
self.client.reset_user_credentials()
return entities
Expand Down
7 changes: 7 additions & 0 deletions cartoframes/data/observatory/catalog/repository/repo_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,13 @@ def get_variables_groups(self, filters=None):
entity = 'datasets/{0}/variables_groups'.format(filters.pop('dataset'))
return self._fetch_entity(entity, filters)

def get_entities_filters(self, filters=None):
self.set_external_credentials()
if filters is not None:
filters['filter_catalog'] = False
entities = self._fetch_entity('entities', filters)
return {'id': [result['id'] for result in entities['results']]}

def _get_filter_id(self, filters, use_slug=False):
if isinstance(filters, dict):
filter_id = filters.get('id')
Expand Down
44 changes: 33 additions & 11 deletions tests/unit/data/observatory/catalog/repository/test_dataset_repo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@
class TestDatasetRepo(object):

@patch.object(RepoClient, 'get_datasets')
def test_get_all(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_all(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = [db_dataset1, db_dataset2]
mocked_filters.side_effect = lambda f: f
repo = DatasetRepository()

# When
Expand All @@ -32,10 +34,12 @@ def test_get_all(self, mocked_repo):
assert datasets == test_datasets

@patch.object(RepoClient, 'get_datasets')
@patch.object(RepoClient, 'get_entities_filters')
@patch('cartoframes.data.observatory.catalog.repository.entity_repo.get_subscription_ids')
def test_get_all_credentials(self, mocked_get_subscription_ids, mocked_get_datasets):
def test_get_all_credentials(self, mocked_get_subscription_ids, mocked_filters, mocked_get_datasets):
# Given
mocked_get_subscription_ids.return_value = [db_dataset1['id'], db_dataset2['id']]
mocked_filters.side_effect = lambda f: f
mocked_get_datasets.return_value = [db_dataset1, db_dataset2]
credentials = Credentials('user', '1234')
repo = DatasetRepository()
Expand All @@ -49,9 +53,11 @@ def test_get_all_credentials(self, mocked_get_subscription_ids, mocked_get_datas
assert datasets == test_datasets

@patch.object(RepoClient, 'get_datasets')
def test_get_all_when_empty(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_all_when_empty(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = []
mocked_filters.side_effect = lambda f: f
repo = DatasetRepository()

# When
Expand All @@ -62,9 +68,11 @@ def test_get_all_when_empty(self, mocked_repo):
assert datasets == []

@patch.object(RepoClient, 'get_datasets')
def test_get_all_only_uses_allowed_filters(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_all_only_uses_allowed_filters(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = [db_dataset1, db_dataset2]
mocked_filters.side_effect = lambda f: f
repo = DatasetRepository()
filters = {
COUNTRY_FILTER: 'usa',
Expand All @@ -90,9 +98,11 @@ def test_get_all_only_uses_allowed_filters(self, mocked_repo):
assert datasets == test_datasets

@patch.object(RepoClient, 'get_datasets')
def test_get_by_id(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_by_id(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = [db_dataset1]
mocked_filters.side_effect = lambda f: f
requested_id = db_dataset1['id']
repo = DatasetRepository()

Expand All @@ -104,9 +114,11 @@ def test_get_by_id(self, mocked_repo):
assert dataset == test_dataset1

@patch.object(RepoClient, 'get_datasets')
def test_get_by_id_unknown_fails(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_by_id_unknown_fails(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = []
mocked_filters.side_effect = lambda f: f
requested_id = 'unknown_id'
repo = DatasetRepository()

Expand All @@ -115,9 +127,11 @@ def test_get_by_id_unknown_fails(self, mocked_repo):
repo.get_by_id(requested_id)

@patch.object(RepoClient, 'get_datasets')
def test_get_by_slug(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_by_slug(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = [db_dataset1]
mocked_filters.side_effect = lambda f: f
requested_slug = db_dataset1['slug']
repo = DatasetRepository()

Expand All @@ -129,9 +143,11 @@ def test_get_by_slug(self, mocked_repo):
assert dataset == test_dataset1

@patch.object(RepoClient, 'get_datasets')
def test_get_by_id_list(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_by_id_list(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = [db_dataset1, db_dataset2]
mocked_filters.side_effect = lambda f: f
repo = DatasetRepository()

# When
Expand All @@ -143,9 +159,11 @@ def test_get_by_id_list(self, mocked_repo):
assert datasets == test_datasets

@patch.object(RepoClient, 'get_datasets')
def test_get_by_slug_list(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_by_slug_list(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = [db_dataset1, db_dataset2]
mocked_filters.side_effect = lambda f: f
repo = DatasetRepository()

# When
Expand All @@ -157,9 +175,11 @@ def test_get_by_slug_list(self, mocked_repo):
assert datasets == test_datasets

@patch.object(RepoClient, 'get_datasets')
def test_get_by_slug_and_id_list(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_by_slug_and_id_list(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = [db_dataset1, db_dataset2]
mocked_filters.side_effect = lambda f: f
repo = DatasetRepository()

# When
Expand All @@ -171,9 +191,11 @@ def test_get_by_slug_and_id_list(self, mocked_repo):
assert datasets == test_datasets

@patch.object(RepoClient, 'get_datasets')
def test_missing_fields_are_mapped_as_None(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_missing_fields_are_mapped_as_None(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = [{'id': 'dataset1'}]
mocked_filters.side_effect = lambda f: f
repo = DatasetRepository()

expected_datasets = CatalogList([Dataset({
Expand Down
48 changes: 36 additions & 12 deletions tests/unit/data/observatory/catalog/repository/test_geography_repo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@
class TestGeographyRepo(object):

@patch.object(RepoClient, 'get_geographies')
def test_get_all(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_all(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = [db_geography1, db_geography2]
mocked_filters.side_effect = lambda f: f
repo = GeographyRepository()

# When
Expand All @@ -32,10 +34,12 @@ def test_get_all(self, mocked_repo):
assert geographies == test_geographies

@patch.object(RepoClient, 'get_geographies')
@patch.object(RepoClient, 'get_entities_filters')
@patch('cartoframes.data.observatory.catalog.repository.entity_repo.get_subscription_ids')
def test_get_all_credentials(self, mocked_get_subscription_ids, mocked_get_geographies):
def test_get_all_credentials(self, mocked_get_subscription_ids, mocked_filters, mocked_get_geographies):
# Given
mocked_get_subscription_ids.return_value = [db_geography1['id'], db_geography2['id']]
mocked_filters.side_effect = lambda f: f
mocked_get_geographies.return_value = [db_geography1, db_geography2]
credentials = Credentials('user', '1234')
repo = GeographyRepository()
Expand All @@ -49,9 +53,11 @@ def test_get_all_credentials(self, mocked_get_subscription_ids, mocked_get_geogr
assert geographies == test_geographies

@patch.object(RepoClient, 'get_geographies')
def test_get_all_when_empty(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_all_when_empty(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = []
mocked_filters.side_effect = lambda f: f
repo = GeographyRepository()

# When
Expand All @@ -62,9 +68,11 @@ def test_get_all_when_empty(self, mocked_repo):
assert geographies == []

@patch.object(RepoClient, 'get_geographies')
def test_get_all_only_uses_allowed_filters(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_all_only_uses_allowed_filters(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = [db_geography1, db_geography2]
mocked_filters.side_effect = lambda f: f
repo = GeographyRepository()
filters = {
COUNTRY_FILTER: 'usa',
Expand All @@ -89,9 +97,11 @@ def test_get_all_only_uses_allowed_filters(self, mocked_repo):
assert geographies == test_geographies

@patch.object(RepoClient, 'get_geographies')
def test_get_by_id(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_by_id(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = [db_geography1]
mocked_filters.side_effect = lambda f: f
requested_id = db_geography1['id']
repo = GeographyRepository()

Expand All @@ -104,9 +114,11 @@ def test_get_by_id(self, mocked_repo):
assert geography == test_geography1

@patch.object(RepoClient, 'get_geographies')
def test_get_by_id_unknown_fails(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_by_id_unknown_fails(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = []
mocked_filters.side_effect = lambda f: f
requested_id = 'unknown_id'
repo = GeographyRepository()

Expand All @@ -115,9 +127,11 @@ def test_get_by_id_unknown_fails(self, mocked_repo):
repo.get_by_id(requested_id)

@patch.object(RepoClient, 'get_geographies')
def test_get_by_slug(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_by_slug(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = [db_geography1]
mocked_filters.side_effect = lambda f: f
requested_slug = db_geography1['slug']
repo = GeographyRepository()

Expand All @@ -129,9 +143,11 @@ def test_get_by_slug(self, mocked_repo):
assert geography == test_geography1

@patch.object(RepoClient, 'get_geographies')
def test_get_by_id_list(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_by_id_list(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = [db_geography1, db_geography2]
mocked_filters.side_effect = lambda f: f
repo = GeographyRepository()

# When
Expand All @@ -143,9 +159,11 @@ def test_get_by_id_list(self, mocked_repo):
assert geographies == test_geographies

@patch.object(RepoClient, 'get_geographies')
def test_get_by_slug_list(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_by_slug_list(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = [db_geography1, db_geography2]
mocked_filters.side_effect = lambda f: f
repo = GeographyRepository()

# When
Expand All @@ -157,9 +175,11 @@ def test_get_by_slug_list(self, mocked_repo):
assert geographies == test_geographies

@patch.object(RepoClient, 'get_geographies')
def test_get_by_slug_and_id_list(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_by_slug_and_id_list(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = [db_geography1, db_geography2]
mocked_filters.side_effect = lambda f: f
repo = GeographyRepository()

# When
Expand All @@ -171,9 +191,11 @@ def test_get_by_slug_and_id_list(self, mocked_repo):
assert geographies == test_geographies

@patch.object(RepoClient, 'get_geographies')
def test_get_all_with_join_filters(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_get_all_with_join_filters(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = [db_geography1, db_geography2]
mocked_filters.side_effect = lambda f: f
repo = GeographyRepository()

# When
Expand All @@ -185,9 +207,11 @@ def test_get_all_with_join_filters(self, mocked_repo):
assert geographies == test_geographies

@patch.object(RepoClient, 'get_geographies')
def test_missing_fields_are_mapped_as_None(self, mocked_repo):
@patch.object(RepoClient, 'get_entities_filters')
def test_missing_fields_are_mapped_as_None(self, mocked_filters, mocked_repo):
# Given
mocked_repo.return_value = [{'id': 'geography1'}]
mocked_filters.side_effect = lambda f: f
repo = GeographyRepository()

expected_geographies = CatalogList([Geography({
Expand Down