From 1311270f23048ce5cb98d4abaf6d5bf4bd52d05a Mon Sep 17 00:00:00 2001 From: Allison Suarez Miranda Date: Tue, 10 May 2022 13:22:38 -0700 Subject: [PATCH 1/2] fix: better behavior for application of filters Signed-off-by: Allison Suarez Miranda --- .../search_service/proxy/es_search_proxy.py | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/search/search_service/proxy/es_search_proxy.py b/search/search_service/proxy/es_search_proxy.py index 5adea5cae6..822c373baa 100644 --- a/search/search_service/proxy/es_search_proxy.py +++ b/search/search_service/proxy/es_search_proxy.py @@ -1,6 +1,7 @@ # Copyright Contributors to the Amundsen project. # SPDX-License-Identifier: Apache-2.0 +import json import logging from typing import ( Any, Dict, List, Union, @@ -198,21 +199,24 @@ def _build_filters(self, resource: Resource, filters: List[Filter]) -> List: filter_queries: List = [] for filter in filters: - filter_name = mapping.get(filter.name) if mapping is not None \ - and mapping.get(filter.name) is not None else filter.name + if mapping is not None and mapping.get(filter.name): + # only apply filter to query if field exists for the given resource + filter_name = mapping.get(filter.name) - queries_per_term = [Q(WILDCARD_QUERY, **{filter_name: term}) for term in filter.values] + queries_per_term = [Q(WILDCARD_QUERY, **{filter_name: term}) for term in filter.values] - if filter.operation == 'OR': - filter_queries.append(Q(BOOL_QUERY, should=queries_per_term, minimum_should_match=1)) + if filter.operation == 'OR': + filter_queries.append(Q(BOOL_QUERY, should=queries_per_term, minimum_should_match=1)) - elif filter.operation == 'AND': - for q in queries_per_term: - filter_queries.append(q) + elif filter.operation == 'AND': + for q in queries_per_term: + filter_queries.append(q) + else: + msg = f"Invalid operation {filter.operation} for filter {filter_name} with values {filter.values}" + raise ValueError(msg) else: - msg = f"Invalid operation {filter.operation} for filter {filter_name} with values {filter.values}" - raise ValueError(msg) + LOGGER.info("Filter {filter.name} does not apply to {resource}") return filter_queries @@ -291,13 +295,13 @@ def execute_queries(self, queries: Dict[Resource, Q], for resource in queries.keys(): query_for_resource = queries.get(resource) search = Search(index=self.get_index_for_resource(resource_type=resource)).query(query_for_resource) - LOGGER.info(search.to_dict()) + # pagination start_from = page_index * results_per_page end = results_per_page * (page_index + 1) search = search[start_from:end] - + LOGGER.info(json.dumps(search.to_dict())) multisearch = multisearch.add(search) try: response = multisearch.execute() From 0d2c7f53b83c524c3e3fc3f79d53804afc7c3cd7 Mon Sep 17 00:00:00 2001 From: Allison Suarez Miranda Date: Tue, 10 May 2022 13:43:50 -0700 Subject: [PATCH 2/2] explicit comparison to None Signed-off-by: Allison Suarez Miranda --- search/search_service/proxy/es_search_proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search/search_service/proxy/es_search_proxy.py b/search/search_service/proxy/es_search_proxy.py index 822c373baa..3c56b08fb9 100644 --- a/search/search_service/proxy/es_search_proxy.py +++ b/search/search_service/proxy/es_search_proxy.py @@ -199,7 +199,7 @@ def _build_filters(self, resource: Resource, filters: List[Filter]) -> List: filter_queries: List = [] for filter in filters: - if mapping is not None and mapping.get(filter.name): + if mapping is not None and mapping.get(filter.name) is not None: # only apply filter to query if field exists for the given resource filter_name = mapping.get(filter.name)