Skip to content

Commit

Permalink
Backend support for domain hierarchy listing
Browse files Browse the repository at this point in the history
  • Loading branch information
@sonika-shah
sonika-shah committed Dec 30, 2024
1 parent 24b668a commit 236747b
Show file tree
Hide file tree
Showing 7 changed files with 537 additions and 101 deletions.
49 changes: 49 additions & 0 deletions openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,15 @@
import static org.openmetadata.service.Entity.DOMAIN;
import static org.openmetadata.service.Entity.FIELD_ASSETS;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.jdbi.v3.sqlobject.transaction.Transaction;
import org.openmetadata.schema.EntityInterface;
import org.openmetadata.schema.entity.data.EntityHierarchy;
import org.openmetadata.schema.entity.domains.Domain;
import org.openmetadata.schema.type.EntityReference;
import org.openmetadata.schema.type.Include;
Expand All @@ -30,8 +35,11 @@
import org.openmetadata.schema.type.api.BulkOperationResult;
import org.openmetadata.service.Entity;
import org.openmetadata.service.resources.domains.DomainResource;
import org.openmetadata.service.util.EntityUtil;
import org.openmetadata.service.util.EntityUtil.Fields;
import org.openmetadata.service.util.FullyQualifiedName;
import org.openmetadata.service.util.JsonUtils;
import org.openmetadata.service.util.ResultList;

@Slf4j
public class DomainRepository extends EntityRepository<Domain> {
Expand Down Expand Up @@ -140,6 +148,47 @@ public EntityInterface getParentEntity(Domain entity, String fields) {
: null;
}

public List<EntityHierarchy> buildHierarchy(String fieldsParam, int limit) {
fieldsParam = EntityUtil.addField(fieldsParam, Entity.FIELD_PARENT);
Fields fields = getFields(fieldsParam);
ResultList<Domain> resultList = listAfter(null, fields, new ListFilter(null), limit, null);
List<Domain> domains = resultList.getData();

/*
Maintaining hierarchy in terms of EntityHierarchy to get all other fields of Domain like style,
which would have been restricted if built using hierarchy of Domain, as Domain.getChildren() returns List<EntityReference>
and EntityReference does not support additional properties
*/
List<EntityHierarchy> rootDomains = new ArrayList<>();

Map<UUID, EntityHierarchy> entityHierarchyMap =
domains.stream()
.collect(
Collectors.toMap(
Domain::getId,
domain -> {
EntityHierarchy entityHierarchy =
JsonUtils.readValue(JsonUtils.pojoToJson(domain), EntityHierarchy.class);
entityHierarchy.setChildren(new ArrayList<>());
return entityHierarchy;
}));

for (Domain domain : domains) {
EntityHierarchy entityHierarchy = entityHierarchyMap.get(domain.getId());

if (domain.getParent() != null) {
EntityHierarchy parentHierarchy = entityHierarchyMap.get(domain.getParent().getId());
if (parentHierarchy != null) {
parentHierarchy.getChildren().add(entityHierarchy);
}
} else {
rootDomains.add(entityHierarchy);
}
}

return rootDomains;
}

public class DomainUpdater extends EntityUpdater {
public DomainUpdater(Domain original, Domain updated, Operation operation) {
super(original, updated, operation);
Expand Down
2 changes: 1 addition & 1 deletion openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2620,7 +2620,7 @@ public boolean isDelete() {
* version goes to v-1 and new version v0 replaces v1 for the entity.
* </ol>
*
* @see TableRepository.TableUpdater#entitySpecificUpdate() for example.
* @see TableRepository.TableUpdater #entitySpecificUpdate() for example.
*/
public class EntityUpdater {
private static volatile long sessionTimeoutMillis = 10L * 60 * 1000; // 10 minutes
Expand Down
30 changes: 30 additions & 0 deletions ...data-service/src/main/java/org/openmetadata/service/resources/domains/DomainResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import javax.ws.rs.core.UriInfo;
import lombok.extern.slf4j.Slf4j;
import org.openmetadata.schema.api.domains.CreateDomain;
import org.openmetadata.schema.entity.data.EntityHierarchy;
import org.openmetadata.schema.entity.domains.Domain;
import org.openmetadata.schema.type.ChangeEvent;
import org.openmetadata.schema.type.EntityHistory;
Expand All @@ -57,6 +58,7 @@
import org.openmetadata.service.resources.Collection;
import org.openmetadata.service.resources.EntityResource;
import org.openmetadata.service.security.Authorizer;
import org.openmetadata.service.util.EntityHierarchyList;
import org.openmetadata.service.util.ResultList;

@Slf4j
Expand Down Expand Up @@ -433,4 +435,32 @@ public Response delete(
String name) {
return deleteByName(uriInfo, securityContext, name, true, true);
}

@GET
@Path("/hierarchy")
@Operation(
operationId = "listDomainsHierarchy",
summary = "List domains in hierarchical order",
description = "Get a list of Domains in hierarchical order.",
responses = {
@ApiResponse(
responseCode = "200",
description = "List of Domains in hierarchical order",
content =
@Content(
mediaType = "application/json",
schema = @Schema(implementation = EntityHierarchyList.class)))
})
public ResultList<EntityHierarchy> listHierarchy(
@Context UriInfo uriInfo,
@Context SecurityContext securityContext,
@Parameter(
description = "Fields requested in the returned resource",
schema = @Schema(type = "string", example = FIELDS))
@QueryParam("fields")
String fieldsParam,
@DefaultValue("10") @Min(0) @Max(1000000) @QueryParam("limit") int limitParam) {

return new EntityHierarchyList(repository.buildHierarchy(fieldsParam, limitParam));
}
}
166 changes: 117 additions & 49 deletions ...vice/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.json.JsonObject;
import javax.net.ssl.SSLContext;
Expand Down Expand Up @@ -468,50 +469,10 @@ public Response search(SearchRequest request, SubjectContext subjectContext) thr
.getIndexMapping(GLOSSARY_TERM)
.getIndexName(clusterAlias))) {
searchSourceBuilder.query(QueryBuilders.boolQuery().must(searchSourceBuilder.query()));

if (request.isGetHierarchy()) {
QueryBuilder baseQuery =
QueryBuilders.boolQuery()
.should(searchSourceBuilder.query())
.should(QueryBuilders.matchPhraseQuery("fullyQualifiedName", request.getQuery()))
.should(QueryBuilders.matchPhraseQuery("name", request.getQuery()))
.should(QueryBuilders.matchPhraseQuery("displayName", request.getQuery()))
.should(
QueryBuilders.matchPhraseQuery(
"glossary.fullyQualifiedName", request.getQuery()))
.should(QueryBuilders.matchPhraseQuery("glossary.displayName", request.getQuery()))
.must(QueryBuilders.matchQuery("status", "Approved"))
.minimumShouldMatch(1);
searchSourceBuilder.query(baseQuery);

SearchResponse searchResponse =
client.search(
new es.org.elasticsearch.action.search.SearchRequest(request.getIndex())
.source(searchSourceBuilder),
RequestOptions.DEFAULT);

// Extract parent terms from aggregation
BoolQueryBuilder parentTermQueryBuilder = QueryBuilders.boolQuery();
Terms parentTerms = searchResponse.getAggregations().get("fqnParts_agg");

// Build es query to get parent terms for the user input query , to build correct hierarchy
if (!parentTerms.getBuckets().isEmpty() && !request.getQuery().equals("*")) {
parentTerms.getBuckets().stream()
.map(Terms.Bucket::getKeyAsString)
.forEach(
parentTerm ->
parentTermQueryBuilder.should(
QueryBuilders.matchQuery("fullyQualifiedName", parentTerm)));

searchSourceBuilder.query(
parentTermQueryBuilder
.minimumShouldMatch(1)
.must(QueryBuilders.matchQuery("status", "Approved")));
}
searchSourceBuilder.sort(SortBuilders.fieldSort("fullyQualifiedName").order(SortOrder.ASC));
}
}

buildHierarchyQuery(request, searchSourceBuilder, client);

/* for performance reasons ElasticSearch doesn't provide accurate hits
if we enable trackTotalHits parameter it will try to match every result, count and return hits
however in most cases for search results an approximate value is good enough.
Expand Down Expand Up @@ -579,15 +540,89 @@ public Response getDocByID(String indexName, String entityId) throws IOException
return getResponse(NOT_FOUND, "Document not found.");
}

private void buildHierarchyQuery(
SearchRequest request, SearchSourceBuilder searchSourceBuilder, RestHighLevelClient client)
throws IOException {

if (!request.isGetHierarchy()) {
return;
}

String indexName = request.getIndex();
String glossaryTermIndex =
Entity.getSearchRepository().getIndexMapping(GLOSSARY_TERM).getIndexName(clusterAlias);
String domainIndex =
Entity.getSearchRepository().getIndexMapping(DOMAIN).getIndexName(clusterAlias);

BoolQueryBuilder baseQuery =
QueryBuilders.boolQuery()
.should(searchSourceBuilder.query())
.should(QueryBuilders.matchPhraseQuery("fullyQualifiedName", request.getQuery()))
.should(QueryBuilders.matchPhraseQuery("name", request.getQuery()))
.should(QueryBuilders.matchPhraseQuery("displayName", request.getQuery()));

if (indexName.equalsIgnoreCase(glossaryTermIndex)) {
baseQuery
.should(QueryBuilders.matchPhraseQuery("glossary.fullyQualifiedName", request.getQuery()))
.should(QueryBuilders.matchPhraseQuery("glossary.displayName", request.getQuery()))
.must(QueryBuilders.matchQuery("status", "Approved"));
} else if (indexName.equalsIgnoreCase(domainIndex)) {
baseQuery
.should(QueryBuilders.matchPhraseQuery("parent.fullyQualifiedName", request.getQuery()))
.should(QueryBuilders.matchPhraseQuery("parent.displayName", request.getQuery()));
}

baseQuery.minimumShouldMatch(1);
searchSourceBuilder.query(baseQuery);

SearchResponse searchResponse =
client.search(
new es.org.elasticsearch.action.search.SearchRequest(request.getIndex())
.source(searchSourceBuilder),
RequestOptions.DEFAULT);

Terms parentTerms = searchResponse.getAggregations().get("fqnParts_agg");

// Build es query to get parent terms for the user input query , to build correct hierarchy
// In case of default search , no need to get parent terms they are already present in the
// response
if (parentTerms != null
&& !parentTerms.getBuckets().isEmpty()
&& !request.getQuery().equals("*")) {
BoolQueryBuilder parentTermQueryBuilder = QueryBuilders.boolQuery();

parentTerms.getBuckets().stream()
.map(Terms.Bucket::getKeyAsString)
.forEach(
parentTerm ->
parentTermQueryBuilder.should(
QueryBuilders.matchQuery("fullyQualifiedName", parentTerm)));
if (indexName.equalsIgnoreCase(glossaryTermIndex)) {
parentTermQueryBuilder
.minimumShouldMatch(1)
.must(QueryBuilders.matchQuery("status", "Approved"));
} else {
parentTermQueryBuilder.minimumShouldMatch(1);
}
searchSourceBuilder.query(parentTermQueryBuilder);
}

searchSourceBuilder.sort(SortBuilders.fieldSort("fullyQualifiedName").order(SortOrder.ASC));
}

public List<?> buildSearchHierarchy(SearchRequest request, SearchResponse searchResponse) {
List<?> response = new ArrayList<>();
if (request
.getIndex()
.equalsIgnoreCase(
Entity.getSearchRepository()
.getIndexMapping(GLOSSARY_TERM)
.getIndexName(clusterAlias))) {

String indexName = request.getIndex();
String glossaryTermIndex =
Entity.getSearchRepository().getIndexMapping(GLOSSARY_TERM).getIndexName(clusterAlias);
String domainIndex =
Entity.getSearchRepository().getIndexMapping(DOMAIN).getIndexName(clusterAlias);

if (indexName.equalsIgnoreCase(glossaryTermIndex)) {
response = buildGlossaryTermSearchHierarchy(searchResponse);
} else if (indexName.equalsIgnoreCase(domainIndex)) {
response = buildDomainSearchHierarchy(searchResponse);
}
return response;
}
Expand Down Expand Up @@ -643,6 +678,36 @@ public List<EntityHierarchy> buildGlossaryTermSearchHierarchy(SearchResponse sea
return new ArrayList<>(rootTerms.values());
}

public List<EntityHierarchy> buildDomainSearchHierarchy(SearchResponse searchResponse) {
Map<String, EntityHierarchy> entityHierarchyMap =
Arrays.stream(searchResponse.getHits().getHits())
.map(hit -> JsonUtils.readValue(hit.getSourceAsString(), EntityHierarchy.class))
.collect(
Collectors.toMap(
EntityHierarchy::getFullyQualifiedName,
entity -> {
entity.setChildren(new ArrayList<>());
return entity;
}));

List<EntityHierarchy> rootDomains = new ArrayList<>();

entityHierarchyMap
.values()
.forEach(
entity -> {
String parentFqn = getParentFQN(entity.getFullyQualifiedName());
EntityHierarchy parentEntity = entityHierarchyMap.get(parentFqn);
if (parentEntity != null) {
parentEntity.getChildren().add(entity);
} else {
rootDomains.add(entity);
}
});

return rootDomains;
}

@Override
public SearchResultListMapper listWithOffset(
String filter,
Expand Down Expand Up @@ -1797,7 +1862,10 @@ private static SearchSourceBuilder buildDomainsSearch(String query, int from, in
buildSearchQueryBuilder(query, DomainIndex.getFields());
FunctionScoreQueryBuilder queryBuilder = boostScore(queryStringBuilder);
HighlightBuilder hb = buildHighlights(new ArrayList<>());
return searchBuilder(queryBuilder, hb, from, size);
SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, hb, from, size);
searchSourceBuilder.aggregation(
AggregationBuilders.terms("fqnParts_agg").field("fqnParts").size(1000));
return addAggregation(searchSourceBuilder);
}

private static SearchSourceBuilder buildCostAnalysisReportDataSearch(
Expand Down
Loading

0 comments on commit 236747b

Please sign in to comment.