Skip to content
This repository has been archived by the owner on Dec 22, 2022. It is now read-only.

pds-api-56: crawl the hierarchical tree #29

Merged
merged 15 commits into from
May 20, 2021
Merged

pds-api-56: crawl the hierarchical tree #29

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
bf0c6da
start with the acceptance criteria being checked
May 11, 2021
fc3faf2
cosmetic changes
May 11, 2021
d387c7d
update the API
May 12, 2021
fc6d4e8
basics for products of a bundle
May 12, 2021
316a625
now get products of a bundle. works and tested
May 12, 2021
d93d08b
prepare for finding owners
May 18, 2021
fa85dcb
implement bundle from collection
May 18, 2021
131b3e5
editorial
May 18, 2021
08d4e93
fix what should not have changed
May 18, 2021
1b1a898
find bundles of a product
May 18, 2021
9c28602
collections that reference product
May 18, 2021
007eac3
fix whitespacing
May 19, 2021
d2131bf
Merge branch 'master' of github.com:NASA-PDS/registry-api-service int…
May 19, 2021
8b720fd
get actual collection
May 19, 2021
516ad80
do negative checks
May 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@
<dependency>
<groupId>gov.nasa.pds</groupId>
<artifactId>api</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.2.0-SNAPSHOT</version>
</dependency>

<dependency>
Expand Down
136 changes: 122 additions & 14 deletions src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchUtil;
import gov.nasa.pds.api.engineering.elasticsearch.entities.EntityProduct;
import gov.nasa.pds.api.model.ProductWithXmlLabel;
import gov.nasa.pds.model.ErrorMessage;
import gov.nasa.pds.model.Product;
import gov.nasa.pds.model.Products;
import gov.nasa.pds.model.Summary;
Expand All @@ -22,15 +21,9 @@
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.CookieValue;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.multipart.MultipartFile;

import javax.validation.constraints.*;
import javax.validation.Valid;
import javax.servlet.http.HttpServletRequest;
import java.io.IOException;
Expand Down Expand Up @@ -82,7 +75,9 @@ public ResponseEntity<Products> collectionsOfABundle(@ApiParam(value = "lidvid (
return this.getBundlesCollections(lidvid, start, limit, fields, sort, onlySummary);
}

private Products getCollectionChildren(String lidvid, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) throws IOException {
private Products getCollectionChildren(String lidvid, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) throws IOException
{
if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid);
MyBundlesApiController.log.info("request bundle lidvid, collections children: " + lidvid);

GetRequest getBundleRequest = new GetRequest(this.esRegistryConnection.getRegistryIndex(),
Expand Down Expand Up @@ -113,7 +108,8 @@ private Products getCollectionChildren(String lidvid, int start, int limit, List

if (getBundleResponse.isExists()) {
MyBundlesApiController.log.info("get response " + getBundleResponse.toString());
List<String> collections = (List<String>) getBundleResponse.getSourceAsMap().get("ref_lid_collection");
@SuppressWarnings("unchecked")
List<String> collections = (List<String>) getBundleResponse.getSourceAsMap().get("ref_lid_collection");
String collectionLidVid;
int i=0;
for (String collectionLid : collections ) {
Expand Down Expand Up @@ -168,9 +164,7 @@ private Products getCollectionChildren(String lidvid, int start, int limit, List
}

}




private ResponseEntity<Products> getBundlesCollections(String lidvid, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) {
String accept = this.request.getHeader("Accept");
MyBundlesApiController.log.info("accept value is " + accept);
Expand All @@ -196,8 +190,122 @@ private ResponseEntity<Products> getBundlesCollections(String lidvid, int start,

}
else return new ResponseEntity<Products>(HttpStatus.NOT_IMPLEMENTED);

}


@Override
public ResponseEntity<Products> productsOfABundle(String lidvid, @Valid Integer start, @Valid Integer limit,
@Valid List<String> fields, @Valid List<String> sort, @Valid Boolean onlySummary) {
String accept = this.request.getHeader("Accept");
MyBundlesApiController.log.info("accept value is " + accept);
if ((accept != null
&& (accept.contains("application/json")
|| accept.contains("text/html")
|| accept.contains("application/xml")
|| accept.contains("application/pds4+xml")
|| accept.contains("*/*")))
|| (accept == null)) {

try {


Products products = this.getProductChildren(lidvid, start, limit, fields, sort, onlySummary);

return new ResponseEntity<Products>(products, HttpStatus.OK);

} catch (IOException e) {
log.error("Couldn't serialize response for content type " + accept, e);
return new ResponseEntity<Products>(HttpStatus.INTERNAL_SERVER_ERROR);
}

}
else return new ResponseEntity<Products>(HttpStatus.NOT_IMPLEMENTED);
}

@SuppressWarnings("unchecked")
private Products getProductChildren(String lidvid, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) throws IOException
{
if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid);
MyBundlesApiController.log.info("request bundle lidvid, children of products: " + lidvid);

GetRequest getBundleRequest = new GetRequest(this.esRegistryConnection.getRegistryIndex(), lidvid);
HashSet<String> uniqueProperties = new HashSet<String>();
List<String> productLidvids = new ArrayList<String>();
Products products = new Products();
RestHighLevelClient restHighLevelClient = this.esRegistryConnection.getRestHighLevelClient();
Summary summary = new Summary();

if (sort == null) { sort = Arrays.asList(); }

summary.setStart(start);
summary.setLimit(limit);
summary.setSort(sort);
products.setSummary(summary);

try
{
GetResponse getBundleResponse = restHighLevelClient.get(getBundleRequest, RequestOptions.DEFAULT);

if (getBundleResponse.isExists())
{
MyBundlesApiController.log.info("get response " + getBundleResponse.toString());
List<String> collections = (List<String>) getBundleResponse.getSourceAsMap().get("ref_lid_collection");
String collectionLidVid;

for (String collectionLid : collections )
{
collectionLidVid = this.getLatestLidVidFromLid(collectionLid) + "::P1";
MyBundlesApiController.log.info("get collection with lidvid " + collectionLidVid);
GetRequest getCollectionRequest = new GetRequest(this.esRegistryConnection.getRegistryRefIndex(), collectionLidVid);
GetResponse getCollectionResponse = restHighLevelClient.get(getCollectionRequest, RequestOptions.DEFAULT);

if (getCollectionResponse.isExists())
{
MyBundlesApiController.log.info("get ref response " + getCollectionResponse.toString());
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The iteratable class available on https://github.com/NASA-PDS/registry-api-service/tree/master/src/main/java/gov/nasa/pds/api/engineering/elasticsearch/business can be used to find the list of products of a collection between start and start+limit without having to load all products in memory.

This method could also be implemented as an iterator.

That will be convenient to re-use the code since we want also to optimize the code which access the products of the collection by doing a single request for all the products instead of one per product as you do in line 277.

Since that works anyway here, we can work on the code re-useability in the optimization ticket #13

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It cannot. In order to complete the search, first have to get the bundle then all collections then all products. The limits and pagination are over all products not bundles and collections, therefore bundle and collections have to be read to build the products list before pagination can take place. Ugly, but that is part of one of the various emails with @jordanpadams were the punch line was pagination is products not collections in this case.

As I stated in that email also, the iterator also reads all the data. It is not possible to do fractional searches unless state is saved and all these interfaces are REST.

Object temp = getCollectionResponse.getSourceAsMap().get("product_lidvid");
if (temp instanceof String) { productLidvids.add((String)temp); }
else { productLidvids.addAll((List<String>) temp); }
}
else
{
MyBundlesApiController.log.warn("Couldn't get collection child " + collectionLidVid + " of bundle " + lidvid + " in elasticSearch");
}
}
MyBundlesApiController.log.info("total number of product lidvids " + Integer.toString(productLidvids.size()));
for (int i=start ; i < start+limit && i < productLidvids.size() ; i++)
{
MyBundlesApiController.log.info("fetch product from lidvid " + productLidvids.get(i));
GetRequest getProductRequest = new GetRequest(this.esRegistryConnection.getRegistryIndex(), productLidvids.get(i));
GetResponse getProductResponse = restHighLevelClient.get(getProductRequest, RequestOptions.DEFAULT);

if (getProductResponse.isExists())
{
Map<String, Object> sourceAsMap = getProductResponse.getSourceAsMap();
Map<String, Object> filteredMapJsonProperties = this.getFilteredProperties(sourceAsMap, fields);

uniqueProperties.addAll(filteredMapJsonProperties.keySet());

if (!onlySummary)
{
EntityProduct entityProduct = objectMapper.convertValue(sourceAsMap, EntityProduct.class);
ProductWithXmlLabel product = ElasticSearchUtil.ESentityProductToAPIProduct(entityProduct);
product.setProperties(filteredMapJsonProperties);
products.addDataItem(product);
}
}
else
{
MyBundlesApiController.log.warn("Couldn't get product child " + productLidvids.get(i) + " of bundle " + lidvid + " in elasticSearch");
}
}
}
}
catch (IOException e)
{
MyBundlesApiController.log.error("Couldn't get bundle " + lidvid + " from elasticSearch", e);
throw(e);
}

summary.setProperties(new ArrayList<String>(uniqueProperties));
return products;
}
}
85 changes: 72 additions & 13 deletions src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@


import gov.nasa.pds.api.base.CollectionsApi;
import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchRegistrySearchRequestBuilder;
import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchUtil;
import gov.nasa.pds.api.engineering.elasticsearch.business.CollectionProductRefBusinessObject;
import gov.nasa.pds.api.engineering.elasticsearch.business.CollectionProductRelationships;
Expand All @@ -15,8 +14,11 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import io.swagger.annotations.*;

import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
Expand All @@ -25,7 +27,6 @@
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestParam;


import javax.validation.Valid;
import javax.servlet.http.HttpServletRequest;

Expand Down Expand Up @@ -112,18 +113,10 @@ public ResponseEntity<Products> productsOfACollection(@ApiParam(value = "lidvid


private Products getProductChildren(String lidvid, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) throws IOException {
if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid);
MyCollectionsApiController.log.info("request bundle lidvid, collections children: " + lidvid);




SearchResponse searchCollectionRefResponse = null;

RestHighLevelClient restHighLevelClient = this.esRegistryConnection.getRestHighLevelClient();

try {
if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid);

Products products = new Products();

HashSet<String> uniqueProperties = new HashSet<String>();
Expand Down Expand Up @@ -180,7 +173,73 @@ private Products getProductChildren(String lidvid, int start, int limit, List<St
}

}



@Override
public ResponseEntity<Products> bundlesContainingCollection(String lidvid, @Valid Integer start, @Valid Integer limit,
@Valid List<String> fields, @Valid List<String> sort, @Valid Boolean summaryOnly)
{
String accept = this.request.getHeader("Accept");
MyCollectionsApiController.log.info("accept value is " + accept);

if ((accept != null
&& (accept.contains("application/json")
|| accept.contains("text/html")
|| accept.contains("application/xml")
|| accept.contains("application/pds4+xml")
|| accept.contains("*/*")))
|| (accept == null))
{
try
{
Products products = this.getContainingBundle(lidvid, start, limit, fields, sort, summaryOnly);
return new ResponseEntity<Products>(products, HttpStatus.OK);
}
catch (IOException e)
{
log.error("Couldn't serialize response for content type " + accept, e);
return new ResponseEntity<Products>(HttpStatus.INTERNAL_SERVER_ERROR);
}
}
else return new ResponseEntity<Products>(HttpStatus.NOT_IMPLEMENTED);
}

private Products getContainingBundle(String lidvid, int start, int limit, List<String> fields, List<String> sort, boolean summaryOnly) throws IOException
{
if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid);
MyCollectionsApiController.log.info("find all bundles containing the collection lidvid: " + lidvid);
MyCollectionsApiController.log.info("find all bundles containing the collection lid: " + lidvid.substring(0, lidvid.indexOf("::")));
HashSet<String> uniqueProperties = new HashSet<String>();
Products products = new Products();
SearchRequest request = new SearchRequest(this.esRegistryConnection.getRegistryIndex());
SearchResponse response;
SearchSourceBuilder builder = new SearchSourceBuilder();
Summary summary = new Summary();

if (sort == null) { sort = Arrays.asList(); }

summary.setStart(start);
summary.setLimit(limit);
summary.setSort(sort);
products.setSummary(summary);
builder.query(QueryBuilders.matchQuery("ref_lid_collection", lidvid.substring(0, lidvid.indexOf("::"))));
request.source(builder);
response = this.esRegistryConnection.getRestHighLevelClient().search(request,RequestOptions.DEFAULT);
for (int i = start ; start < limit && i < response.getHits().getHits().length ; i++)
{
Map<String, Object> sourceAsMap = response.getHits().getAt(i).getSourceAsMap();
Map<String, Object> filteredMapJsonProperties = this.getFilteredProperties(sourceAsMap, fields);

uniqueProperties.addAll(filteredMapJsonProperties.keySet());

if (!summaryOnly) {
EntityProduct entityProduct = objectMapper.convertValue(sourceAsMap, EntityProduct.class);
ProductWithXmlLabel product = ElasticSearchUtil.ESentityProductToAPIProduct(entityProduct);
product.setProperties(filteredMapJsonProperties);
products.addDataItem(product);
}
}
summary.setProperties(new ArrayList<String>(uniqueProperties));
return products;
}
}
Loading