Skip to content
This repository has been archived by the owner on Dec 22, 2022. It is now read-only.

Commit

Permalink
pds-api-56: crawl the hierarchical tree (#29)
Browse files Browse the repository at this point in the history 
* start with the acceptance criteria being checked

* cosmetic changes

* update the API

The POM needs to request the correct version of the api JAR file.

Update the 3 API interfaces with the new methods. Currently they are blank place holders and the appropriate branches will be created to fill in each of the bodies. The branches will be created from here then merged back.

* basics for products of a bundle

* now get products of a bundle. works and tested

* prepare for finding owners

* implement bundle from collection

Basically use the search capabilities to find where the collection lid shows up in the ref_lid_collection variable. Pretty straight forward.

* editorial

* fix what should not have changed

* find bundles of a product

Pretty straight foward finding all collections that hold a this product then all bundles that reference those collections.

* collections that reference product

* fix whitespacing

* get actual collection

* do negative checks

If the lidvid is wrong, make sure that nothing is returned. This was added because it was found that the double search of /products/x/bundles returned multiple responses instead of 0. Because the second search had no criteria it returned the entire database. Added code to prevent this then added negative checks for all URLs.

Co-authored-by: Al Niessner <Al.Niessner@xxx.xxx>
  • Loading branch information
@al-niessner
al-niessner and Al Niessner authored May 20, 2021
1 parent add147f commit 9c8d291
Show file tree
Hide file tree
Showing 6 changed files with 398 additions and 37 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@
<dependency>
<groupId>gov.nasa.pds</groupId>
<artifactId>api</artifactId>
<version>0.0.1-SNAPSHOT</version>
<version>0.2.0-SNAPSHOT</version>
</dependency>

<dependency>
Expand Down
136 changes: 122 additions & 14 deletions src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchUtil;
import gov.nasa.pds.api.engineering.elasticsearch.entities.EntityProduct;
import gov.nasa.pds.api.model.ProductWithXmlLabel;
import gov.nasa.pds.model.ErrorMessage;
import gov.nasa.pds.model.Product;
import gov.nasa.pds.model.Products;
import gov.nasa.pds.model.Summary;
Expand All @@ -22,15 +21,9 @@
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.CookieValue;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.multipart.MultipartFile;

import javax.validation.constraints.*;
import javax.validation.Valid;
import javax.servlet.http.HttpServletRequest;
import java.io.IOException;
Expand Down Expand Up @@ -82,7 +75,9 @@ public ResponseEntity<Products> collectionsOfABundle(@ApiParam(value = "lidvid (
return this.getBundlesCollections(lidvid, start, limit, fields, sort, onlySummary);
}

private Products getCollectionChildren(String lidvid, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) throws IOException {
private Products getCollectionChildren(String lidvid, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) throws IOException
{
if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid);
MyBundlesApiController.log.info("request bundle lidvid, collections children: " + lidvid);

GetRequest getBundleRequest = new GetRequest(this.esRegistryConnection.getRegistryIndex(),
Expand Down Expand Up @@ -113,7 +108,8 @@ private Products getCollectionChildren(String lidvid, int start, int limit, List

if (getBundleResponse.isExists()) {
MyBundlesApiController.log.info("get response " + getBundleResponse.toString());
List<String> collections = (List<String>) getBundleResponse.getSourceAsMap().get("ref_lid_collection");
@SuppressWarnings("unchecked")
List<String> collections = (List<String>) getBundleResponse.getSourceAsMap().get("ref_lid_collection");
String collectionLidVid;
int i=0;
for (String collectionLid : collections ) {
Expand Down Expand Up @@ -168,9 +164,7 @@ private Products getCollectionChildren(String lidvid, int start, int limit, List
}

}




private ResponseEntity<Products> getBundlesCollections(String lidvid, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) {
String accept = this.request.getHeader("Accept");
MyBundlesApiController.log.info("accept value is " + accept);
Expand All @@ -196,8 +190,122 @@ private ResponseEntity<Products> getBundlesCollections(String lidvid, int start,

}
else return new ResponseEntity<Products>(HttpStatus.NOT_IMPLEMENTED);

}


@Override
public ResponseEntity<Products> productsOfABundle(String lidvid, @Valid Integer start, @Valid Integer limit,
@Valid List<String> fields, @Valid List<String> sort, @Valid Boolean onlySummary) {
String accept = this.request.getHeader("Accept");
MyBundlesApiController.log.info("accept value is " + accept);
if ((accept != null
&& (accept.contains("application/json")
|| accept.contains("text/html")
|| accept.contains("application/xml")
|| accept.contains("application/pds4+xml")
|| accept.contains("*/*")))
|| (accept == null)) {

try {


Products products = this.getProductChildren(lidvid, start, limit, fields, sort, onlySummary);

return new ResponseEntity<Products>(products, HttpStatus.OK);

} catch (IOException e) {
log.error("Couldn't serialize response for content type " + accept, e);
return new ResponseEntity<Products>(HttpStatus.INTERNAL_SERVER_ERROR);
}

}
else return new ResponseEntity<Products>(HttpStatus.NOT_IMPLEMENTED);
}

@SuppressWarnings("unchecked")
private Products getProductChildren(String lidvid, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) throws IOException
{
if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid);
MyBundlesApiController.log.info("request bundle lidvid, children of products: " + lidvid);

GetRequest getBundleRequest = new GetRequest(this.esRegistryConnection.getRegistryIndex(), lidvid);
HashSet<String> uniqueProperties = new HashSet<String>();
List<String> productLidvids = new ArrayList<String>();
Products products = new Products();
RestHighLevelClient restHighLevelClient = this.esRegistryConnection.getRestHighLevelClient();
Summary summary = new Summary();

if (sort == null) { sort = Arrays.asList(); }

summary.setStart(start);
summary.setLimit(limit);
summary.setSort(sort);
products.setSummary(summary);

try
{
GetResponse getBundleResponse = restHighLevelClient.get(getBundleRequest, RequestOptions.DEFAULT);

if (getBundleResponse.isExists())
{
MyBundlesApiController.log.info("get response " + getBundleResponse.toString());
List<String> collections = (List<String>) getBundleResponse.getSourceAsMap().get("ref_lid_collection");
String collectionLidVid;

for (String collectionLid : collections )
{
collectionLidVid = this.getLatestLidVidFromLid(collectionLid) + "::P1";
MyBundlesApiController.log.info("get collection with lidvid " + collectionLidVid);
GetRequest getCollectionRequest = new GetRequest(this.esRegistryConnection.getRegistryRefIndex(), collectionLidVid);
GetResponse getCollectionResponse = restHighLevelClient.get(getCollectionRequest, RequestOptions.DEFAULT);

if (getCollectionResponse.isExists())
{
MyBundlesApiController.log.info("get ref response " + getCollectionResponse.toString());
Object temp = getCollectionResponse.getSourceAsMap().get("product_lidvid");
if (temp instanceof String) { productLidvids.add((String)temp); }
else { productLidvids.addAll((List<String>) temp); }
}
else
{
MyBundlesApiController.log.warn("Couldn't get collection child " + collectionLidVid + " of bundle " + lidvid + " in elasticSearch");
}
}
MyBundlesApiController.log.info("total number of product lidvids " + Integer.toString(productLidvids.size()));
for (int i=start ; i < start+limit && i < productLidvids.size() ; i++)
{
MyBundlesApiController.log.info("fetch product from lidvid " + productLidvids.get(i));
GetRequest getProductRequest = new GetRequest(this.esRegistryConnection.getRegistryIndex(), productLidvids.get(i));
GetResponse getProductResponse = restHighLevelClient.get(getProductRequest, RequestOptions.DEFAULT);

if (getProductResponse.isExists())
{
Map<String, Object> sourceAsMap = getProductResponse.getSourceAsMap();
Map<String, Object> filteredMapJsonProperties = this.getFilteredProperties(sourceAsMap, fields);

uniqueProperties.addAll(filteredMapJsonProperties.keySet());

if (!onlySummary)
{
EntityProduct entityProduct = objectMapper.convertValue(sourceAsMap, EntityProduct.class);
ProductWithXmlLabel product = ElasticSearchUtil.ESentityProductToAPIProduct(entityProduct);
product.setProperties(filteredMapJsonProperties);
products.addDataItem(product);
}
}
else
{
MyBundlesApiController.log.warn("Couldn't get product child " + productLidvids.get(i) + " of bundle " + lidvid + " in elasticSearch");
}
}
}
}
catch (IOException e)
{
MyBundlesApiController.log.error("Couldn't get bundle " + lidvid + " from elasticSearch", e);
throw(e);
}

summary.setProperties(new ArrayList<String>(uniqueProperties));
return products;
}
}
85 changes: 72 additions & 13 deletions src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@


import gov.nasa.pds.api.base.CollectionsApi;
import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchRegistrySearchRequestBuilder;
import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchUtil;
import gov.nasa.pds.api.engineering.elasticsearch.business.CollectionProductRefBusinessObject;
import gov.nasa.pds.api.engineering.elasticsearch.business.CollectionProductRelationships;
Expand All @@ -15,8 +14,11 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import io.swagger.annotations.*;

import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
Expand All @@ -25,7 +27,6 @@
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestParam;


import javax.validation.Valid;
import javax.servlet.http.HttpServletRequest;

Expand Down Expand Up @@ -112,18 +113,10 @@ public ResponseEntity<Products> productsOfACollection(@ApiParam(value = "lidvid


private Products getProductChildren(String lidvid, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) throws IOException {
if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid);
MyCollectionsApiController.log.info("request bundle lidvid, collections children: " + lidvid);




SearchResponse searchCollectionRefResponse = null;

RestHighLevelClient restHighLevelClient = this.esRegistryConnection.getRestHighLevelClient();

try {
if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid);

Products products = new Products();

HashSet<String> uniqueProperties = new HashSet<String>();
Expand Down Expand Up @@ -180,7 +173,73 @@ private Products getProductChildren(String lidvid, int start, int limit, List<St
}

}



@Override
public ResponseEntity<Products> bundlesContainingCollection(String lidvid, @Valid Integer start, @Valid Integer limit,
@Valid List<String> fields, @Valid List<String> sort, @Valid Boolean summaryOnly)
{
String accept = this.request.getHeader("Accept");
MyCollectionsApiController.log.info("accept value is " + accept);

if ((accept != null
&& (accept.contains("application/json")
|| accept.contains("text/html")
|| accept.contains("application/xml")
|| accept.contains("application/pds4+xml")
|| accept.contains("*/*")))
|| (accept == null))
{
try
{
Products products = this.getContainingBundle(lidvid, start, limit, fields, sort, summaryOnly);
return new ResponseEntity<Products>(products, HttpStatus.OK);
}
catch (IOException e)
{
log.error("Couldn't serialize response for content type " + accept, e);
return new ResponseEntity<Products>(HttpStatus.INTERNAL_SERVER_ERROR);
}
}
else return new ResponseEntity<Products>(HttpStatus.NOT_IMPLEMENTED);
}

private Products getContainingBundle(String lidvid, int start, int limit, List<String> fields, List<String> sort, boolean summaryOnly) throws IOException
{
if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid);
MyCollectionsApiController.log.info("find all bundles containing the collection lidvid: " + lidvid);
MyCollectionsApiController.log.info("find all bundles containing the collection lid: " + lidvid.substring(0, lidvid.indexOf("::")));
HashSet<String> uniqueProperties = new HashSet<String>();
Products products = new Products();
SearchRequest request = new SearchRequest(this.esRegistryConnection.getRegistryIndex());
SearchResponse response;
SearchSourceBuilder builder = new SearchSourceBuilder();
Summary summary = new Summary();

if (sort == null) { sort = Arrays.asList(); }

summary.setStart(start);
summary.setLimit(limit);
summary.setSort(sort);
products.setSummary(summary);
builder.query(QueryBuilders.matchQuery("ref_lid_collection", lidvid.substring(0, lidvid.indexOf("::"))));
request.source(builder);
response = this.esRegistryConnection.getRestHighLevelClient().search(request,RequestOptions.DEFAULT);
for (int i = start ; start < limit && i < response.getHits().getHits().length ; i++)
{
Map<String, Object> sourceAsMap = response.getHits().getAt(i).getSourceAsMap();
Map<String, Object> filteredMapJsonProperties = this.getFilteredProperties(sourceAsMap, fields);

uniqueProperties.addAll(filteredMapJsonProperties.keySet());

if (!summaryOnly) {
EntityProduct entityProduct = objectMapper.convertValue(sourceAsMap, EntityProduct.class);
ProductWithXmlLabel product = ElasticSearchUtil.ESentityProductToAPIProduct(entityProduct);
product.setProperties(filteredMapJsonProperties);
products.addDataItem(product);
}
}
summary.setProperties(new ArrayList<String>(uniqueProperties));
return products;
}
}
Loading

0 comments on commit 9c8d291

Please sign in to comment.