Skip to content
This repository has been archived by the owner on Dec 22, 2022. It is now read-only.

Issue 13.1 #45

Merged
merged 33 commits into from
Aug 10, 2021
Merged

Issue 13.1 #45

Show file tree
Hide file tree
Changes from 31 commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
c623396
bundles more efficent
Jun 22, 2021
439a308
refactored some more
Jun 22, 2021
b85e78a
update collections controller
Jun 22, 2021
208a95b
update product
Jun 23, 2021
4d2f2cf
Merge branch 'main' into issue_13.1
al-niessner Jun 23, 2021
0731a41
Merge branch 'main' into issue_13.1
al-niessner Jun 23, 2021
dee3ff3
merge of main repair
Jun 23, 2021
d85ce47
make all fo issue_56 checks work
Jun 23, 2021
f50daa9
Merge branch 'main' into issue_13.1
Jul 1, 2021
508bcf1
also collect default fields
Jul 1, 2021
b3bfb7d
request maximum data
Jul 1, 2021
2fdfda2
request maximum data
Jul 2, 2021
d12d817
request maximum data
Jul 2, 2021
fa58a54
request maximum data
Jul 2, 2021
cb9f030
revert and find another way
Jul 2, 2021
2e2e52b
updated the bundle endpoint
Jul 7, 2021
47e54e5
second rework
Jul 7, 2021
78b561c
++ operator behaves oddly
Jul 7, 2021
b8bc666
fix auto checks
Jul 7, 2021
ea50daf
gamma debug messages
Jul 7, 2021
3819ca7
too much
Jul 7, 2021
3289191
making a lidvid is expensive??
Jul 7, 2021
0d94733
have to jump forward at times
Jul 7, 2021
339488f
delay is aback to large gets from ES
Jul 7, 2021
b480f0c
more auto check fixes
Jul 7, 2021
5ba30ee
more timing data
Jul 13, 2021
6c50437
test no lid to lidvid safety
Jul 13, 2021
f1634c4
option 1
Jul 13, 2021
a22f69a
requested updates
Jul 23, 2021
fc86d5e
add comments
Jul 28, 2021
5ba6c93
remove extra use of limit
Jul 30, 2021
bd66f5d
remove extra use of limit
Jul 30, 2021
40b9239
get full list of products if ready
Aug 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
265 changes: 89 additions & 176 deletions src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,15 @@


import gov.nasa.pds.api.base.BundlesApi;
import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchHitIterator;
import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchRegistrySearchRequestBuilder;
import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchUtil;
import gov.nasa.pds.api.engineering.elasticsearch.business.ProductBusinessObject;
import gov.nasa.pds.api.engineering.elasticsearch.entities.EntityProduct;
import gov.nasa.pds.api.model.xml.ProductWithXmlLabel;
import gov.nasa.pds.api.model.xml.XMLMashallableProperyValue;
import gov.nasa.pds.model.Product;
import gov.nasa.pds.model.PropertyArrayValues;
import gov.nasa.pds.model.Products;
import gov.nasa.pds.model.Summary;

import com.fasterxml.jackson.databind.ObjectMapper;
import io.swagger.annotations.*;

import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpStatus;
Expand Down Expand Up @@ -53,8 +44,9 @@ public MyBundlesApiController(ObjectMapper objectMapper, HttpServletRequest requ

}

public ResponseEntity<Product> bundleByLidvid(@ApiParam(value = "lidvid (urn)",required=true) @PathVariable("lidvid") String lidvid
) {
@Override
public ResponseEntity<Product> bundleByLidvid(@ApiParam(value = "lidvid (urn)",required=true) @PathVariable("lidvid") String lidvid)
al-niessner marked this conversation as resolved.
Show resolved Hide resolved
{
return this.getProductResponseEntity(lidvid);
}

Expand All @@ -81,100 +73,54 @@ public ResponseEntity<Products> collectionsOfABundle(@ApiParam(value = "lidvid (
return this.getBundlesCollections(lidvid, start, limit, fields, sort, onlySummary);
}

@SuppressWarnings("unchecked")
private Products getCollectionChildren(String lidvid, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) throws IOException
private List<String> getRefLidCollection (String lidvid, int limit) throws IOException
al-niessner marked this conversation as resolved.
Show resolved Hide resolved
{
List<String> reflids = new ArrayList<String>();

for (final Map<String, Object> bundle : new ElasticSearchHitIterator(this.esRegistryConnection.getRestHighLevelClient(),
ElasticSearchRegistrySearchRequestBuilder.getQueryFieldFromLidvid(lidvid, "ref_lid_collection",
this.esRegistryConnection.getRegistryIndex())))
{
if (bundle.get("ref_lid_collection") instanceof String)
al-niessner marked this conversation as resolved.
Show resolved Hide resolved
{ reflids.add(this.productBO.getLatestLidVidFromLid(bundle.get("ref_lid_collection").toString())); }
else
{
@SuppressWarnings("unchecked")
List<String> clids = (List<String>)bundle.get("ref_lid_collection");
for (String clid : clids)
{ reflids.add(this.productBO.getLatestLidVidFromLid(clid)); }
}
}
return reflids;
}

private Products getCollectionChildren(String lidvid, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) throws IOException
{
if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.productBO.getLatestLidVidFromLid(lidvid);
MyBundlesApiController.log.info("request bundle lidvid, collections children: " + lidvid);

GetRequest getBundleRequest = new ElasticSearchRegistrySearchRequestBuilder().getGetProductRequest(lidvid);
GetResponse getBundleResponse = null;

RestHighLevelClient restHighLevelClient = this.esRegistryConnection.getRestHighLevelClient();

try {
getBundleResponse = restHighLevelClient.get(getBundleRequest,
RequestOptions.DEFAULT);

Products products = new Products();

HashSet<String> uniqueProperties = new HashSet<String>();

Summary summary = new Summary();

summary.setStart(start);
summary.setLimit(limit);

if (sort == null) {
sort = Arrays.asList();
}
summary.setSort(sort);

products.setSummary(summary);

if (getBundleResponse.isExists()) {
MyBundlesApiController.log.info("get response " + getBundleResponse.toString());
@SuppressWarnings("unchecked")
List<String> collections = (List<String>) getBundleResponse.getSourceAsMap().get("ref_lid_collection");
String collectionLidVid;
int i=0;
for (String collectionLid : collections ) {
if ((i>=start) && (i<start+limit)) {
collectionLidVid = this.productBO.getLatestLidVidFromLid(collectionLid);

//this.productBO.getProduct(collectionLidVid);
GetRequest getCollectionRequest = new ElasticSearchRegistrySearchRequestBuilder().getGetProductRequest(collectionLidVid);

//GetRequest getCollectionRequest = new GetRequest(this.esRegistryConnection.getRegistryIndex(),
// collectionLidVid);


GetResponse getCollectionResponse = null;

getCollectionResponse = restHighLevelClient.get(getCollectionRequest,
RequestOptions.DEFAULT);

if (getCollectionResponse.isExists()) {

MyBundlesApiController.log.info("get response " + getCollectionResponse.toString());
Map<String, Object> sourceAsMap = getCollectionResponse.getSourceAsMap();
Map<String, XMLMashallableProperyValue> filteredMapJsonProperties = ProductBusinessObject.getFilteredProperties(sourceAsMap, fields, null);

uniqueProperties.addAll(filteredMapJsonProperties.keySet());



if (!onlySummary) {
EntityProduct entityCollection = objectMapper.convertValue(sourceAsMap, EntityProduct.class);

Product collection = ElasticSearchUtil.ESentityProductToAPIProduct(entityCollection, this.getBaseURL());
collection.setProperties((Map<String, PropertyArrayValues>)(Map<String, ?>)filteredMapJsonProperties);
products.addDataItem(collection);
}


}
else {
MyBundlesApiController.log.warn("Couldn't get collection child " + collectionLidVid + " of bundle " + lidvid + " in elasticSearch");
}
}
i+=1;

}

}


summary.setProperties(new ArrayList<String>(uniqueProperties));

return products;


} catch (IOException e) {
MyBundlesApiController.log.error("Couldn't get bundle " + lidvid + " from elasticSearch", e);
throw(e);
}


HashSet<String> uniqueProperties = new HashSet<String>();
List<String> clidvids = this.getRefLidCollection(lidvid, limit);
al-niessner marked this conversation as resolved.
Show resolved Hide resolved
Products products = new Products();
Summary summary = new Summary();

if (sort == null) { sort = Arrays.asList(); }

summary.setStart(start);
summary.setLimit(limit);
summary.setSort(sort);
products.setSummary(summary);

if (0 < clidvids.size())
{
this.fillProductsFromLidvids(products, uniqueProperties,
clidvids.subList(start, clidvids.size() < start+limit ? clidvids.size(): start+limit),
fields, onlySummary);
}
else MyBundlesApiController.log.warn ("Did not find any collections for bundle lidvid: " + lidvid);

summary.setProperties(new ArrayList<String>(uniqueProperties));
return products;
}

private ResponseEntity<Products> getBundlesCollections(String lidvid, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) {
Expand Down Expand Up @@ -231,18 +177,17 @@ public ResponseEntity<Products> productsOfABundle(String lidvid, @Valid Integer
else return new ResponseEntity<Products>(HttpStatus.NOT_IMPLEMENTED);
}

@SuppressWarnings("unchecked")
private Products getProductChildren(String lidvid, int start, int limit, List<String> fields, List<String> sort, boolean onlySummary) throws IOException
{

if (!lidvid.contains("::")) lidvid = this.productBO.getLatestLidVidFromLid(lidvid);

MyBundlesApiController.log.info("request bundle lidvid, children of products: " + lidvid);
GetRequest getBundleRequest = new ElasticSearchRegistrySearchRequestBuilder().getGetProductRequest(lidvid);

int iteration=0,wsize=0;
HashSet<String> uniqueProperties = new HashSet<String>();
List<String> productLidvids = new ArrayList<String>();
List<String> clidvids = this.getRefLidCollection(lidvid, limit);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it is unclear to me here as well: we are asking for product children of bundle from start to start+limit, I don't see how that can match with the collection pagination, so we can not use the limit parameter here

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As state above, it is used a random number more than anything else. It could be fixed to 10 or 3 or 50 or 100. Much bigger than 100 and elasticsearch returns an error regularly.

It is an interesting question of what to set the limit of the collection of a bundle to when looking for the product.

The idea is that if the user wants to wait for bigger chunks like 100 products then also getting 100 collections is better than 10 collections then having to ask again for the other 90 in the case of one product per collection.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This code no longer exists (outdated) so not sure what comments are required.

List<String> plidvids = new ArrayList<String>();
List<String> wlidvids = new ArrayList<String>();
Products products = new Products();
RestHighLevelClient restHighLevelClient = this.esRegistryConnection.getRestHighLevelClient();
Summary summary = new Summary();

if (sort == null) { sort = Arrays.asList(); }
Expand All @@ -252,75 +197,43 @@ private Products getProductChildren(String lidvid, int start, int limit, List<St
summary.setSort(sort);
products.setSummary(summary);

try
if (0 < clidvids.size())
{
GetResponse getBundleResponse = restHighLevelClient.get(getBundleRequest, RequestOptions.DEFAULT);


if (getBundleResponse.isExists())
{
MyBundlesApiController.log.info("get response " + getBundleResponse.toString());
List<String> collections = (List<String>) getBundleResponse.getSourceAsMap().get("ref_lid_collection");
String collectionLidVid;

for (String collectionLid : collections )
{

collectionLidVid = productBO.getLatestLidVidFromLid(collectionLid) + "::P1";

// TODO change the request to get collections from their lidvid and not from the ID which is not right, we are missing packets
MyBundlesApiController.log.info("get collection with lidvid " + collectionLidVid);
GetRequest getCollectionRequest = new GetRequest(this.esRegistryConnection.getRegistryRefIndex(), collectionLidVid);
GetResponse getCollectionResponse = restHighLevelClient.get(getCollectionRequest, RequestOptions.DEFAULT);

if (getCollectionResponse.isExists())
{
MyBundlesApiController.log.info("get ref response " + getCollectionResponse.toString());
Object temp = getCollectionResponse.getSourceAsMap().get("product_lidvid");
if (temp instanceof String) { productLidvids.add((String)temp); }
else { productLidvids.addAll((List<String>) temp); }
}
else
{
MyBundlesApiController.log.warn("Couldn't get collection child " + collectionLidVid + " of bundle " + lidvid + " in elasticSearch");
}
}
MyBundlesApiController.log.info("total number of product lidvids " + Integer.toString(productLidvids.size()));
for (int i=start ; i < start+limit && i < productLidvids.size() ; i++)
{
MyBundlesApiController.log.info("fetch product from lidvid " + productLidvids.get(i));
GetRequest getProductRequest = new GetRequest(this.esRegistryConnection.getRegistryIndex(), productLidvids.get(i));
GetResponse getProductResponse = restHighLevelClient.get(getProductRequest, RequestOptions.DEFAULT);

if (getProductResponse.isExists())
{
Map<String, Object> sourceAsMap = getProductResponse.getSourceAsMap();
Map<String, XMLMashallableProperyValue> filteredMapJsonProperties = ProductBusinessObject.getFilteredProperties(sourceAsMap, fields, null);

uniqueProperties.addAll(filteredMapJsonProperties.keySet());

if (!onlySummary)
{
EntityProduct entityProduct = objectMapper.convertValue(sourceAsMap, EntityProduct.class);

Product product = ElasticSearchUtil.ESentityProductToAPIProduct(entityProduct, this.getBaseURL());
product.setProperties((Map<String, PropertyArrayValues>)(Map<String, ?>)filteredMapJsonProperties);

products.addDataItem(product);
}
}
else
{
MyBundlesApiController.log.warn("Couldn't get product child " + productLidvids.get(i) + " of bundle " + lidvid + " in elasticSearch");
}
}
}
}
catch (IOException e)
for (final Map<String,Object> hit : new ElasticSearchHitIterator(this.esRegistryConnection.getRestHighLevelClient(),
ElasticSearchRegistrySearchRequestBuilder.getQueryFieldFromKVP("collection_lidvid", clidvids, "product_lidvid",
this.esRegistryConnection.getRegistryRefIndex())))
{
wlidvids.clear();
wsize = 0;

if (hit.get("product_lidvid") instanceof String)
al-niessner marked this conversation as resolved.
Show resolved Hide resolved
{ wlidvids.add(this.productBO.getLatestLidVidFromLid(hit.get("product_lidvid").toString())); }
else
{
@SuppressWarnings("unchecked")
List<String> plids = (List<String>)hit.get("product_lidvid");

if (start <= iteration || start < iteration+plids.size()) { wlidvids.addAll(plids); }
else { wsize = plids.size(); }
}

if (start <= iteration || start < iteration+wlidvids.size())
{ plidvids.addAll(wlidvids.subList(start <= iteration ? 0 : start-iteration, wlidvids.size())); }

if (limit <= plidvids.size()) { break; }
else { iteration = iteration + wlidvids.size() + wsize; }
}
}
else MyBundlesApiController.log.warn ("Did not find any collections for bundle lidvid: " + lidvid);

MyBundlesApiController.log.info("found " + Integer.toString(plidvids.size()) + " products in this bundle");

if (0 < plidvids.size())
{
MyBundlesApiController.log.error("Couldn't get bundle " + lidvid + " from elasticSearch", e);
throw(e);
}
this.fillProductsFromLidvids(products, uniqueProperties,
plidvids.subList(0, plidvids.size() < limit ? plidvids.size() : limit), fields, onlySummary);
}
else MyBundlesApiController.log.warn ("Did not find any products for bundle lidvid: " + lidvid);

summary.setProperties(new ArrayList<String>(uniqueProperties));
return products;
Expand Down
Loading