diff --git a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java index 33f9772..203a478 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java +++ b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java @@ -2,24 +2,15 @@ import gov.nasa.pds.api.base.BundlesApi; +import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchHitIterator; import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchRegistrySearchRequestBuilder; -import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchUtil; -import gov.nasa.pds.api.engineering.elasticsearch.business.ProductBusinessObject; -import gov.nasa.pds.api.engineering.elasticsearch.entities.EntityProduct; -import gov.nasa.pds.api.model.xml.ProductWithXmlLabel; -import gov.nasa.pds.api.model.xml.XMLMashallableProperyValue; import gov.nasa.pds.model.Product; -import gov.nasa.pds.model.PropertyArrayValues; import gov.nasa.pds.model.Products; import gov.nasa.pds.model.Summary; import com.fasterxml.jackson.databind.ObjectMapper; import io.swagger.annotations.*; -import org.elasticsearch.action.get.GetRequest; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; @@ -53,8 +44,9 @@ public MyBundlesApiController(ObjectMapper objectMapper, HttpServletRequest requ } - public ResponseEntity bundleByLidvid(@ApiParam(value = "lidvid (urn)",required=true) @PathVariable("lidvid") String lidvid -) { + @Override + public ResponseEntity bundleByLidvid(@ApiParam(value = "lidvid (urn)",required=true) @PathVariable("lidvid") String lidvid) + { return this.getProductResponseEntity(lidvid); } @@ -81,100 +73,54 @@ public ResponseEntity collectionsOfABundle(@ApiParam(value = "lidvid ( return this.getBundlesCollections(lidvid, start, limit, fields, sort, onlySummary); } - @SuppressWarnings("unchecked") - private Products getCollectionChildren(String lidvid, int start, int limit, List fields, List sort, boolean onlySummary) throws IOException + private List getRefLidCollection (String lidvid) throws IOException + { + List reflids = new ArrayList(); + + for (final Map bundle : new ElasticSearchHitIterator(this.esRegistryConnection.getRestHighLevelClient(), + ElasticSearchRegistrySearchRequestBuilder.getQueryFieldFromLidvid(lidvid, "ref_lid_collection", + this.esRegistryConnection.getRegistryIndex()))) + { + if (bundle.get("ref_lid_collection") instanceof String) + { reflids.add(this.productBO.getLatestLidVidFromLid(bundle.get("ref_lid_collection").toString())); } + else + { + @SuppressWarnings("unchecked") + List clids = (List)bundle.get("ref_lid_collection"); + for (String clid : clids) + { reflids.add(this.productBO.getLatestLidVidFromLid(clid)); } + } + } + return reflids; + } + + private Products getCollectionChildren(String lidvid, int start, int limit, List fields, List sort, boolean onlySummary) throws IOException { if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.productBO.getLatestLidVidFromLid(lidvid); MyBundlesApiController.log.info("request bundle lidvid, collections children: " + lidvid); - - GetRequest getBundleRequest = new ElasticSearchRegistrySearchRequestBuilder().getGetProductRequest(lidvid); - GetResponse getBundleResponse = null; - - RestHighLevelClient restHighLevelClient = this.esRegistryConnection.getRestHighLevelClient(); - - try { - getBundleResponse = restHighLevelClient.get(getBundleRequest, - RequestOptions.DEFAULT); - - Products products = new Products(); - - HashSet uniqueProperties = new HashSet(); - - Summary summary = new Summary(); - - summary.setStart(start); - summary.setLimit(limit); - - if (sort == null) { - sort = Arrays.asList(); - } - summary.setSort(sort); - - products.setSummary(summary); - - if (getBundleResponse.isExists()) { - MyBundlesApiController.log.info("get response " + getBundleResponse.toString()); - @SuppressWarnings("unchecked") - List collections = (List) getBundleResponse.getSourceAsMap().get("ref_lid_collection"); - String collectionLidVid; - int i=0; - for (String collectionLid : collections ) { - if ((i>=start) && (i sourceAsMap = getCollectionResponse.getSourceAsMap(); - Map filteredMapJsonProperties = ProductBusinessObject.getFilteredProperties(sourceAsMap, fields, null); - - uniqueProperties.addAll(filteredMapJsonProperties.keySet()); - - - - if (!onlySummary) { - EntityProduct entityCollection = objectMapper.convertValue(sourceAsMap, EntityProduct.class); - - Product collection = ElasticSearchUtil.ESentityProductToAPIProduct(entityCollection, this.getBaseURL()); - collection.setProperties((Map)(Map)filteredMapJsonProperties); - products.addDataItem(collection); - } - - - } - else { - MyBundlesApiController.log.warn("Couldn't get collection child " + collectionLidVid + " of bundle " + lidvid + " in elasticSearch"); - } - } - i+=1; - - } - - } - - - summary.setProperties(new ArrayList(uniqueProperties)); - - return products; - - - } catch (IOException e) { - MyBundlesApiController.log.error("Couldn't get bundle " + lidvid + " from elasticSearch", e); - throw(e); - } - + + HashSet uniqueProperties = new HashSet(); + List clidvids = this.getRefLidCollection(lidvid); + Products products = new Products(); + Summary summary = new Summary(); + + if (sort == null) { sort = Arrays.asList(); } + + summary.setStart(start); + summary.setLimit(limit); + summary.setSort(sort); + products.setSummary(summary); + + if (0 < clidvids.size()) + { + this.fillProductsFromLidvids(products, uniqueProperties, + clidvids.subList(start, clidvids.size() < start+limit ? clidvids.size(): start+limit), + fields, onlySummary); + } + else MyBundlesApiController.log.warn ("Did not find any collections for bundle lidvid: " + lidvid); + + summary.setProperties(new ArrayList(uniqueProperties)); + return products; } private ResponseEntity getBundlesCollections(String lidvid, int start, int limit, List fields, List sort, boolean onlySummary) { @@ -231,18 +177,17 @@ public ResponseEntity productsOfABundle(String lidvid, @Valid Integer else return new ResponseEntity(HttpStatus.NOT_IMPLEMENTED); } - @SuppressWarnings("unchecked") private Products getProductChildren(String lidvid, int start, int limit, List fields, List sort, boolean onlySummary) throws IOException { - if (!lidvid.contains("::")) lidvid = this.productBO.getLatestLidVidFromLid(lidvid); - MyBundlesApiController.log.info("request bundle lidvid, children of products: " + lidvid); - GetRequest getBundleRequest = new ElasticSearchRegistrySearchRequestBuilder().getGetProductRequest(lidvid); + + int iteration=0,wsize=0; HashSet uniqueProperties = new HashSet(); - List productLidvids = new ArrayList(); + List clidvids = this.getRefLidCollection(lidvid); + List plidvids = new ArrayList(); + List wlidvids = new ArrayList(); Products products = new Products(); - RestHighLevelClient restHighLevelClient = this.esRegistryConnection.getRestHighLevelClient(); Summary summary = new Summary(); if (sort == null) { sort = Arrays.asList(); } @@ -252,75 +197,43 @@ private Products getProductChildren(String lidvid, int start, int limit, List collections = (List) getBundleResponse.getSourceAsMap().get("ref_lid_collection"); - String collectionLidVid; - - for (String collectionLid : collections ) - { - - collectionLidVid = productBO.getLatestLidVidFromLid(collectionLid) + "::P1"; - - // TODO change the request to get collections from their lidvid and not from the ID which is not right, we are missing packets - MyBundlesApiController.log.info("get collection with lidvid " + collectionLidVid); - GetRequest getCollectionRequest = new GetRequest(this.esRegistryConnection.getRegistryRefIndex(), collectionLidVid); - GetResponse getCollectionResponse = restHighLevelClient.get(getCollectionRequest, RequestOptions.DEFAULT); - - if (getCollectionResponse.isExists()) - { - MyBundlesApiController.log.info("get ref response " + getCollectionResponse.toString()); - Object temp = getCollectionResponse.getSourceAsMap().get("product_lidvid"); - if (temp instanceof String) { productLidvids.add((String)temp); } - else { productLidvids.addAll((List) temp); } - } - else - { - MyBundlesApiController.log.warn("Couldn't get collection child " + collectionLidVid + " of bundle " + lidvid + " in elasticSearch"); - } - } - MyBundlesApiController.log.info("total number of product lidvids " + Integer.toString(productLidvids.size())); - for (int i=start ; i < start+limit && i < productLidvids.size() ; i++) - { - MyBundlesApiController.log.info("fetch product from lidvid " + productLidvids.get(i)); - GetRequest getProductRequest = new GetRequest(this.esRegistryConnection.getRegistryIndex(), productLidvids.get(i)); - GetResponse getProductResponse = restHighLevelClient.get(getProductRequest, RequestOptions.DEFAULT); - - if (getProductResponse.isExists()) - { - Map sourceAsMap = getProductResponse.getSourceAsMap(); - Map filteredMapJsonProperties = ProductBusinessObject.getFilteredProperties(sourceAsMap, fields, null); - - uniqueProperties.addAll(filteredMapJsonProperties.keySet()); - - if (!onlySummary) - { - EntityProduct entityProduct = objectMapper.convertValue(sourceAsMap, EntityProduct.class); - - Product product = ElasticSearchUtil.ESentityProductToAPIProduct(entityProduct, this.getBaseURL()); - product.setProperties((Map)(Map)filteredMapJsonProperties); - - products.addDataItem(product); - } - } - else - { - MyBundlesApiController.log.warn("Couldn't get product child " + productLidvids.get(i) + " of bundle " + lidvid + " in elasticSearch"); - } - } - } - } - catch (IOException e) + for (final Map hit : new ElasticSearchHitIterator(this.esRegistryConnection.getRestHighLevelClient(), + ElasticSearchRegistrySearchRequestBuilder.getQueryFieldFromKVP("collection_lidvid", clidvids, "product_lidvid", + this.esRegistryConnection.getRegistryRefIndex()))) + { + wlidvids.clear(); + wsize = 0; + + if (hit.get("product_lidvid") instanceof String) + { wlidvids.add(this.productBO.getLatestLidVidFromLid(hit.get("product_lidvid").toString())); } + else + { + @SuppressWarnings("unchecked") + List plids = (List)hit.get("product_lidvid"); + + if (start <= iteration || start < iteration+plids.size()) { wlidvids.addAll(plids); } + else { wsize = plids.size(); } + } + + if (start <= iteration || start < iteration+wlidvids.size()) + { plidvids.addAll(wlidvids.subList(start <= iteration ? 0 : start-iteration, wlidvids.size())); } + + if (limit <= plidvids.size()) { break; } + else { iteration = iteration + wlidvids.size() + wsize; } + } + } + else MyBundlesApiController.log.warn ("Did not find any collections for bundle lidvid: " + lidvid); + + MyBundlesApiController.log.info("found " + Integer.toString(plidvids.size()) + " products in this bundle"); + + if (0 < plidvids.size()) { - MyBundlesApiController.log.error("Couldn't get bundle " + lidvid + " from elasticSearch", e); - throw(e); - } + this.fillProductsFromLidvids(products, uniqueProperties, + plidvids.subList(0, plidvids.size() < limit ? plidvids.size() : limit), fields, onlySummary); + } + else MyBundlesApiController.log.warn ("Did not find any products for bundle lidvid: " + lidvid); summary.setProperties(new ArrayList(uniqueProperties)); return products; diff --git a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java index 1241f33..d8d475e 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java +++ b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java @@ -2,16 +2,10 @@ import gov.nasa.pds.api.base.CollectionsApi; +import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchHitIterator; +import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchRegistrySearchRequestBuilder; import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchUtil; -import gov.nasa.pds.api.engineering.elasticsearch.business.CollectionProductRefBusinessObject; -import gov.nasa.pds.api.engineering.elasticsearch.business.CollectionProductRelationships; -import gov.nasa.pds.api.engineering.elasticsearch.entities.EntityProduct; -import gov.nasa.pds.api.engineering.elasticsearch.entities.EntitytProductWithBlob; -import gov.nasa.pds.api.engineering.exceptions.UnsupportedElasticSearchProperty; -import gov.nasa.pds.api.model.xml.ProductWithXmlLabel; -import gov.nasa.pds.api.model.xml.XMLMashallableProperyValue; import gov.nasa.pds.model.Product; -import gov.nasa.pds.model.PropertyArrayValues; import gov.nasa.pds.model.Products; import gov.nasa.pds.model.Summary; @@ -19,10 +13,6 @@ import io.swagger.annotations.*; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; @@ -35,7 +25,6 @@ import javax.servlet.http.HttpServletRequest; import gov.nasa.pds.api.engineering.elasticsearch.business.LidVidNotFoundException; -import gov.nasa.pds.api.engineering.elasticsearch.business.ProductBusinessObject; import java.io.IOException; import java.util.ArrayList; @@ -125,68 +114,63 @@ public ResponseEntity productsOfACollection(@ApiParam(value = "lidvid - @SuppressWarnings("unchecked") private Products getProductChildren(String lidvid, int start, int limit, List fields, List sort, boolean onlySummary) throws IOException, LidVidNotFoundException { - if (!lidvid.contains("::")) lidvid = this.productBO.getLatestLidVidFromLid(lidvid); MyCollectionsApiController.log.info("request collection lidvid, collections children: " + lidvid); - - try { - Products products = new Products(); - - HashSet uniqueProperties = new HashSet(); - - Summary summary = new Summary(); - - summary.setStart(start); - summary.setLimit(limit); - - if (sort == null) { - sort = Arrays.asList(); - } - summary.setSort(sort); - - products.setSummary(summary); - - CollectionProductRefBusinessObject collectionProductRefBO = new CollectionProductRefBusinessObject(this.esRegistryConnection); - CollectionProductRelationships collectionProductRelationships = collectionProductRefBO.getCollectionProductsIterable(lidvid, start, limit); - - for (EntityProduct eProd : collectionProductRelationships) { - if (eProd != null) { - MyCollectionsApiController.log.info("request lidvdid: " + eProd.getLidVid() ); - - Product product = ElasticSearchUtil.ESentityProductToAPIProduct(eProd, this.getBaseURL()); - - Map filteredMapJsonProperties = ProductBusinessObject.getFilteredProperties( - eProd.getProperties(), - fields, - null); - - uniqueProperties.addAll(filteredMapJsonProperties.keySet()); - - if (!onlySummary) { - product.setProperties((Map)(Map)filteredMapJsonProperties); - - products.addDataItem(product); - } - } - else { - MyCollectionsApiController.log.warn("Couldn't get one product child of collection " + lidvid + " in elasticSearch"); - - } - } - - - - summary.setProperties(new ArrayList(uniqueProperties)); - return products; - - - } catch (IOException e) { - MyCollectionsApiController.log.error("Couldn't get bundle " + lidvid + " from elasticSearch", e); - throw(e); + + int iteration=0,wsize=0; + HashSet uniqueProperties = new HashSet(); + List productLidvids = new ArrayList(); + List pageOfLidvids = new ArrayList(); + Products products = new Products(); + Summary summary = new Summary(); + + if (sort == null) { sort = Arrays.asList(); } + + summary.setStart(start); + summary.setLimit(limit); + summary.setSort(sort); + products.setSummary(summary); + + for (final Map kvp : new ElasticSearchHitIterator(this.esRegistryConnection.getRestHighLevelClient(), + ElasticSearchRegistrySearchRequestBuilder.getQueryFieldFromKVP("collection_lidvid", lidvid, "product_lidvid", + this.esRegistryConnection.getRegistryRefIndex()))) + { + pageOfLidvids.clear(); + wsize = 0; + + if (kvp.get("product_lidvid") instanceof String) + { pageOfLidvids.add(this.productBO.getLatestLidVidFromLid(kvp.get("product_lidvid").toString())); } + else + { + @SuppressWarnings("unchecked") + List clids = (List)kvp.get("product_lidvid"); + + // if we are working with data that we care about (between start and start + limit) then record them + if (start <= iteration || start < iteration+clids.size()) {pageOfLidvids.addAll(clids); } + // else just modify the counter to skip them without wasting CPU cycles processing them + else { wsize = clids.size(); } + } + + // if any data from the pages then add them to the complete roster + if (start <= iteration || start < iteration+pageOfLidvids.size()) + { productLidvids.addAll(pageOfLidvids.subList(start <= iteration ? 0 : start-iteration, pageOfLidvids.size())); } + + // if the limit of data has been found then break out of the loop + if (limit <= productLidvids.size()) { break; } + // otherwise update all of hte indices for the next iteration + else { iteration = iteration + pageOfLidvids.size() + wsize; } } - + + if (0 < productLidvids.size()) + { + this.fillProductsFromLidvids(products, uniqueProperties, + productLidvids.subList(0, productLidvids.size() < limit ? productLidvids.size() : limit), fields, onlySummary); + } + else MyCollectionsApiController.log.warn("Did not find any products for collection lidvid: " + lidvid); + + summary.setProperties(new ArrayList(uniqueProperties)); + return products; } @@ -218,7 +202,6 @@ public ResponseEntity bundlesContainingCollection(String lidvid, @Vali else return new ResponseEntity(HttpStatus.NOT_IMPLEMENTED); } - @SuppressWarnings("unchecked") private Products getContainingBundle(String lidvid, int start, int limit, List fields, List sort, boolean summaryOnly) throws IOException { @@ -228,9 +211,8 @@ private Products getContainingBundle(String lidvid, int start, int limit, List uniqueProperties = new HashSet(); Products products = new Products(); - SearchRequest request = new SearchRequest(this.esRegistryConnection.getRegistryIndex()); - SearchResponse response; - SearchSourceBuilder builder = new SearchSourceBuilder(); + SearchRequest request = ElasticSearchRegistrySearchRequestBuilder.getQueryFieldsFromKVP("ref_lid_collection", + lidvid.substring(0, lidvid.indexOf("::")), fields, this.esRegistryConnection.getRegistryIndex(), false); Summary summary = new Summary(); if (sort == null) { sort = Arrays.asList(); } @@ -239,35 +221,9 @@ private Products getContainingBundle(String lidvid, int start, int limit, List sourceAsMap = response.getHits().getAt(i).getSourceAsMap(); - Map filteredMapJsonProperties = ProductBusinessObject.getFilteredProperties( - sourceAsMap, - fields, - new ArrayList(Arrays.asList(ElasticSearchUtil.elasticPropertyToJsonProperty(EntitytProductWithBlob.BLOB_PROPERTY))) - ); - - uniqueProperties.addAll(filteredMapJsonProperties.keySet()); - - if (!summaryOnly) { - EntityProduct entityProduct = objectMapper.convertValue(sourceAsMap, EntityProduct.class); - - Product product = ElasticSearchUtil.ESentityProductToAPIProduct(entityProduct, this.getBaseURL()); - product.setProperties((Map)(Map)filteredMapJsonProperties); - - products.addDataItem(product); - } - } - } catch (UnsupportedElasticSearchProperty e) { - log.error("This should never happen " + e.getMessage()); - } + request.source().size(limit); + request.source().from(start); + this.fillProductsFromParents(products, uniqueProperties, ElasticSearchUtil.collate(this.esRegistryConnection.getRestHighLevelClient(), request), summaryOnly); summary.setProperties(new ArrayList(uniqueProperties)); return products; } diff --git a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiBareController.java b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiBareController.java index be23777..0e7a30b 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiBareController.java +++ b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiBareController.java @@ -5,7 +5,6 @@ import java.net.URL; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -14,14 +13,10 @@ import javax.servlet.http.HttpServletRequest; import org.antlr.v4.runtime.misc.ParseCancellationException; -import org.elasticsearch.action.get.GetRequest; -import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -32,15 +27,16 @@ import com.fasterxml.jackson.databind.ObjectMapper; +import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchHitIterator; import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchRegistryConnection; import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchRegistrySearchRequestBuilder; import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchUtil; import gov.nasa.pds.api.engineering.elasticsearch.business.ProductBusinessObject; import gov.nasa.pds.api.engineering.elasticsearch.entities.EntityProduct; -import gov.nasa.pds.api.engineering.elasticsearch.entities.EntitytProductWithBlob; -import gov.nasa.pds.api.engineering.exceptions.UnsupportedElasticSearchProperty; + import gov.nasa.pds.api.model.xml.ProductWithXmlLabel; import gov.nasa.pds.api.model.xml.XMLMashallableProperyValue; + import gov.nasa.pds.model.Product; import gov.nasa.pds.model.PropertyArrayValues; import gov.nasa.pds.model.Products; @@ -82,42 +78,42 @@ public MyProductsApiBareController(ObjectMapper objectMapper, HttpServletRequest } - - - private boolean proxyRunsOnDefaultPort() { - return (((this.context.getScheme() == "https") && (this.context.getServerPort() == 443)) - || ((this.context.getScheme() == "http") && (this.context.getServerPort() == 80))); + @SuppressWarnings("unchecked") +protected void fillProductsFromLidvids (Products products, HashSet uniqueProperties, List lidvids, List fields, boolean onlySummary) throws IOException + { + for (final Map kvp : new ElasticSearchHitIterator(lidvids.size(), this.esRegistryConnection.getRestHighLevelClient(), + ElasticSearchRegistrySearchRequestBuilder.getQueryFieldsFromKVP("lidvid", + lidvids, fields, this.esRegistryConnection.getRegistryIndex()))) + { + uniqueProperties.addAll(kvp.keySet()); + + if (!onlySummary) + { + products.addDataItem(ElasticSearchUtil.ESentityProductToAPIProduct(objectMapper.convertValue(kvp, EntityProduct.class), this.getBaseURL())); + products.getData().get(products.getData().size()-1).setProperties((Map)(Map)ProductBusinessObject.getFilteredProperties(kvp, null, null)); + } + } + } - - - protected URL getBaseURL() { - try { - MyProductsApiBareController.log.debug("contextPath is: " + this.contextPath); - - URL baseURL; - if (this.proxyRunsOnDefaultPort()) { - baseURL = new URL(this.context.getScheme(), this.context.getServerName(), this.contextPath); - } - else { - baseURL = new URL(this.context.getScheme(), this.context.getServerName(), this.context.getServerPort(), this.contextPath); - } - - MyProductsApiBareController.log.info("baseUrl is " + baseURL.toString()); - return baseURL; - + @SuppressWarnings("unchecked") + protected void fillProductsFromParents (Products products, HashSet uniqueProperties, List> results, boolean onlySummary) throws IOException + { + for (Map kvp : results) + { + uniqueProperties.addAll(kvp.keySet()); - } catch (MalformedURLException e) { - log.error("Server URL was not retrieved"); - return null; + if (!onlySummary) + { + products.addDataItem(ElasticSearchUtil.ESentityProductToAPIProduct(objectMapper.convertValue(kvp, EntityProduct.class), this.getBaseURL())); + products.getData().get(products.getData().size()-1).setProperties((Map)(Map)ProductBusinessObject.getFilteredProperties(kvp, null, null)); + } } } - + @SuppressWarnings("unchecked") protected Products getProducts(String q, int start, int limit, List fields, List sort, boolean onlySummary) throws IOException { - - SearchRequest searchRequest = this.searchRequestBuilder.getSearchProductsRequest(q, fields, start, limit, this.presetCriteria); @@ -246,5 +242,31 @@ protected ResponseEntity getProductResponseEntity(String lidvid){ return new ResponseEntity(HttpStatus.NOT_IMPLEMENTED); } + private boolean proxyRunsOnDefaultPort() { + return (((this.context.getScheme() == "https") && (this.context.getServerPort() == 443)) + || ((this.context.getScheme() == "http") && (this.context.getServerPort() == 80))); + } + + protected URL getBaseURL() { + try { + MyProductsApiBareController.log.debug("contextPath is: " + this.contextPath); + + URL baseURL; + if (this.proxyRunsOnDefaultPort()) { + baseURL = new URL(this.context.getScheme(), this.context.getServerName(), this.contextPath); + } + else { + baseURL = new URL(this.context.getScheme(), this.context.getServerName(), this.context.getServerPort(), this.contextPath); + } + + MyProductsApiBareController.log.info("baseUrl is " + baseURL.toString()); + return baseURL; + + + } catch (MalformedURLException e) { + log.error("Server URL was not retrieved"); + return null; + } + } } diff --git a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java index 849e6c6..49584e5 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java +++ b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java @@ -10,15 +10,7 @@ import javax.servlet.http.HttpServletRequest; import javax.validation.Valid; -import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.builder.SearchSourceBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; @@ -30,15 +22,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import gov.nasa.pds.api.base.ProductsApi; +import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchHitIterator; +import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchRegistrySearchRequestBuilder; import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchUtil; -import gov.nasa.pds.api.engineering.elasticsearch.business.ProductBusinessObject; -import gov.nasa.pds.api.engineering.elasticsearch.entities.EntityProduct; -import gov.nasa.pds.api.engineering.elasticsearch.entities.EntitytProductWithBlob; -import gov.nasa.pds.api.engineering.exceptions.UnsupportedElasticSearchProperty; -import gov.nasa.pds.api.model.xml.ProductWithXmlLabel; -import gov.nasa.pds.api.model.xml.XMLMashallableProperyValue; import gov.nasa.pds.model.Product; -import gov.nasa.pds.model.PropertyArrayValues; import gov.nasa.pds.model.Products; import gov.nasa.pds.model.Summary; import io.swagger.annotations.ApiParam; @@ -105,22 +92,15 @@ public ResponseEntity bundlesContainingProduct(String lidvid, @Valid I } - @SuppressWarnings("unchecked") private Products getContainingBundle(String lidvid, @Valid Integer start, @Valid Integer limit, @Valid List fields, @Valid List sort, @Valid Boolean summaryOnly) throws IOException - { - ProductBusinessObject productBO = new ProductBusinessObject(this.esRegistryConnection); - + { if (!lidvid.contains("::")) lidvid = productBO.getLatestLidVidFromLid(lidvid); - - boolean haveMatches = false; - MyProductsApiController.log.info("find all bundles containing the collection lidvid: " + lidvid); - HashSet uniqueProperties = new HashSet(); + MyProductsApiController.log.info("find all bundles containing the product lidvid: " + lidvid); + + HashSet uniqueProperties = new HashSet(); + List collectionLIDs = this.getCollectionLidvids(lidvid, true); Products products = new Products(); - BoolQueryBuilder query = QueryBuilders.boolQuery(); - SearchRequest request = new SearchRequest(this.esRegistryConnection.getRegistryIndex()); - SearchResponse response; - SearchSourceBuilder builder = new SearchSourceBuilder(); Summary summary = new Summary(); if (sort == null) { sort = Arrays.asList(); } @@ -129,45 +109,20 @@ private Products getContainingBundle(String lidvid, @Valid Integer start, @Valid summary.setLimit(limit); summary.setSort(sort); products.setSummary(summary); - for (SearchHit hit : this.getCollections(lidvid)) - { - haveMatches = true; - query.should (QueryBuilders.matchQuery("ref_lid_collection", hit.getSourceAsMap().get("collection_lid"))); - } - - if (haveMatches) + + if (0 < collectionLIDs.size()) { - builder.query(query); - request.source(builder); - response = this.esRegistryConnection.getRestHighLevelClient().search(request,RequestOptions.DEFAULT); + SearchRequest request = ElasticSearchRegistrySearchRequestBuilder.getQueryFieldsFromKVP + ("ref_lid_collection", collectionLIDs, fields, this.esRegistryConnection.getRegistryIndex(), false); - try { - - for (int i = start ; start < limit && i < response.getHits().getHits().length ; i++) - { - Map sourceAsMap = response.getHits().getAt(i).getSourceAsMap(); - Map filteredMapJsonProperties = ProductBusinessObject.getFilteredProperties( - sourceAsMap, - fields, - new ArrayList(Arrays.asList(ElasticSearchUtil.elasticPropertyToJsonProperty(EntitytProductWithBlob.BLOB_PROPERTY))) - ); - - uniqueProperties.addAll(filteredMapJsonProperties.keySet()); - - if (!summaryOnly) - { - EntityProduct entityProduct = objectMapper.convertValue(sourceAsMap, EntityProduct.class); - - Product product = ElasticSearchUtil.ESentityProductToAPIProduct(entityProduct, this.getBaseURL()); - product.setProperties((Map)(Map)filteredMapJsonProperties); - - products.addDataItem(product); - } - } - } catch (UnsupportedElasticSearchProperty e) { - log.error("this should never happen " + e.getMessage()); - } + request.source().from(start); + request.source().size(limit); + this.fillProductsFromParents(products, uniqueProperties, + ElasticSearchUtil.collate(this.esRegistryConnection.getRestHighLevelClient(), request), + summaryOnly); } + else MyProductsApiController.log.warn ("No parent collection for product LIDVID: " + lidvid); + summary.setProperties(new ArrayList(uniqueProperties)); return products; } @@ -201,29 +156,37 @@ public ResponseEntity collectionsContainingProduct(String lidvid, @Val else return new ResponseEntity(HttpStatus.NOT_IMPLEMENTED); } - private SearchHits getCollections (String lidvid) throws IOException + private List getCollectionLidvids (String lidvid, boolean noVer) throws IOException { - SearchRequest request = new SearchRequest(this.esRegistryConnection.getRegistryRefIndex()); - SearchResponse response; - SearchSourceBuilder builder = new SearchSourceBuilder(); + List fields = new ArrayList(), lidvids = new ArrayList(); + String field = noVer ? "collection_lid" : "collection_lidvid"; - builder.query(QueryBuilders.matchQuery("product_lidvid", lidvid)); - request.source(builder); - response = this.esRegistryConnection.getRestHighLevelClient().search(request,RequestOptions.DEFAULT); - log.info("number of hits: " + Integer.toString(response.getHits().getHits().length)); - return response.getHits(); + fields.add(field); + for (final Map kvp : new ElasticSearchHitIterator(this.esRegistryConnection.getRestHighLevelClient(), + ElasticSearchRegistrySearchRequestBuilder.getQueryFieldsFromKVP("product_lidvid", + lidvid, fields, this.esRegistryConnection.getRegistryRefIndex(), false))) + { + if (kvp.get(field) instanceof String) + { lidvids.add(kvp.get(field).toString()); } + else + { + @SuppressWarnings("unchecked") + List clids = (List)kvp.get(field); + for (String clid : clids) { lidvids.add(clid); } + } + } + return lidvids; } - @SuppressWarnings("unchecked") private Products getContainingCollection(String lidvid, @Valid Integer start, @Valid Integer limit, @Valid List fields, @Valid List sort, @Valid Boolean summaryOnly) throws IOException { if (!lidvid.contains("::")) lidvid = this.productBO.getLatestLidVidFromLid(lidvid); - - MyProductsApiController.log.info("find all bundles containing the collection lidvid: " + lidvid); - HashSet uniqueProperties = new HashSet(); + MyProductsApiController.log.info("find all bundles containing the product lidvid: " + lidvid); + + HashSet uniqueProperties = new HashSet(); + List collectionLidvids = this.getCollectionLidvids(lidvid, false); Products products = new Products(); - SearchHits hits = this.getCollections(lidvid); Summary summary = new Summary(); if (sort == null) { sort = Arrays.asList(); } @@ -233,32 +196,12 @@ private Products getContainingCollection(String lidvid, @Valid Integer start, @V summary.setSort(sort); products.setSummary(summary); - try { - - for (int i = start ; start < limit && i < hits.getHits().length ; i++) - { - GetRequest request = new GetRequest(this.esRegistryConnection.getRegistryIndex(), (String)hits.getAt(i).getSourceAsMap().get("collection_lidvid")); - Map sourceAsMap = this.esRegistryConnection.getRestHighLevelClient().get(request, RequestOptions.DEFAULT).getSourceAsMap(); - Map filteredMapJsonProperties = ProductBusinessObject.getFilteredProperties( - sourceAsMap, - fields, - new ArrayList(Arrays.asList(ElasticSearchUtil.elasticPropertyToJsonProperty(EntitytProductWithBlob.BLOB_PROPERTY))) - ); - - uniqueProperties.addAll(filteredMapJsonProperties.keySet()); - - if (!summaryOnly) { - EntityProduct entityProduct = objectMapper.convertValue(sourceAsMap, EntityProduct.class); + if (0 < collectionLidvids.size()) + { this.fillProductsFromLidvids(products, uniqueProperties, + collectionLidvids.subList(start, collectionLidvids.size() < start+limit ? collectionLidvids.size() : +limit), fields, + summaryOnly); } + else MyProductsApiController.log.warn("Did not find a product with lidvid: " + lidvid); - Product product = ElasticSearchUtil.ESentityProductToAPIProduct(entityProduct, this.getBaseURL()); - product.setProperties((Map)(Map)filteredMapJsonProperties); - - products.addDataItem(product); - } - } - } catch (UnsupportedElasticSearchProperty e) { - log.error("this should never happen " + e.getMessage()); - } summary.setProperties(new ArrayList(uniqueProperties)); return products; } diff --git a/src/main/java/gov/nasa/pds/api/engineering/elasticsearch/ElasticSearchHitIterator.java b/src/main/java/gov/nasa/pds/api/engineering/elasticsearch/ElasticSearchHitIterator.java new file mode 100644 index 0000000..15f9a06 --- /dev/null +++ b/src/main/java/gov/nasa/pds/api/engineering/elasticsearch/ElasticSearchHitIterator.java @@ -0,0 +1,78 @@ +package gov.nasa.pds.api.engineering.elasticsearch; + +import java.io.IOException; +import java.util.Iterator; +import java.util.Map; + +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; + +public class ElasticSearchHitIterator implements Iterable>,Iterator> +{ + private int size=10; // define size to use here to prevent page skipping if elasticsearch default size ever changes + private int at=0, page=0; + private SearchHits currentBatch; + private RestHighLevelClient client; + private SearchRequest request; + + public ElasticSearchHitIterator (RestHighLevelClient client, SearchRequest request) throws IOException + { + super(); + this.client = client; + this.request = request; + this.currentBatch = this.fetch(); + } + + public ElasticSearchHitIterator (int size, RestHighLevelClient client, SearchRequest request) throws IOException + { + super(); + this.client = client; + this.request = request; + this.size = size; + this.currentBatch = this.fetch(); + } + + private SearchHits fetch() throws IOException + { + this.request.source().from(this.page * this.size); + this.request.source().size(this.size); + return this.client.search (this.request, RequestOptions.DEFAULT).getHits(); + } + + private SearchHit getAt() throws IOException + { + if (this.size <= this.at) + { + this.page++; + this.at = 0; + this.currentBatch = this.fetch(); + } + + return this.currentBatch.getAt(this.at); + } + + @Override + public boolean hasNext() { return this.currentBatch == null ? false : (this.at + this.page * this.size) < this.currentBatch.getTotalHits().value; } + + @Override + public Iterator> iterator() { return this; } + + @Override + public Map next() + { + if (this.hasNext()) + { + try + { + SearchHit hit = this.getAt(); + at++; + return hit.getSourceAsMap(); + } + catch (IOException ioe) { return null; } + } + else { return null; } + } +} diff --git a/src/main/java/gov/nasa/pds/api/engineering/elasticsearch/ElasticSearchRegistrySearchRequestBuilder.java b/src/main/java/gov/nasa/pds/api/engineering/elasticsearch/ElasticSearchRegistrySearchRequestBuilder.java index da1e04d..6e7e27c 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/elasticsearch/ElasticSearchRegistrySearchRequestBuilder.java +++ b/src/main/java/gov/nasa/pds/api/engineering/elasticsearch/ElasticSearchRegistrySearchRequestBuilder.java @@ -6,6 +6,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.concurrent.TimeUnit; import java.lang.Math; @@ -18,11 +19,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.databind.JavaType; -import com.fasterxml.jackson.databind.introspect.BeanPropertyDefinition; - -import ch.qos.logback.core.joran.util.beans.BeanDescription; - import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -44,6 +40,8 @@ public class ElasticSearchRegistrySearchRequestBuilder { private static final Logger log = LoggerFactory.getLogger(ElasticSearchRegistrySearchRequestBuilder.class); + private static final String[] DEFAULT_ALL_FIELDS = { "*" }; + private static final String[] DEFAULT_BLOB = { "ops:Label_File_Info/ops:blob" }; private String registryIndex; private String registryRefIndex; @@ -245,4 +243,85 @@ public SearchRequest getSearchCollectionRequest(String queryString, List } + static public SearchRequest getQueryFieldFromLidvid (String lidvid, String field, String es_index) + { + List fields = new ArrayList(), lidvids = new ArrayList(); + Map> kvps = new HashMap>(); + fields.add(field); + lidvids.add(lidvid); + kvps.put("lidvid", lidvids); + return getQueryForKVPs (kvps, fields, es_index); + } + + static public SearchRequest getQueryFieldFromKVP (String key, Listvalues, String field, String es_index) + { + List fields = new ArrayList(); + Map> kvps = new HashMap>(); + fields.add(field); + kvps.put(key, values); + return getQueryForKVPs (kvps, fields, es_index); + } + + static public SearchRequest getQueryFieldFromKVP (String key, String value, String field, String es_index) + { + List fields = new ArrayList(), values = new ArrayList(); + Map> kvps = new HashMap>(); + fields.add(field); + values.add(value); + kvps.put(key, values); + return getQueryForKVPs (kvps, fields, es_index); + } + + static public SearchRequest getQueryFieldsFromKVP (String key, String value, List fields, String es_index, boolean term) + { + List values = new ArrayList(); + Map> kvps = new HashMap>(); + values.add(value); + kvps.put(key, values); + return getQueryForKVPs (kvps, fields, es_index, term); + } + + static public SearchRequest getQueryFieldsFromKVP (String key, List values, List fields, String es_index) + { + Map> kvps = new HashMap>(); + kvps.put(key, values); + return getQueryForKVPs (kvps, fields, es_index); + } + + static public SearchRequest getQueryFieldsFromKVP (String key, List values, List fields, String es_index, boolean term) + { + Map> kvps = new HashMap>(); + kvps.put(key, values); + return getQueryForKVPs (kvps, fields, es_index, term); + } + + static public SearchRequest getQueryForKVPs (Map> kvps, List fields, String es_index) + { + return getQueryForKVPs (kvps, fields, es_index, true); + } + + static public SearchRequest getQueryForKVPs (Map> kvps, List fields, String es_index, boolean term) + { + String[] aFields = new String[fields == null ? 0 : fields.size() + EntityProduct.JSON_PROPERTIES.length]; + if (fields != null) + { + for (int i = 0 ; i < EntityProduct.JSON_PROPERTIES.length ; i++) aFields[i] = EntityProduct.JSON_PROPERTIES[i]; + for (int i = 0 ; i < fields.size(); i++) aFields[i+EntityProduct.JSON_PROPERTIES.length] = ElasticSearchUtil.jsonPropertyToElasticProperty(fields.get(i)); + } + + BoolQueryBuilder find_kvps = QueryBuilders.boolQuery(); + SearchRequest request = new SearchRequest(es_index) + .source(new SearchSourceBuilder().query(find_kvps) + .fetchSource(fields == null ? DEFAULT_ALL_FIELDS : aFields, DEFAULT_BLOB)); + + for (Entry> key : kvps.entrySet()) + { + for (String value : key.getValue()) + { + if (term) find_kvps.should (QueryBuilders.termQuery (key.getKey(), value)); + else find_kvps.should (QueryBuilders.matchQuery (key.getKey(), value)); + } + } + return request; + } } diff --git a/src/main/java/gov/nasa/pds/api/engineering/elasticsearch/ElasticSearchUtil.java b/src/main/java/gov/nasa/pds/api/engineering/elasticsearch/ElasticSearchUtil.java index 0bdd614..c271783 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/elasticsearch/ElasticSearchUtil.java +++ b/src/main/java/gov/nasa/pds/api/engineering/elasticsearch/ElasticSearchUtil.java @@ -1,19 +1,20 @@ package gov.nasa.pds.api.engineering.elasticsearch; -import java.net.MalformedURLException; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import org.apache.http.client.utils.URIBuilder; - +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.search.SearchHit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.web.servlet.support.ServletUriComponentsBuilder; import gov.nasa.pds.api.engineering.elasticsearch.entities.EntityProduct; import gov.nasa.pds.api.engineering.elasticsearch.entities.EntitytProductWithBlob; @@ -123,10 +124,8 @@ static private Product addPropertiesFromESEntity( product.setMetadata(meta); product.setObservingSystemComponents(observationSystemComponent); product.setTargets(targets); - - return product; - + return product; } static public ProductWithXmlLabel ESentityProductToAPIProduct(EntitytProductWithBlob ep, URL baseURL) { @@ -134,7 +133,6 @@ static public ProductWithXmlLabel ESentityProductToAPIProduct(EntitytProductWith ProductWithXmlLabel product = new ProductWithXmlLabel(); product.setLabelXml(ep.getPDS4XML()); return (ProductWithXmlLabel)addPropertiesFromESEntity(product, ep, baseURL); - } @@ -144,8 +142,16 @@ static public Product ESentityProductToAPIProduct(EntityProduct ep, URL baseURL) Product product = new Product(); return addPropertiesFromESEntity(product, ep, baseURL); - - } + + static public List> collate (RestHighLevelClient client, SearchRequest request) throws IOException + { + List> results = new ArrayList>(); + for (SearchHit hit : client.search(request, RequestOptions.DEFAULT).getHits()) + { + results.add(hit.getSourceAsMap()); + } + return results; + } } diff --git a/src/main/java/gov/nasa/pds/api/engineering/elasticsearch/business/ProductBusinessObject.java b/src/main/java/gov/nasa/pds/api/engineering/elasticsearch/business/ProductBusinessObject.java index e871929..6c0cf5e 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/elasticsearch/business/ProductBusinessObject.java +++ b/src/main/java/gov/nasa/pds/api/engineering/elasticsearch/business/ProductBusinessObject.java @@ -4,7 +4,6 @@ import java.net.URL; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -19,13 +18,11 @@ import org.elasticsearch.search.SearchHit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.lang.Nullable; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import gov.nasa.pds.api.engineering.controllers.MyProductsApiBareController; import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchRegistryConnection; import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchRegistrySearchRequestBuilder; import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchUtil; @@ -98,7 +95,6 @@ public String getLatestLidVidFromLid(String lid) throws IOException { } - @SuppressWarnings({ "unchecked", "rawtypes" }) private static XMLMashallableProperyValue object2PropertyValue(Object o) { XMLMashallableProperyValue pv = new XMLMashallableProperyValue(); @@ -280,8 +276,5 @@ public ProductWithXmlLabel getProductWithXml(String lidvid, URL baseURL, @Nullab else { return null; } - - - } } diff --git a/verify/issue_56.py b/verify/issue_56.py index d196fce..8aaf103 100755 --- a/verify/issue_56.py +++ b/verify/issue_56.py @@ -28,6 +28,6 @@ this_length = len (result.json()['data']) if 'data' in result.json() else 0 if this_length == length: print ('success', result.status_code, url) - else: print ('failed', length, '!=', this_length) + else: print ('failed', length, '!=', this_length, url) else: print ('failed', expectation, '!=', result.status_code, url) pass