From bf0c6dac8049e38668c5171003f3b5eaa507307b Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 11 May 2021 10:18:10 -0700 Subject: [PATCH 01/14] start with the acceptance criteria being checked --- verify/issue_56.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 verify/issue_56.py diff --git a/verify/issue_56.py b/verify/issue_56.py new file mode 100755 index 0000000..cb0c925 --- /dev/null +++ b/verify/issue_56.py @@ -0,0 +1,30 @@ +#! /usr/bin/env python3 +'''verify the acceptance criteria for issue 4 + +*requires* requests +''' + +import requests + +TEST_N_CRITERIA=[ + ('http://localhost:8080/bundles/urn:nasa:pds:izenberg_pdart14_meap::1.0/collections',200,4), #57 + ('http://localhost:8080//bundles/urn:nasa:pds:izenberg_pdart14_meap::1.0/products',200,999), #59 + ('http://localhost:8080/collections/urn:nasa:pds:izenberg_pdart14_meap:data_eetable::1.0/bundles',200,999), #62 + ('http://localhost:8080/collections/urn:nasa:pds:izenberg_pdart14_meap:data_eetable::1.0/products',200,3), #58 + ('http://localhost:8080/products/urn:nasa:pds:izenberg_pdart14_meap:data_eetable:ele_evt_8hr_orbit_2012-2013::1.0/bundles',200,999), #60 + ('http://localhost:8080/products/urn:nasa:pds:izenberg_pdart14_meap:data_eetable:ele_evt_8hr_orbit_2012-2013::1.0/collections',200,999), #61 + ] + +for url,expectation,length in TEST_N_CRITERIA: + result = requests.get(url) + + if result.status_code == expectation: + this_length = len (result.json()['data']) if 'data' in result.json() else 0 + if this_length == length: + print ('success', result.status_code, url) + else: print ('failed', length, '!=', this_length) + else: print ('failed', expectation, '!=', result.status_code, url) + pass + + +# need fixin: 59, 60, 61, 62 From fc3faf2f4f1ff5bebf5ee1a67ed058b747ee2c18 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 11 May 2021 10:32:29 -0700 Subject: [PATCH 02/14] cosmetic changes --- .../engineering/elasticsearch/Antlr4SearchListenerTest.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/test/java/gov/nasa/pds/api/engineering/elasticsearch/Antlr4SearchListenerTest.java b/src/test/java/gov/nasa/pds/api/engineering/elasticsearch/Antlr4SearchListenerTest.java index a16feb8..11dc5da 100644 --- a/src/test/java/gov/nasa/pds/api/engineering/elasticsearch/Antlr4SearchListenerTest.java +++ b/src/test/java/gov/nasa/pds/api/engineering/elasticsearch/Antlr4SearchListenerTest.java @@ -9,7 +9,6 @@ import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.WildcardQueryBuilder; import org.junit.Test; @@ -136,7 +135,6 @@ public void testGroupedStatementAndExclusiveInequality() @Test public void testGroupedStatementAndInclusiveInequality() { - boolean result = true; String qs = "( timestamp_A ge 12 and timestamp_B le 27 )"; BoolQueryBuilder query = this.run(qs); @@ -233,7 +231,7 @@ public void testNestedGrouping() public void testNoWildcard() { String qs = "ref_lid_target eq urn:nasa:pds:context:target:planet.mercury"; - BoolQueryBuilder nest, query = this.run(qs); + BoolQueryBuilder query = this.run(qs); Assertions.assertEquals (query.must().size(), 1); Assertions.assertEquals (query.mustNot().size(), 0); @@ -247,7 +245,7 @@ public void testNoWildcard() public void testNoWildcardQuoted() { String qs = "ref_lid_target eq \"urn:nasa:pds:context:target:planet.mercury\""; - BoolQueryBuilder nest, query = this.run(qs); + BoolQueryBuilder query = this.run(qs); Assertions.assertEquals (query.must().size(), 1); Assertions.assertEquals (query.mustNot().size(), 0); From d387c7dacba68cbbc85252468a36261dd69226b9 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 11 May 2021 18:20:18 -0700 Subject: [PATCH 03/14] update the API The POM needs to request the correct version of the api JAR file. Update the 3 API interfaces with the new methods. Currently they are blank place holders and the appropriate branches will be created to fill in each of the bodies. The branches will be created from here then merged back. --- pom.xml | 2 +- .../controllers/MyBundlesApiController.java | 14 +++++----- .../MyCollectionsApiController.java | 18 ++++++------- .../controllers/MyProductsApiController.java | 26 ++++++++++++------- 4 files changed, 32 insertions(+), 28 deletions(-) diff --git a/pom.xml b/pom.xml index cba82b1..b4fccb9 100644 --- a/pom.xml +++ b/pom.xml @@ -220,7 +220,7 @@ gov.nasa.pds api - 0.0.1-SNAPSHOT + 0.2.0-SNAPSHOT diff --git a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java index 3a6fb00..6ab0795 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java +++ b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java @@ -5,7 +5,6 @@ import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchUtil; import gov.nasa.pds.api.engineering.elasticsearch.entities.EntityProduct; import gov.nasa.pds.api.model.ProductWithXmlLabel; -import gov.nasa.pds.model.ErrorMessage; import gov.nasa.pds.model.Product; import gov.nasa.pds.model.Products; import gov.nasa.pds.model.Summary; @@ -22,15 +21,9 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; -import org.springframework.web.bind.annotation.CookieValue; import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RequestPart; -import org.springframework.web.multipart.MultipartFile; -import javax.validation.constraints.*; import javax.validation.Valid; import javax.servlet.http.HttpServletRequest; import java.io.IOException; @@ -199,5 +192,12 @@ private ResponseEntity getBundlesCollections(String lidvid, int start, } + @Override + public ResponseEntity productsOfABundle(String arg0, @Valid Integer arg1, @Valid Integer arg2, + @Valid List arg3, @Valid List arg4, @Valid Boolean arg5) { + // TODO Auto-generated method stub + return null; + } + } diff --git a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java index b693093..867a599 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java +++ b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java @@ -2,7 +2,6 @@ import gov.nasa.pds.api.base.CollectionsApi; -import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchRegistrySearchRequestBuilder; import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchUtil; import gov.nasa.pds.api.engineering.elasticsearch.business.CollectionProductRefBusinessObject; import gov.nasa.pds.api.engineering.elasticsearch.business.CollectionProductRelationships; @@ -15,8 +14,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.swagger.annotations.*; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.RestHighLevelClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; @@ -113,13 +110,6 @@ public ResponseEntity productsOfACollection(@ApiParam(value = "lidvid private Products getProductChildren(String lidvid, int start, int limit, List fields, List sort, boolean onlySummary) throws IOException { MyCollectionsApiController.log.info("request bundle lidvid, collections children: " + lidvid); - - - - - SearchResponse searchCollectionRefResponse = null; - - RestHighLevelClient restHighLevelClient = this.esRegistryConnection.getRestHighLevelClient(); try { if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid); @@ -180,6 +170,14 @@ private Products getProductChildren(String lidvid, int start, int limit, List bundlesContainingCollection(String arg0, @Valid Integer arg1, @Valid Integer arg2, + @Valid List arg3, @Valid List arg4, @Valid Boolean arg5) { + // TODO Auto-generated method stub + return null; + } diff --git a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java index 5d3e148..0c361c4 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java +++ b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java @@ -1,19 +1,12 @@ package gov.nasa.pds.api.engineering.controllers; -import java.io.IOException; import java.util.List; -import java.util.Map; import javax.servlet.http.HttpServletRequest; import javax.validation.Valid; -import org.elasticsearch.action.get.GetRequest; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.PathVariable; @@ -22,9 +15,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import gov.nasa.pds.api.base.ProductsApi; -import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchUtil; -import gov.nasa.pds.api.engineering.elasticsearch.entities.EntityProduct; -import gov.nasa.pds.api.model.ProductWithXmlLabel; import gov.nasa.pds.model.Product; import gov.nasa.pds.model.Products; import io.swagger.annotations.ApiParam; @@ -61,6 +51,22 @@ public ResponseEntity productsByLidvid(@ApiParam(value = "lidvid (urn)" return this.getProductResponseEntity(lidvid); } + + @Override + public ResponseEntity bundlesContainingProduct(String arg0, @Valid Integer arg1, @Valid Integer arg2, + @Valid List arg3, @Valid List arg4, @Valid Boolean arg5) { + // TODO Auto-generated method stub + return null; + } + + + @Override + public ResponseEntity collectionsContainingProduct(String arg0, @Valid Integer arg1, @Valid Integer arg2, + @Valid List arg3, @Valid List arg4, @Valid Boolean arg5) { + // TODO Auto-generated method stub + return null; + } + From fc6d4e812794ddc049de0706cc8ee66938a71251 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 12 May 2021 13:58:21 -0700 Subject: [PATCH 04/14] basics for products of a bundle --- .../controllers/MyBundlesApiController.java | 121 ++++++++++++++++-- 1 file changed, 111 insertions(+), 10 deletions(-) diff --git a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java index 6ab0795..01ec668 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java +++ b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java @@ -106,7 +106,8 @@ private Products getCollectionChildren(String lidvid, int start, int limit, List if (getBundleResponse.isExists()) { MyBundlesApiController.log.info("get response " + getBundleResponse.toString()); - List collections = (List) getBundleResponse.getSourceAsMap().get("ref_lid_collection"); + @SuppressWarnings("unchecked") + List collections = (List) getBundleResponse.getSourceAsMap().get("ref_lid_collection"); String collectionLidVid; int i=0; for (String collectionLid : collections ) { @@ -161,9 +162,7 @@ private Products getCollectionChildren(String lidvid, int start, int limit, List } } - - - + private ResponseEntity getBundlesCollections(String lidvid, int start, int limit, List fields, List sort, boolean onlySummary) { String accept = this.request.getHeader("Accept"); MyBundlesApiController.log.info("accept value is " + accept); @@ -189,15 +188,117 @@ private ResponseEntity getBundlesCollections(String lidvid, int start, } else return new ResponseEntity(HttpStatus.NOT_IMPLEMENTED); - } @Override - public ResponseEntity productsOfABundle(String arg0, @Valid Integer arg1, @Valid Integer arg2, - @Valid List arg3, @Valid List arg4, @Valid Boolean arg5) { - // TODO Auto-generated method stub - return null; + public ResponseEntity productsOfABundle(String lidvid, @Valid Integer start, @Valid Integer limit, + @Valid List fields, @Valid List sort, @Valid Boolean onlySummary) { + String accept = this.request.getHeader("Accept"); + MyBundlesApiController.log.info("accept value is " + accept); + if ((accept != null + && (accept.contains("application/json") + || accept.contains("text/html") + || accept.contains("application/xml") + || accept.contains("application/pds4+xml") + || accept.contains("*/*"))) + || (accept == null)) { + + try { + + + Products products = this.getProductChildren(lidvid, start, limit, fields, sort, onlySummary); + + return new ResponseEntity(products, HttpStatus.OK); + + } catch (IOException e) { + log.error("Couldn't serialize response for content type " + accept, e); + return new ResponseEntity(HttpStatus.INTERNAL_SERVER_ERROR); + } + + } + else return new ResponseEntity(HttpStatus.NOT_IMPLEMENTED); } - + @SuppressWarnings("unchecked") + private Products getProductChildren(String lidvid, int start, int limit, List fields, List sort, boolean onlySummary) throws IOException + { + MyBundlesApiController.log.info("request bundle lidvid, children of products: " + lidvid); + + GetRequest getBundleRequest = new GetRequest(this.esRegistryConnection.getRegistryIndex(), lidvid); + HashSet uniqueProperties = new HashSet(); + List productLidvids = new ArrayList(); + Products products = new Products(); + RestHighLevelClient restHighLevelClient = this.esRegistryConnection.getRestHighLevelClient(); + Summary summary = new Summary(); + + if (sort == null) { sort = Arrays.asList(); } + + summary.setStart(start); + summary.setLimit(limit); + summary.setSort(sort); + products.setSummary(summary); + + try + { + GetResponse getBundleResponse = restHighLevelClient.get(getBundleRequest, RequestOptions.DEFAULT); + + if (getBundleResponse.isExists()) + { + MyBundlesApiController.log.info("get response " + getBundleResponse.toString()); + List collections = (List) getBundleResponse.getSourceAsMap().get("ref_lid_product"); + String collectionLidVid; + + for (String collectionLid : collections ) + { + collectionLidVid = this.getLatestLidVidFromLid(collectionLid); + GetRequest getCollectionRequest = new GetRequest(this.esRegistryConnection.getRegistryRefIndex(), collectionLidVid); + GetResponse getCollectionResponse = restHighLevelClient.get(getCollectionRequest, RequestOptions.DEFAULT); + + if (getCollectionResponse.isExists()) + { + MyBundlesApiController.log.info("get response " + getCollectionResponse.toString()); + productLidvids.addAll((List) getBundleResponse.getSourceAsMap().get("product_lidvid")); + } + else + { + MyBundlesApiController.log.warn("Couldn't get collection child " + collectionLidVid + " of bundle " + lidvid + " in elasticSearch"); + } + } + + for (int i=start ; i < start+limit && i < productLidvids.size() ; i++) + { + GetRequest getProductRequest = new GetRequest(this.esRegistryConnection.getRegistryIndex(), productLidvids.get(i)); + GetResponse getProductResponse = restHighLevelClient.get(getProductRequest, RequestOptions.DEFAULT); + + if (getProductResponse.isExists()) + { + Map sourceAsMap = getProductResponse.getSourceAsMap(); + Map filteredMapJsonProperties = this.getFilteredProperties(sourceAsMap, fields); + + uniqueProperties.addAll(filteredMapJsonProperties.keySet()); + + if (!onlySummary) + { + EntityProduct entityProduct = objectMapper.convertValue(sourceAsMap, EntityProduct.class); + ProductWithXmlLabel product = ElasticSearchUtil.ESentityProductToAPIProduct(entityProduct); + product.setProperties(filteredMapJsonProperties); + products.addDataItem(product); + } + } + else + { + MyBundlesApiController.log.warn("Couldn't get product child " + productLidvids.get(i) + " of bundle " + lidvid + " in elasticSearch"); + } + } + } + } + catch (IOException e) + { + MyBundlesApiController.log.error("Couldn't get bundle " + lidvid + " from elasticSearch", e); + throw(e); + } + + summary.setProperties(new ArrayList(uniqueProperties)); + return products; + } } From 316a62510703c4d3f2deeb567ec59a7c49a3ac89 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 12 May 2021 16:54:40 -0700 Subject: [PATCH 05/14] now get products of a bundle. works and tested --- .../controllers/MyBundlesApiController.java | 16 ++++++++++------ verify/issue_56.py | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java index 01ec668..96c5299 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java +++ b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java @@ -245,28 +245,32 @@ private Products getProductChildren(String lidvid, int start, int limit, List collections = (List) getBundleResponse.getSourceAsMap().get("ref_lid_product"); + List collections = (List) getBundleResponse.getSourceAsMap().get("ref_lid_collection"); String collectionLidVid; for (String collectionLid : collections ) { - collectionLidVid = this.getLatestLidVidFromLid(collectionLid); + collectionLidVid = this.getLatestLidVidFromLid(collectionLid) + "::P1"; + MyBundlesApiController.log.info("get collection with lidvid " + collectionLidVid); GetRequest getCollectionRequest = new GetRequest(this.esRegistryConnection.getRegistryRefIndex(), collectionLidVid); GetResponse getCollectionResponse = restHighLevelClient.get(getCollectionRequest, RequestOptions.DEFAULT); if (getCollectionResponse.isExists()) { - MyBundlesApiController.log.info("get response " + getCollectionResponse.toString()); - productLidvids.addAll((List) getBundleResponse.getSourceAsMap().get("product_lidvid")); + MyBundlesApiController.log.info("get ref response " + getCollectionResponse.toString()); + Object temp = getCollectionResponse.getSourceAsMap().get("product_lidvid"); + if (temp instanceof String) { productLidvids.add((String)temp); } + else { productLidvids.addAll((List) temp); } } else { MyBundlesApiController.log.warn("Couldn't get collection child " + collectionLidVid + " of bundle " + lidvid + " in elasticSearch"); } } - + MyBundlesApiController.log.info("total number of product lidvids " + Integer.toString(productLidvids.size())); for (int i=start ; i < start+limit && i < productLidvids.size() ; i++) - { + { + MyBundlesApiController.log.info("fetch product from lidvid " + productLidvids.get(i)); GetRequest getProductRequest = new GetRequest(this.esRegistryConnection.getRegistryIndex(), productLidvids.get(i)); GetResponse getProductResponse = restHighLevelClient.get(getProductRequest, RequestOptions.DEFAULT); diff --git a/verify/issue_56.py b/verify/issue_56.py index cb0c925..8881034 100755 --- a/verify/issue_56.py +++ b/verify/issue_56.py @@ -8,7 +8,7 @@ TEST_N_CRITERIA=[ ('http://localhost:8080/bundles/urn:nasa:pds:izenberg_pdart14_meap::1.0/collections',200,4), #57 - ('http://localhost:8080//bundles/urn:nasa:pds:izenberg_pdart14_meap::1.0/products',200,999), #59 + ('http://localhost:8080//bundles/urn:nasa:pds:izenberg_pdart14_meap::1.0/products',200,12), #59 ('http://localhost:8080/collections/urn:nasa:pds:izenberg_pdart14_meap:data_eetable::1.0/bundles',200,999), #62 ('http://localhost:8080/collections/urn:nasa:pds:izenberg_pdart14_meap:data_eetable::1.0/products',200,3), #58 ('http://localhost:8080/products/urn:nasa:pds:izenberg_pdart14_meap:data_eetable:ele_evt_8hr_orbit_2012-2013::1.0/bundles',200,999), #60 From d93d08b8634a97a938e545ee85c0ec02ae50f761 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 18 May 2021 10:20:50 -0700 Subject: [PATCH 06/14] prepare for finding owners --- .../controllers/MyBundlesApiController.java | 5 +- .../MyCollectionsApiController.java | 51 ++++++++-- .../controllers/MyProductsApiController.java | 98 +++++++++++++++++-- 3 files changed, 136 insertions(+), 18 deletions(-) diff --git a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java index 96c5299..ece7bce 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java +++ b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyBundlesApiController.java @@ -75,7 +75,9 @@ public ResponseEntity collectionsOfABundle(@ApiParam(value = "lidvid ( return this.getBundlesCollections(lidvid, start, limit, fields, sort, onlySummary); } - private Products getCollectionChildren(String lidvid, int start, int limit, List fields, List sort, boolean onlySummary) throws IOException { + private Products getCollectionChildren(String lidvid, int start, int limit, List fields, List sort, boolean onlySummary) throws IOException + { + if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid); MyBundlesApiController.log.info("request bundle lidvid, collections children: " + lidvid); GetRequest getBundleRequest = new GetRequest(this.esRegistryConnection.getRegistryIndex(), @@ -222,6 +224,7 @@ public ResponseEntity productsOfABundle(String lidvid, @Valid Integer @SuppressWarnings("unchecked") private Products getProductChildren(String lidvid, int start, int limit, List fields, List sort, boolean onlySummary) throws IOException { + if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid); MyBundlesApiController.log.info("request bundle lidvid, children of products: " + lidvid); GetRequest getBundleRequest = new GetRequest(this.esRegistryConnection.getRegistryIndex(), lidvid); diff --git a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java index 867a599..7765bdb 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java +++ b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java @@ -22,7 +22,6 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestParam; - import javax.validation.Valid; import javax.servlet.http.HttpServletRequest; @@ -109,11 +108,10 @@ public ResponseEntity productsOfACollection(@ApiParam(value = "lidvid private Products getProductChildren(String lidvid, int start, int limit, List fields, List sort, boolean onlySummary) throws IOException { + if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid); MyCollectionsApiController.log.info("request bundle lidvid, collections children: " + lidvid); try { - if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid); - Products products = new Products(); HashSet uniqueProperties = new HashSet(); @@ -173,12 +171,47 @@ private Products getProductChildren(String lidvid, int start, int limit, List bundlesContainingCollection(String arg0, @Valid Integer arg1, @Valid Integer arg2, - @Valid List arg3, @Valid List arg4, @Valid Boolean arg5) { - // TODO Auto-generated method stub - return null; + public ResponseEntity bundlesContainingCollection(String lidvid, @Valid Integer start, @Valid Integer limit, + @Valid List fields, @Valid List sort, @Valid Boolean summaryOnly) + { + String accept = this.request.getHeader("Accept"); + MyCollectionsApiController.log.info("accept value is " + accept); + + if ((accept != null + && (accept.contains("application/json") + || accept.contains("text/html") + || accept.contains("application/xml") + || accept.contains("application/pds4+xml") + || accept.contains("*/*"))) + || (accept == null)) + { + try + { + Products products = this.getContainingBundle(lidvid, start, limit, fields, sort, summaryOnly); + return new ResponseEntity(products, HttpStatus.OK); + } + catch (IOException e) + { + log.error("Couldn't serialize response for content type " + accept, e); + return new ResponseEntity(HttpStatus.INTERNAL_SERVER_ERROR); + } + } + else return new ResponseEntity(HttpStatus.NOT_IMPLEMENTED); } - - + private Products getContainingBundle(String lidvid, int start, int limit, List fields, List sort, boolean summaryOnly) throws IOException + { + if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid); + MyCollectionsApiController.log.info("find all bundles containing the collection lidvid: " + lidvid); + Products products = new Products(); + Summary summary = new Summary(); + + if (sort == null) { sort = Arrays.asList(); } + + summary.setStart(start); + summary.setLimit(limit); + summary.setSort(sort); + products.setSummary(summary); + return products; + } } \ No newline at end of file diff --git a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java index 0c361c4..a388732 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java +++ b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java @@ -1,5 +1,7 @@ package gov.nasa.pds.api.engineering.controllers; +import java.io.IOException; +import java.util.Arrays; import java.util.List; import javax.servlet.http.HttpServletRequest; @@ -7,6 +9,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.PathVariable; @@ -17,6 +20,7 @@ import gov.nasa.pds.api.base.ProductsApi; import gov.nasa.pds.model.Product; import gov.nasa.pds.model.Products; +import gov.nasa.pds.model.Summary; import io.swagger.annotations.ApiParam; @@ -53,18 +57,96 @@ public ResponseEntity productsByLidvid(@ApiParam(value = "lidvid (urn)" @Override - public ResponseEntity bundlesContainingProduct(String arg0, @Valid Integer arg1, @Valid Integer arg2, - @Valid List arg3, @Valid List arg4, @Valid Boolean arg5) { - // TODO Auto-generated method stub - return null; + public ResponseEntity bundlesContainingProduct(String lidvid, @Valid Integer start, @Valid Integer limit, + @Valid List fields, @Valid List sort, @Valid Boolean summaryOnly) { + String accept = this.request.getHeader("Accept"); + MyProductsApiController.log.info("accept value is " + accept); + + if ((accept != null + && (accept.contains("application/json") + || accept.contains("text/html") + || accept.contains("application/xml") + || accept.contains("application/pds4+xml") + || accept.contains("*/*"))) + || (accept == null)) + { + try + { + Products products = this.getContainingBundle(lidvid, start, limit, fields, sort, summaryOnly); + return new ResponseEntity(products, HttpStatus.OK); + } + catch (IOException e) + { + log.error("Couldn't serialize response for content type " + accept, e); + return new ResponseEntity(HttpStatus.INTERNAL_SERVER_ERROR); + } + } + else return new ResponseEntity(HttpStatus.NOT_IMPLEMENTED); + } + + + private Products getContainingBundle(String lidvid, @Valid Integer start, @Valid Integer limit, + @Valid List fields, @Valid List sort, @Valid Boolean summaryOnly) throws IOException + { + if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid); + MyProductsApiController.log.info("find all bundles containing the collection lidvid: " + lidvid); + Products products = new Products(); + Summary summary = new Summary(); + + if (sort == null) { sort = Arrays.asList(); } + + summary.setStart(start); + summary.setLimit(limit); + summary.setSort(sort); + products.setSummary(summary); + return products; } @Override - public ResponseEntity collectionsContainingProduct(String arg0, @Valid Integer arg1, @Valid Integer arg2, - @Valid List arg3, @Valid List arg4, @Valid Boolean arg5) { - // TODO Auto-generated method stub - return null; + public ResponseEntity collectionsContainingProduct(String lidvid, @Valid Integer start, @Valid Integer limit, + @Valid List fields, @Valid List sort, @Valid Boolean summaryOnly) { + String accept = this.request.getHeader("Accept"); + MyProductsApiController.log.info("accept value is " + accept); + + if ((accept != null + && (accept.contains("application/json") + || accept.contains("text/html") + || accept.contains("application/xml") + || accept.contains("application/pds4+xml") + || accept.contains("*/*"))) + || (accept == null)) + { + try + { + Products products = this.getContainingCollection(lidvid, start, limit, fields, sort, summaryOnly); + return new ResponseEntity(products, HttpStatus.OK); + } + catch (IOException e) + { + log.error("Couldn't serialize response for content type " + accept, e); + return new ResponseEntity(HttpStatus.INTERNAL_SERVER_ERROR); + } + } + else return new ResponseEntity(HttpStatus.NOT_IMPLEMENTED); + } + + + private Products getContainingCollection(String lidvid, @Valid Integer start, @Valid Integer limit, + @Valid List fields, @Valid List sort, @Valid Boolean summaryOnly) throws IOException + { + if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid); + MyProductsApiController.log.info("find all bundles containing the collection lidvid: " + lidvid); + Products products = new Products(); + Summary summary = new Summary(); + + if (sort == null) { sort = Arrays.asList(); } + + summary.setStart(start); + summary.setLimit(limit); + summary.setSort(sort); + products.setSummary(summary); + return products; } From fa85dcb8c5f5c354920c0c02e916438de6f1a2d6 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 18 May 2021 14:53:02 -0700 Subject: [PATCH 07/14] implement bundle from collection Basically use the search capabilities to find where the collection lid shows up in the ref_lid_collection variable. Pretty straight forward. --- .../MyCollectionsApiController.java | 28 +++++++++++++++++++ verify/issue_56.py | 4 +-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java index 7765bdb..00c0bd0 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java +++ b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyCollectionsApiController.java @@ -14,6 +14,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.swagger.annotations.*; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; @@ -203,7 +208,12 @@ private Products getContainingBundle(String lidvid, int start, int limit, List uniqueProperties = new HashSet(); Products products = new Products(); + SearchRequest request = new SearchRequest(this.esRegistryConnection.getRegistryIndex()); + SearchResponse response; + SearchSourceBuilder builder = new SearchSourceBuilder(); Summary summary = new Summary(); if (sort == null) { sort = Arrays.asList(); } @@ -212,6 +222,24 @@ private Products getContainingBundle(String lidvid, int start, int limit, List sourceAsMap = response.getHits().getAt(i).getSourceAsMap(); + Map filteredMapJsonProperties = this.getFilteredProperties(sourceAsMap, fields); + + uniqueProperties.addAll(filteredMapJsonProperties.keySet()); + + if (!summaryOnly) { + EntityProduct entityProduct = objectMapper.convertValue(sourceAsMap, EntityProduct.class); + ProductWithXmlLabel product = ElasticSearchUtil.ESentityProductToAPIProduct(entityProduct); + product.setProperties(filteredMapJsonProperties); + products.addDataItem(product); + } + } + summary.setProperties(new ArrayList(uniqueProperties)); return products; } } \ No newline at end of file diff --git a/verify/issue_56.py b/verify/issue_56.py index 8881034..dc2a499 100755 --- a/verify/issue_56.py +++ b/verify/issue_56.py @@ -9,7 +9,7 @@ TEST_N_CRITERIA=[ ('http://localhost:8080/bundles/urn:nasa:pds:izenberg_pdart14_meap::1.0/collections',200,4), #57 ('http://localhost:8080//bundles/urn:nasa:pds:izenberg_pdart14_meap::1.0/products',200,12), #59 - ('http://localhost:8080/collections/urn:nasa:pds:izenberg_pdart14_meap:data_eetable::1.0/bundles',200,999), #62 + ('http://localhost:8080/collections/urn:nasa:pds:izenberg_pdart14_meap:data_eetable::1.0/bundles',200,1), #62 ('http://localhost:8080/collections/urn:nasa:pds:izenberg_pdart14_meap:data_eetable::1.0/products',200,3), #58 ('http://localhost:8080/products/urn:nasa:pds:izenberg_pdart14_meap:data_eetable:ele_evt_8hr_orbit_2012-2013::1.0/bundles',200,999), #60 ('http://localhost:8080/products/urn:nasa:pds:izenberg_pdart14_meap:data_eetable:ele_evt_8hr_orbit_2012-2013::1.0/collections',200,999), #61 @@ -27,4 +27,4 @@ pass -# need fixin: 59, 60, 61, 62 +# need fixin: 60, 61, 62 From 131b3e563c2ddd64b469bb0a12d0a3d068f9b4b3 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 18 May 2021 14:57:18 -0700 Subject: [PATCH 08/14] editorial --- pom.xml | 3 ++- verify/issue_56.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index b4fccb9..ef220c9 100644 --- a/pom.xml +++ b/pom.xml @@ -220,7 +220,8 @@ gov.nasa.pds api - 0.2.0-SNAPSHOT + + 0.2.0-20210512.222845-4 diff --git a/verify/issue_56.py b/verify/issue_56.py index dc2a499..c72aeaa 100755 --- a/verify/issue_56.py +++ b/verify/issue_56.py @@ -27,4 +27,4 @@ pass -# need fixin: 60, 61, 62 +# need fixin: 60, 61 From 08d4e933f8cd7be1d14c8ad2eeffe1289f38e0a0 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 18 May 2021 15:04:11 -0700 Subject: [PATCH 09/14] fix what should not have changed --- pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index ef220c9..b4fccb9 100644 --- a/pom.xml +++ b/pom.xml @@ -220,8 +220,7 @@ gov.nasa.pds api - - 0.2.0-20210512.222845-4 + 0.2.0-SNAPSHOT From 1b1a89890694f99810d3f0408a6bbbe8a9201274 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 18 May 2021 15:55:24 -0700 Subject: [PATCH 10/14] find bundles of a product Pretty straight foward finding all collections that hold a this product then all bundles that reference those collections. --- .../controllers/MyProductsApiController.java | 57 +++++++++++++++++-- verify/issue_56.py | 6 +- 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java index a388732..fe74f42 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java +++ b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java @@ -1,12 +1,24 @@ package gov.nasa.pds.api.engineering.controllers; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Map; import javax.servlet.http.HttpServletRequest; import javax.validation.Valid; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; @@ -18,6 +30,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import gov.nasa.pds.api.base.ProductsApi; +import gov.nasa.pds.api.engineering.elasticsearch.ElasticSearchUtil; +import gov.nasa.pds.api.engineering.elasticsearch.entities.EntityProduct; +import gov.nasa.pds.api.model.ProductWithXmlLabel; import gov.nasa.pds.model.Product; import gov.nasa.pds.model.Products; import gov.nasa.pds.model.Summary; @@ -90,7 +105,12 @@ private Products getContainingBundle(String lidvid, @Valid Integer start, @Valid { if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid); MyProductsApiController.log.info("find all bundles containing the collection lidvid: " + lidvid); + HashSet uniqueProperties = new HashSet(); Products products = new Products(); + BoolQueryBuilder query = QueryBuilders.boolQuery(); + SearchRequest request = new SearchRequest(this.esRegistryConnection.getRegistryIndex()); + SearchResponse response; + SearchSourceBuilder builder = new SearchSourceBuilder(); Summary summary = new Summary(); if (sort == null) { sort = Arrays.asList(); } @@ -99,6 +119,28 @@ private Products getContainingBundle(String lidvid, @Valid Integer start, @Valid summary.setLimit(limit); summary.setSort(sort); products.setSummary(summary); + for (SearchHit hit : this.getCollections(lidvid)) + { + query.should (QueryBuilders.matchQuery("ref_lid_collection", hit.getSourceAsMap().get("collection_lid"))); + } + builder.query(query); + request.source(builder); + response = this.esRegistryConnection.getRestHighLevelClient().search(request,RequestOptions.DEFAULT); + for (int i = start ; start < limit && i < response.getHits().getHits().length ; i++) + { + Map sourceAsMap = response.getHits().getAt(i).getSourceAsMap(); + Map filteredMapJsonProperties = this.getFilteredProperties(sourceAsMap, fields); + + uniqueProperties.addAll(filteredMapJsonProperties.keySet()); + + if (!summaryOnly) { + EntityProduct entityProduct = objectMapper.convertValue(sourceAsMap, EntityProduct.class); + ProductWithXmlLabel product = ElasticSearchUtil.ESentityProductToAPIProduct(entityProduct); + product.setProperties(filteredMapJsonProperties); + products.addDataItem(product); + } + } + summary.setProperties(new ArrayList(uniqueProperties)); return products; } @@ -131,6 +173,17 @@ public ResponseEntity collectionsContainingProduct(String lidvid, @Val else return new ResponseEntity(HttpStatus.NOT_IMPLEMENTED); } + private SearchHits getCollections (String lidvid) throws IOException + { + SearchRequest request = new SearchRequest(this.esRegistryConnection.getRegistryRefIndex()); + SearchResponse response; + SearchSourceBuilder builder = new SearchSourceBuilder(); + + builder.query(QueryBuilders.matchQuery("product_lidvid", lidvid)); + request.source(builder); + response = this.esRegistryConnection.getRestHighLevelClient().search(request,RequestOptions.DEFAULT); + return response.getHits(); + } private Products getContainingCollection(String lidvid, @Valid Integer start, @Valid Integer limit, @Valid List fields, @Valid List sort, @Valid Boolean summaryOnly) throws IOException @@ -148,8 +201,4 @@ private Products getContainingCollection(String lidvid, @Valid Integer start, @V products.setSummary(summary); return products; } - - - - } diff --git a/verify/issue_56.py b/verify/issue_56.py index c72aeaa..962ee6a 100755 --- a/verify/issue_56.py +++ b/verify/issue_56.py @@ -11,8 +11,8 @@ ('http://localhost:8080//bundles/urn:nasa:pds:izenberg_pdart14_meap::1.0/products',200,12), #59 ('http://localhost:8080/collections/urn:nasa:pds:izenberg_pdart14_meap:data_eetable::1.0/bundles',200,1), #62 ('http://localhost:8080/collections/urn:nasa:pds:izenberg_pdart14_meap:data_eetable::1.0/products',200,3), #58 - ('http://localhost:8080/products/urn:nasa:pds:izenberg_pdart14_meap:data_eetable:ele_evt_8hr_orbit_2012-2013::1.0/bundles',200,999), #60 - ('http://localhost:8080/products/urn:nasa:pds:izenberg_pdart14_meap:data_eetable:ele_evt_8hr_orbit_2012-2013::1.0/collections',200,999), #61 + ('http://localhost:8080/products/urn:nasa:pds:izenberg_pdart14_meap:data_eetable:ele_evt_8hr_orbit_2012-2013::1.0/bundles',200,1), #60 + ('http://localhost:8080/products/urn:nasa:pds:izenberg_pdart14_meap:data_eetable:ele_evt_8hr_orbit_2012-2013::1.0/collections',200,1), #61 ] for url,expectation,length in TEST_N_CRITERIA: @@ -27,4 +27,4 @@ pass -# need fixin: 60, 61 +# need fixin: 61 From 9c286025559c164bc7bb17ac25310eab843eeaa3 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 18 May 2021 16:17:22 -0700 Subject: [PATCH 11/14] collections that reference product --- .../controllers/MyProductsApiController.java | 18 +++++++++++++++++- verify/issue_56.py | 3 --- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java index fe74f42..ca9c1af 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java +++ b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java @@ -14,7 +14,6 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -190,7 +189,9 @@ private Products getContainingCollection(String lidvid, @Valid Integer start, @V { if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid); MyProductsApiController.log.info("find all bundles containing the collection lidvid: " + lidvid); + HashSet uniqueProperties = new HashSet(); Products products = new Products(); + SearchHits hits = this.getCollections(lidvid); Summary summary = new Summary(); if (sort == null) { sort = Arrays.asList(); } @@ -199,6 +200,21 @@ private Products getContainingCollection(String lidvid, @Valid Integer start, @V summary.setLimit(limit); summary.setSort(sort); products.setSummary(summary); + for (int i = start ; start < limit && i < hits.getHits().length ; i++) + { + Map sourceAsMap = hits.getAt(i).getSourceAsMap(); + Map filteredMapJsonProperties = this.getFilteredProperties(sourceAsMap, fields); + + uniqueProperties.addAll(filteredMapJsonProperties.keySet()); + + if (!summaryOnly) { + EntityProduct entityProduct = objectMapper.convertValue(sourceAsMap, EntityProduct.class); + ProductWithXmlLabel product = ElasticSearchUtil.ESentityProductToAPIProduct(entityProduct); + product.setProperties(filteredMapJsonProperties); + products.addDataItem(product); + } + } + summary.setProperties(new ArrayList(uniqueProperties)); return products; } } diff --git a/verify/issue_56.py b/verify/issue_56.py index 962ee6a..f023f83 100755 --- a/verify/issue_56.py +++ b/verify/issue_56.py @@ -25,6 +25,3 @@ else: print ('failed', length, '!=', this_length) else: print ('failed', expectation, '!=', result.status_code, url) pass - - -# need fixin: 61 From 007eac3854a9cd01af9e8ee99e3f5f6bc857b23d Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 19 May 2021 15:43:20 -0700 Subject: [PATCH 12/14] fix whitespacing --- .../api/engineering/controllers/MyProductsApiController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java index ca9c1af..e9bcaba 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java +++ b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java @@ -110,7 +110,7 @@ private Products getContainingBundle(String lidvid, @Valid Integer start, @Valid SearchRequest request = new SearchRequest(this.esRegistryConnection.getRegistryIndex()); SearchResponse response; SearchSourceBuilder builder = new SearchSourceBuilder(); - Summary summary = new Summary(); + Summary summary = new Summary(); if (sort == null) { sort = Arrays.asList(); } From 8b720fdf675ba87a9c6fb4d3f910ffeaa2d80b1e Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 19 May 2021 16:06:34 -0700 Subject: [PATCH 13/14] get actual collection --- .../api/engineering/controllers/MyProductsApiController.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java index e9bcaba..57a25b8 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java +++ b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java @@ -10,6 +10,7 @@ import javax.servlet.http.HttpServletRequest; import javax.validation.Valid; +import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.RequestOptions; @@ -202,7 +203,8 @@ private Products getContainingCollection(String lidvid, @Valid Integer start, @V products.setSummary(summary); for (int i = start ; start < limit && i < hits.getHits().length ; i++) { - Map sourceAsMap = hits.getAt(i).getSourceAsMap(); + GetRequest request = new GetRequest(this.esRegistryConnection.getRegistryIndex(), (String)hits.getAt(i).getSourceAsMap().get("collection_lidvid")); + Map sourceAsMap = this.esRegistryConnection.getRestHighLevelClient().get(request, RequestOptions.DEFAULT).getSourceAsMap(); Map filteredMapJsonProperties = this.getFilteredProperties(sourceAsMap, fields); uniqueProperties.addAll(filteredMapJsonProperties.keySet()); From 516ad807e913700463b31388512e049a93676b85 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Thu, 20 May 2021 09:14:19 -0700 Subject: [PATCH 14/14] do negative checks If the lidvid is wrong, make sure that nothing is returned. This was added because it was found that the double search of /products/x/bundles returned multiple responses instead of 0. Because the second search had no criteria it returned the entire database. Added code to prevent this then added negative checks for all URLs. --- .../controllers/MyProductsApiController.java | 38 +++++++++++-------- verify/issue_56.py | 6 +++ 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java index 57a25b8..fc4126f 100644 --- a/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java +++ b/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiController.java @@ -104,6 +104,7 @@ private Products getContainingBundle(String lidvid, @Valid Integer start, @Valid @Valid List fields, @Valid List sort, @Valid Boolean summaryOnly) throws IOException { if (!lidvid.contains("::") && !lidvid.endsWith(":")) lidvid = this.getLatestLidVidFromLid(lidvid); + boolean haveMatches = false; MyProductsApiController.log.info("find all bundles containing the collection lidvid: " + lidvid); HashSet uniqueProperties = new HashSet(); Products products = new Products(); @@ -121,24 +122,30 @@ private Products getContainingBundle(String lidvid, @Valid Integer start, @Valid products.setSummary(summary); for (SearchHit hit : this.getCollections(lidvid)) { + haveMatches = true; query.should (QueryBuilders.matchQuery("ref_lid_collection", hit.getSourceAsMap().get("collection_lid"))); } - builder.query(query); - request.source(builder); - response = this.esRegistryConnection.getRestHighLevelClient().search(request,RequestOptions.DEFAULT); - for (int i = start ; start < limit && i < response.getHits().getHits().length ; i++) + + if (haveMatches) { - Map sourceAsMap = response.getHits().getAt(i).getSourceAsMap(); - Map filteredMapJsonProperties = this.getFilteredProperties(sourceAsMap, fields); - - uniqueProperties.addAll(filteredMapJsonProperties.keySet()); - - if (!summaryOnly) { - EntityProduct entityProduct = objectMapper.convertValue(sourceAsMap, EntityProduct.class); - ProductWithXmlLabel product = ElasticSearchUtil.ESentityProductToAPIProduct(entityProduct); - product.setProperties(filteredMapJsonProperties); - products.addDataItem(product); - } + builder.query(query); + request.source(builder); + response = this.esRegistryConnection.getRestHighLevelClient().search(request,RequestOptions.DEFAULT); + for (int i = start ; start < limit && i < response.getHits().getHits().length ; i++) + { + Map sourceAsMap = response.getHits().getAt(i).getSourceAsMap(); + Map filteredMapJsonProperties = this.getFilteredProperties(sourceAsMap, fields); + + uniqueProperties.addAll(filteredMapJsonProperties.keySet()); + + if (!summaryOnly) + { + EntityProduct entityProduct = objectMapper.convertValue(sourceAsMap, EntityProduct.class); + ProductWithXmlLabel product = ElasticSearchUtil.ESentityProductToAPIProduct(entityProduct); + product.setProperties(filteredMapJsonProperties); + products.addDataItem(product); + } + } } summary.setProperties(new ArrayList(uniqueProperties)); return products; @@ -182,6 +189,7 @@ private SearchHits getCollections (String lidvid) throws IOException builder.query(QueryBuilders.matchQuery("product_lidvid", lidvid)); request.source(builder); response = this.esRegistryConnection.getRestHighLevelClient().search(request,RequestOptions.DEFAULT); + log.info("number of hits: " + Integer.toString(response.getHits().getHits().length)); return response.getHits(); } diff --git a/verify/issue_56.py b/verify/issue_56.py index f023f83..d196fce 100755 --- a/verify/issue_56.py +++ b/verify/issue_56.py @@ -8,11 +8,17 @@ TEST_N_CRITERIA=[ ('http://localhost:8080/bundles/urn:nasa:pds:izenberg_pdart14_meap::1.0/collections',200,4), #57 + ('http://localhost:8080/bundles/urn:nasa:pds:izenberg_pdart13_meap::1.0/collections',200,0), ('http://localhost:8080//bundles/urn:nasa:pds:izenberg_pdart14_meap::1.0/products',200,12), #59 + ('http://localhost:8080//bundles/urn:nasa:pds:izenberg_pdart13_meap::1.0/products',200,0), ('http://localhost:8080/collections/urn:nasa:pds:izenberg_pdart14_meap:data_eetable::1.0/bundles',200,1), #62 + ('http://localhost:8080/collections/urn:nasa:pds:izenberg_pdart13_meap:data_eetable::1.0/bundles',200,0), ('http://localhost:8080/collections/urn:nasa:pds:izenberg_pdart14_meap:data_eetable::1.0/products',200,3), #58 + ('http://localhost:8080/collections/urn:nasa:pds:izenberg_pdart13_meap:data_eetable::1.0/products',200,0), ('http://localhost:8080/products/urn:nasa:pds:izenberg_pdart14_meap:data_eetable:ele_evt_8hr_orbit_2012-2013::1.0/bundles',200,1), #60 + ('http://localhost:8080/products/urn:nasa:pds:izenberg_pdart13_meap:data_eetable:ele_evt_8hr_orbit_2012-2013::1.0/bundles',200,0), ('http://localhost:8080/products/urn:nasa:pds:izenberg_pdart14_meap:data_eetable:ele_evt_8hr_orbit_2012-2013::1.0/collections',200,1), #61 + ('http://localhost:8080/products/urn:nasa:pds:izenberg_pdart13_meap:data_eetable:ele_evt_8hr_orbit_2012-2013::1.0/collections',200,0), ] for url,expectation,length in TEST_N_CRITERIA: