AtlasOfLivingAustralia · charvolant · Mar 7, 2023 · May 24, 2022 · May 24, 2022 · Mar 1, 2023
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# sds   [![Build Status](https://travis-ci.com/AtlasOfLivingAustralia/sds.svg?branch#master)](https://travis-ci.com/AtlasOfLivingAustralia/sds)
+# sds   [![Build Status](https://travis-ci.com/AtlasOfLivingAustralia/sds.svg?branch=master)](https://app.travis-ci.com/github/AtlasOfLivingAustralia/sds)
 
 
 The sensitive data service manages sensitivity concerns in the conservation and biosecurity areas.

diff --git a/pom.xml b/pom.xml
@@ -9,7 +9,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>au.org.ala</groupId>
     <artifactId>sds</artifactId>
-    <version>1.4.7-SNAPSHOT</version>
+    <version>1.4.7</version>
     <issueManagement>
         <system>GitHub</system>
         <url>https://github.com/AtlasOfLivingAustralia/sds/issues</url>

diff --git a/src/main/java/au/org/ala/sds/model/SDSSpeciesListItemDTO.java b/src/main/java/au/org/ala/sds/model/SDSSpeciesListItemDTO.java
@@ -15,6 +15,7 @@
 package au.org.ala.sds.model;
 
 import com.google.common.collect.Lists;
+import org.codehaus.jackson.annotate.JsonIgnore;
 
 import java.util.List;
 import java.util.Map;
@@ -31,6 +32,7 @@ public class SDSSpeciesListItemDTO {
     private String dataResourceUid;
     private List<Map<String, String>> kvpValues;
     public static final List<String> commonNameLabels= Lists.newArrayList("commonname","vernacularname");
+    private String commonName;
 
     public String getGuid() {
         return guid;
@@ -40,6 +42,19 @@ public void setGuid(String guid) {
         this.guid = guid;
     }
 
+    public String getCommonName() {
+        return commonName;
+    }
+
+    public void setCommonName(String commonName) {
+        this.commonName = commonName;
+    }
+
+    // Added for the change data structure returned by lists
+    public void setLsid(String lsid) {
+        this.guid = lsid;
+    }
+
     public String getName() {
         return name;
     }
@@ -70,28 +85,35 @@ public void setFamily(String family) {
 
     public void setKvpValues(List<Map<String, String>> kvpValues) {
         this.kvpValues = kvpValues;
+
+        // family has moved to kvpValues
+        for(Map<String, String> pair: kvpValues){
+            if("family".equalsIgnoreCase(pair.get("key"))){
+                setFamily(pair.get("value"));
+            }
+        }
     }
     public String getKVPValue(String key){
         for(Map<String, String> pair: kvpValues){
-            if(key.equals(pair.get("key"))){
+            if(key.equalsIgnoreCase(pair.get("key").replaceAll("[^a-zA-Z]", ""))){
                 return pair.get("value");
             }
         }
         return null;
     }
-    public String getKVPValue(List<String> keys){
+    public String getKVPValueCommonName(){
         for(Map<String, String> pair: kvpValues){
-            if(keys.contains(pair.get("key").toLowerCase().replaceAll(" ", ""))){
+            if(commonNameLabels.contains(pair.get("key").toLowerCase().replaceAll(" ", ""))){
                 return pair.get("value");
             }
         }
-        return null;
+        return commonName;
     }
 
     @Override
     public String toString() {
         return "SDSSpeciesListItemDTO{" +
-                "guid='" + guid + '\'' +
+                "lsid='" + guid + '\'' +
                 ", name='" + name + '\'' +
                 ", dataResourceUid='" + dataResourceUid + '\'' +
                 ", kvpValues=" + kvpValues +

diff --git a/src/main/java/au/org/ala/sds/util/SensitiveSpeciesXmlBuilder.java b/src/main/java/au/org/ala/sds/util/SensitiveSpeciesXmlBuilder.java
@@ -71,7 +71,7 @@ public static boolean generateFromWebservices(OutputStream out, Date lastGenerat
         }
 
         //Step 1: get all of the items that have a guid
-        List<SDSSpeciesListItemDTO> guidItems = SpeciesListUtil.getSDSListItems(true);
+        List<SDSSpeciesListItemDTO> guidItems = SpeciesListUtil.getSDSListItems(sdsLists.keySet(), true);
         if(sdsLists.isEmpty() || guidItems == null || guidItems.isEmpty()) {
             return false;
         }
@@ -102,7 +102,7 @@ public static boolean generateFromWebservices(OutputStream out, Date lastGenerat
                 }
                 sensitiveSpecies.setAttribute("guid", item.getGuid());
                 sensitiveSpecies.setAttribute("rank", rank);
-                String commonName = item.getKVPValue(item.commonNameLabels);
+                String commonName = item.getKVPValueCommonName();
                 sensitiveSpecies.setAttribute("commonName", commonName != null ? commonName : "");
                 doc.getRootElement().addContent(sensitiveSpecies);
                 currentGuid = item.getGuid();
@@ -121,7 +121,7 @@ public static boolean generateFromWebservices(OutputStream out, Date lastGenerat
         }
 
         //Step 2: get all the items that could NOT be matched to the current species list
-        List<SDSSpeciesListItemDTO> unmatchedItems = SpeciesListUtil.getSDSListItems(false);
+        List<SDSSpeciesListItemDTO> unmatchedItems = SpeciesListUtil.getSDSListItems(sdsLists.keySet(), false);
         String currentName = "";
         sensitiveSpecies = null;
         instances = null;
@@ -143,7 +143,7 @@ public static boolean generateFromWebservices(OutputStream out, Date lastGenerat
                     logger.error("Unable to get rank for " + item.getName(), e);
                 }
                 sensitiveSpecies.setAttribute("rank", rank);
-                String commonName = item.getKVPValue(item.commonNameLabels);
+                String commonName = item.getKVPValueCommonName();
                 sensitiveSpecies.setAttribute("commonName", commonName != null ? commonName : "");
                 //sensitiveSpecies.setAttribute("commonName", st.getCommonName() != null ? st.getCommonName() : "");
                 doc.getRootElement().addContent(sensitiveSpecies);

diff --git a/src/main/java/au/org/ala/sds/util/SpeciesListUtil.java b/src/main/java/au/org/ala/sds/util/SpeciesListUtil.java
@@ -17,6 +17,7 @@
 
 import au.org.ala.sds.model.SDSSpeciesListDTO;
 import au.org.ala.sds.model.SDSSpeciesListItemDTO;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Logger;
 import org.codehaus.jackson.map.DeserializationConfig;
 import org.codehaus.jackson.map.ObjectMapper;
@@ -25,8 +26,7 @@
 import java.io.InputStream;
 import java.net.URL;
 import java.net.URLConnection;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 /**
  * Provides utility methods required to interface with the species list tool. Allows the SDS
@@ -83,21 +83,50 @@ public static Map<String,SDSSpeciesListDTO> getSDSLists(){
      * Retrieves the "isSDS" species list items ordering them by guid/scientific name
      * @return
      */
-    public static List<SDSSpeciesListItemDTO> getSDSListItems(boolean hasMatch){
+    public static List<SDSSpeciesListItemDTO> getSDSListItems(Collection<String> dataResourceUids, boolean hasMatch){
        try{
-           String suffix = hasMatch ? "&guid=isNotNull:guid&sort=guid" : "&guid=isNull:guid&sort=rawScientificName";
-           URL url = new URL(Configuration.getInstance().getListToolUrl() + "/ws/speciesListItems?isSDS=eq:true" + suffix);
-           ObjectMapper mapper = new ObjectMapper();
-           mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-           URLConnection connection = url.openConnection();
-           logger.debug("Looking up location using " + url);
-           InputStream inStream = connection.getInputStream();
+           String suffix = hasMatch ? "&sort=guid" : "&sort=rawScientificName";
+           List<SDSSpeciesListItemDTO> values = new ArrayList();
 
-           java.util.List<SDSSpeciesListItemDTO> values = mapper.readValue(
-                   inStream,
-                   new TypeReference<List<SDSSpeciesListItemDTO>>(){}
-           );
-           logger.debug(values);
+           String drUids = StringUtils.join(dataResourceUids, ",");
+           int offset = 0;
+           int max = 400;
+           boolean moreRecords = true;
+           while(moreRecords) {
+               URL url = new URL(Configuration.getInstance().getListToolUrl() + "/ws/speciesListItems?isSDS=eq:true" + suffix + "&druid=" + drUids + "&includeKVP=true&max=" + max + "&offset=" + offset);
+               offset += max;
+
+               ObjectMapper mapper = new ObjectMapper();
+               mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+               URLConnection connection = url.openConnection();
+               logger.error("Looking up location using " + url);
+               InputStream inStream = connection.getInputStream();
+
+               java.util.List<SDSSpeciesListItemDTO> drValues = mapper.readValue(
+                       inStream,
+                       new TypeReference<List<SDSSpeciesListItemDTO>>() {
+                       }
+               );
+
+               if (!hasMatch) {
+                   // include only records without an LSID
+                   for (SDSSpeciesListItemDTO item : drValues) {
+                       if (item.getGuid() == null) {
+                           values.add(item);
+                       }
+                   }
+               } else {
+                   // include only records with an LSID
+                   for (SDSSpeciesListItemDTO item : drValues) {
+                       if (item.getGuid() != null) {
+                           values.add(item);
+                       }
+                   }
+               }
+
+               moreRecords = drValues.size() == max;
+           }
+           logger.error(values);
            return values;
        } catch(Exception e){
            logger.error("Unable to get the list items. ", e);
@@ -106,8 +135,8 @@ public static List<SDSSpeciesListItemDTO> getSDSListItems(boolean hasMatch){
     }
 
     public static void main(String[] args){
-        getSDSLists();
-        getSDSListItems(true);
-        getSDSListItems(false);
+        Map<String, SDSSpeciesListDTO> sdsLists = getSDSLists();
+        getSDSListItems(sdsLists.keySet(), true);
+        getSDSListItems(sdsLists.keySet(), false);
     }
 }