encoding for to assist GBIF SE

AtlasOfLivingAustralia · Nov 20, 2019 · c0c2f72 · c0c2f72
1 parent b372093
commit c0c2f72
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 10 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -5,6 +5,7 @@ jdk:
 branches:
   only:
   - master
+  - encoding-fix
   - spring3
 before_install:
 - mkdir -p ~/.m2; wget -q -O ~/.m2/settings.xml https://raw.githubusercontent.com/AtlasOfLivingAustralia/travis-build-configuration/master/travis_maven_settings.xml

diff --git a/pom.xml b/pom.xml
@@ -11,7 +11,7 @@
 	<!--<groupId>au.org.ala</groupId>-->
 	<artifactId>layers-store</artifactId>
 	<packaging>jar</packaging>
-	<version>2.0.2</version>
+	<version>2.0.3-SNAPSHOT</version>
 	<name>ALA Spatial Layers Store</name>
 	<url>http://www.ala.org.au</url>
 

diff --git a/src/main/java/au/org/ala/layers/intersect/SimpleShapeFile.java b/src/main/java/au/org/ala/layers/intersect/SimpleShapeFile.java
@@ -20,13 +20,16 @@
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.nio.channels.FileChannel;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
 import java.util.TreeSet;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.LinkedBlockingQueue;
 
+import static java.nio.charset.StandardCharsets.ISO_8859_1;
+
 /**
  * SimpleShapeFile is a representation of a Shape File for
  * intersections with points
@@ -84,9 +87,6 @@ public class SimpleShapeFile extends Object implements Serializable {
     protected SimpleShapeFile() {
     }
 
-    public static String getDBFEncoding(){
-        return System.getProperty("dbf.encoding", "UTF-8");
-    }
 
     /**
      * Constructor for a SimpleShapeFile, requires .dbf and .shp files present
@@ -1506,6 +1506,12 @@ class DBFField extends Object implements Serializable {
     byte[] data;        //placeholder for reading byte blocks
     /* don't care autoinc */
 
+
+    static String convertToUTF8(String myString){
+        byte[] ptext = myString.getBytes(StandardCharsets.UTF_8);
+        return new String(ptext, StandardCharsets.UTF_8);
+    }
+
     /**
      * constructor for DBFField with first byte separated from
      * rest of the data structure
@@ -1523,15 +1529,15 @@ public DBFField(byte firstbyte, ByteBuffer buffer) {
             ba[i] = buffer.get();
         }
         try {
-            name = (new String(ba, SimpleShapeFile.getDBFEncoding())).trim().toUpperCase();
+            name = convertToUTF8(new String(ba, "ISO-8859-1").trim().toUpperCase());
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
         }
 
         byte[] ba2 = new byte[1];
         ba2[0] = buffer.get();
         try {
-            type = (new String(ba2, SimpleShapeFile.getDBFEncoding())).charAt(0);
+            type = convertToUTF8(new String(ba2, "ISO-8859-1").trim()).charAt(0);
         } catch (Exception e) {
             logger.error(e.getMessage(), e);
         }
@@ -1769,10 +1775,10 @@ record = new String[fields.size()];
             try {
                 switch (f.getType()) {
                     case 'C':            //string
-                        record[i] = (new String(data, SimpleShapeFile.getDBFEncoding())).trim();
+                        record[i] = new String(new String(data, "ISO-8859-1").trim().getBytes("ISO-8859-1"),  StandardCharsets.UTF_8);
                         break;
                     case 'N':            //number as string
-                        record[i] = (new String(data, SimpleShapeFile.getDBFEncoding())).trim();
+                        record[i] = new String(new String(data, "ISO-8859-1").trim().getBytes("ISO-8859-1"),  StandardCharsets.UTF_8);
                         break;
                 }
             } catch (Exception e) {
@@ -1795,10 +1801,10 @@ record = mergeColumns ? new String[1] : new String[columnIdx.length];
             try {
                 switch (f.getType()) {
                     case 'C':            //string
-                        fieldValues[i] = (new String(data, SimpleShapeFile.getDBFEncoding())).trim();
+                        fieldValues[i] = DBFField.convertToUTF8(new String(data, "ISO-8859-1").trim());
                         break;
                     case 'N':            //number as string
-                        fieldValues[i] = (new String(data, SimpleShapeFile.getDBFEncoding())).trim();
+                        fieldValues[i] = DBFField.convertToUTF8(new String(data, "ISO-8859-1").trim());
                         break;
                 }
             } catch (Exception e) {