Fix equals(), add toString(), optimize Maven dependencies, add better support of new languages

redpen-cli/pom.xml

@@ -119,6 +119,11 @@
             <artifactId>commons-cli</artifactId>
             <version>1.2</version>
         </dependency>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <version>1.1.1</version>
+        </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>

redpen-core/pom.xml

@@ -167,11 +167,6 @@
             <version>4.10</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>commons-cli</groupId>
-            <artifactId>commons-cli</artifactId>
-            <version>1.2</version>
-        </dependency>
         <dependency>
             <groupId>org.json</groupId>
             <artifactId>json</artifactId>
@@ -181,13 +176,6 @@
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
             <version>1.7.6</version>
-            <scope>provided</scope>
-            <optional>true</optional>
-        </dependency>
-        <dependency>
-            <groupId>ch.qos.logback</groupId>
-            <artifactId>logback-classic</artifactId>
-            <version>1.1.1</version>
         </dependency>
         <dependency>
             <groupId>org.pegdown</groupId>

redpen-core/src/main/java/cc/redpen/config/Configuration.java

@@ -143,6 +143,23 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE
         return getKey().hashCode();
     }
 
+    @Override public String toString() {
+        return "Configuration{" +
+          "lang='" + lang + '\'' +
+          ", tokenizer=" + tokenizer +
+          ", validatorConfigs=" + validatorConfigs +
+          ", symbolTable=" + symbolTable +
+          '}';
+    }
+
+    public static ConfigurationBuilder builder() {
+        return new ConfigurationBuilder();
+    }
+
+    public static ConfigurationBuilder builder(String lang) {
+        return new ConfigurationBuilder().setLanguage(lang);
+    }
+
     /**
      * Builder class of Configuration.
      */

redpen-core/src/main/java/cc/redpen/config/SymbolTable.java

@@ -171,9 +171,8 @@ public SymbolTable clone() {
     public String toString() {
         return "SymbolTable{" +
                 "symbolDictionary=" + symbolDictionary +
-                ", valueDictionary=" + valueDictionary +
-                ", type='" + variant + '\'' +
                 ", lang='" + lang + '\'' +
+                ", variant='" + variant + '\'' +
                 '}';
     }
 

redpen-core/src/main/java/cc/redpen/config/ValidatorConfiguration.java

@@ -104,7 +104,8 @@ public ValidatorConfiguration addAttribute(String name, boolean value) {
         if (this == o) return true;
         if (!(o instanceof ValidatorConfiguration)) return false;
         ValidatorConfiguration that = (ValidatorConfiguration)o;
-        return Objects.equals(configurationName, that.configurationName);
+        return Objects.equals(configurationName, that.configurationName) &&
+               Objects.equals(attributes, that.attributes);
     }
 
     @Override public int hashCode() {

redpen-core/src/main/java/cc/redpen/model/Document.java

@@ -121,6 +121,14 @@ public String toString() {
                 '}';
     }
 
+    public static DocumentBuilder builder() {
+        return new DocumentBuilder();
+    }
+
+    public static DocumentBuilder builder(RedPenTokenizer tokenizer) {
+        return new DocumentBuilder(tokenizer);
+    }
+
     public static class DocumentBuilder {
         private final RedPenTokenizer tokenizer;
         boolean built = false;

redpen-core/src/main/java/cc/redpen/parser/LaTeXParser.java

@@ -46,7 +46,7 @@ class LaTeXParser extends BaseDocumentParser {
     @Override
     public Document parse(InputStream inputStream, Optional<String> fileName, SentenceExtractor sentenceExtractor, RedPenTokenizer tokenizer)
             throws RedPenException {
-        Document.DocumentBuilder documentBuilder = new Document.DocumentBuilder(tokenizer);
+        Document.DocumentBuilder documentBuilder = Document.builder(tokenizer);
         fileName.ifPresent(documentBuilder::setFileName);
 
         StringBuilder sb = new StringBuilder();

redpen-core/src/main/java/cc/redpen/parser/MarkdownParser.java

@@ -55,7 +55,7 @@ class MarkdownParser extends BaseDocumentParser {
     @Override
     public Document parse(InputStream inputStream, Optional<String> fileName, SentenceExtractor sentenceExtractor, RedPenTokenizer tokenizer)
             throws RedPenException {
-        Document.DocumentBuilder documentBuilder = new Document.DocumentBuilder(tokenizer);
+        Document.DocumentBuilder documentBuilder = Document.builder(tokenizer);
         fileName.ifPresent(documentBuilder::setFileName);
 
         StringBuilder sb = new StringBuilder();

redpen-core/src/main/java/cc/redpen/parser/PlainTextParser.java

@@ -49,7 +49,7 @@ final public class PlainTextParser extends BaseDocumentParser implements Seriali
     @Override
     public Document parse(InputStream is, Optional<String> fileName, SentenceExtractor sentenceExtractor, RedPenTokenizer tokenizer)
             throws RedPenException {
-        Document.DocumentBuilder documentBuilder = new Document.DocumentBuilder(tokenizer);
+        Document.DocumentBuilder documentBuilder = Document.builder(tokenizer);
         fileName.ifPresent(documentBuilder::setFileName);
 
         List<Sentence> headers = new ArrayList<>();

redpen-core/src/main/java/cc/redpen/parser/SentenceExtractor.java

@@ -56,7 +56,7 @@ public class SentenceExtractor {
      * @param fullStopList set of end of sentence characters
      */
     SentenceExtractor(char[] fullStopList) {
-        this(fullStopList, extractRightQuotations(new Configuration.ConfigurationBuilder().build().getSymbolTable()));
+        this(fullStopList, extractRightQuotations(Configuration.builder().build().getSymbolTable()));
     }
 
     /**

redpen-core/src/main/java/cc/redpen/parser/WikiParser.java

@@ -94,7 +94,7 @@ private static boolean check(Pattern p, String target, List<String> head) {
     @Override
     public Document parse(InputStream is, Optional<String> filename, SentenceExtractor sentenceExtractor,
                           RedPenTokenizer tokenizer) throws RedPenException{
-        Document.DocumentBuilder documentBuilder = new Document.DocumentBuilder(tokenizer);
+        Document.DocumentBuilder documentBuilder = Document.builder(tokenizer);
         filename.ifPresent(documentBuilder::setFileName);
         BufferedReader br;
 

redpen-core/src/main/java/cc/redpen/parser/asciidoc/AsciiDocParser.java

@@ -102,7 +102,7 @@ private class State {
 
     @Override
     public Document parse(InputStream inputStream, Optional<String> fileName, SentenceExtractor sentenceExtractor, RedPenTokenizer tokenizer) throws RedPenException {
-        Document.DocumentBuilder documentBuilder = new Document.DocumentBuilder(tokenizer);
+        Document.DocumentBuilder documentBuilder = Document.builder(tokenizer);
         fileName.ifPresent(documentBuilder::setFileName);
 
         Model model = new Model(sentenceExtractor);

redpen-core/src/main/java/cc/redpen/util/DictionaryLoader.java

@@ -89,23 +89,19 @@ private E loadFromFile(File file) throws IOException {
      *
      * @param path           resource path
      * @param dictionaryName name of the resource
-     * @return word list
-     * @throws RedPenException when failed to load the dictionary
+     * @return word collection or empty if resource is missing
      */
-    public E loadCachedFromResource(String path, String dictionaryName) throws RedPenException {
-        E strings = resourceCache.computeIfAbsent(path, e -> {
+    public E loadCachedFromResource(String path, String dictionaryName) {
+        return resourceCache.computeIfAbsent(path, e -> {
             try {
-                return loadFromResource(path);
+                E result = loadFromResource(path);
+                LOG.info("Succeeded to load " + dictionaryName + ".");
+                return result;
             } catch (IOException ioe) {
-                LOG.error(ioe.getMessage());
-                return null;
+                LOG.error("Failed to load " + dictionaryName + ":" + path + ": " + ioe.getMessage());
+                return supplier.get();
             }
         });
-        if (strings == null) {
-            throw new RedPenException("Failed to load " + dictionaryName + ":" + path);
-        }
-        LOG.info("Succeeded to load " + dictionaryName + ".");
-        return strings;
     }
 
 

redpen-core/src/main/java/cc/redpen/util/SpellingUtils.java

@@ -37,14 +37,8 @@ public class SpellingUtils {
     protected static void loadDictionary(String lang) {
         if (dictionaries.get(lang) == null) {
             String defaultDictionaryFile = DEFAULT_RESOURCE_PATH + "/spellchecker-" + lang + ".dat";
-            Set<String> dictionary;
-            try {
-                dictionary = new DictionaryLoader<HashSet<String>>(
-                        HashSet::new, (set, line) -> set.add(line.toLowerCase())
-                ).loadCachedFromResource(defaultDictionaryFile, "spell dictionary");
-            } catch (Exception e) {
-                dictionary = new HashSet<>();
-            }
+            Set<String> dictionary = new DictionaryLoader<Set<String>>(HashSet::new, (set, line) -> set.add(line.toLowerCase()))
+              .loadCachedFromResource(defaultDictionaryFile, "spell dictionary");
             dictionaries.put(lang, Collections.unmodifiableSet(dictionary));
         }
     }

redpen-core/src/main/java/cc/redpen/validator/ValidatorFactory.java

@@ -47,8 +47,7 @@ private static void addValidatorPackage(String packageToAdd) {
     }
 
     public static Validator getInstance(String validatorName) throws RedPenException {
-        Configuration conf = new Configuration.ConfigurationBuilder()
-                .setLanguage("en")
+        Configuration conf = Configuration.builder()
                 .addValidatorConfig(new ValidatorConfiguration(validatorName))
                 .build();
         return getInstance(conf.getValidatorConfigs().get(0), conf.getSymbolTable());

redpen-core/src/main/java/cc/redpen/validator/section/WordFrequencyValidator.java

@@ -63,17 +63,12 @@ protected void init() throws RedPenException {
 
         String defaultDictionaryFile = DEFAULT_RESOURCE_PATH + "/word-frequency-" + getSymbolTable().getLang() + ".dat";
         referenceWordDeviations = new HashMap<>();
-        try {
-            referenceWordFrequencies =
-                    new DictionaryLoader<Map<String, Double>>(HashMap::new, (set, line) -> {
-                        String[] fields = line.split(" ");
-                        set.put(fields[1], Double.valueOf(fields[0]));
-                    }).loadCachedFromResource(defaultDictionaryFile, "word frequencies");
-            referenceStdDeviation = getDeviations(referenceWordFrequencies, referenceWordDeviations);
-
-        } catch (Exception ignored) {
-            referenceWordFrequencies = new HashMap<>();
-        }
+        referenceWordFrequencies =
+                new DictionaryLoader<Map<String, Double>>(HashMap::new, (set, line) -> {
+                    String[] fields = line.split(" ");
+                    set.put(fields[1], Double.valueOf(fields[0]));
+                }).loadCachedFromResource(defaultDictionaryFile, "word frequencies");
+        referenceStdDeviation = getDeviations(referenceWordFrequencies, referenceWordDeviations);
     }
 
     /**

redpen-core/src/main/java/cc/redpen/validator/sentence/DoubleNegativeValidator.java

@@ -25,11 +25,12 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
 import java.util.Set;
 
+import static java.util.Arrays.asList;
+
 /**
  * Detect double negative expressions in Japanese texts.
  */
@@ -82,6 +83,6 @@ protected void init() throws RedPenException {
 
     @Override
     public List<String> getSupportedLanguages() {
-        return Arrays.asList(Locale.JAPANESE.getLanguage(), Locale.ENGLISH.getLanguage());
+        return asList(Locale.JAPANESE.getLanguage(), Locale.ENGLISH.getLanguage());
     }
 }

redpen-core/src/main/java/cc/redpen/validator/sentence/KatakanaSpellCheckValidator.java

@@ -165,8 +165,7 @@ private void checkKatakanaSpell(Sentence sentence, String katakana) {
     protected void init() throws RedPenException {
         boolean disableDefault = getConfigAttributeAsBoolean("disable-default", false);
         if (!disableDefault) {
-            String defaultDictionaryFile = DEFAULT_RESOURCE_PATH
-                    + "/katakana-spellcheck.dat";
+            String defaultDictionaryFile = DEFAULT_RESOURCE_PATH + "/katakana-spellcheck.dat";
             exceptions = WORD_LIST.loadCachedFromResource(defaultDictionaryFile, "katakana word dictionary");
         }
 

redpen-core/src/main/java/cc/redpen/validator/sentence/StartWithCapitalLetterValidator.java

@@ -25,12 +25,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Optional;
-import java.util.Set;
+import java.util.*;
 
 /**
  * Check if the input sentence start with a capital letter.
@@ -82,9 +77,7 @@ public void validate(Sentence sentence) {
 
     @Override
     protected void init() throws RedPenException {
-
-        String defaultDictionaryFile = DEFAULT_RESOURCE_PATH
-                + "/default-capital-case-exception-list.dat";
+        String defaultDictionaryFile = DEFAULT_RESOURCE_PATH + "/default-capital-case-exception-list.dat";
         whiteList = WORD_LIST.loadCachedFromResource(defaultDictionaryFile, "capital letter exception dictionary");
 
         Optional<String> confFile = getConfigAttribute("dict");

redpen-core/src/main/java/cc/redpen/validator/sentence/WeakExpressionValidator.java

@@ -43,15 +43,8 @@ protected void init() throws RedPenException {
         super.init();
 
         String defaultDictionaryFile = DEFAULT_RESOURCE_PATH + "/weak-expressions-" + getSymbolTable().getLang() + ".dat";
-        try {
-            weakExpressions =
-                    new DictionaryLoader<List<String>>(ArrayList::new, (list, line) -> {
-                        list.add(line.trim().toLowerCase());
-                    }).loadCachedFromResource(defaultDictionaryFile, "weak expressions");
-
-        } catch (Exception ignored) {
-            weakExpressions = new ArrayList<>();
-        }
+        weakExpressions = new DictionaryLoader<List<String>>(ArrayList::new, (list, line) -> list.add(line.trim().toLowerCase()))
+          .loadCachedFromResource(defaultDictionaryFile, "weak expressions");
     }
 
     /**

redpen-core/src/test/java/cc/redpen/AsciiDocParserTest.java

@@ -46,7 +46,7 @@ public void setup() {
 
     @Test(expected = NullPointerException.class)
     public void testNullDocument() throws Exception {
-        Configuration configuration = new Configuration.ConfigurationBuilder().build();
+        Configuration configuration = Configuration.builder().build();
         DocumentParser parser = DocumentParser.ASCIIDOC;
         InputStream is = null;
         parser.parse(is, new SentenceExtractor(configuration.getSymbolTable()), configuration.getTokenizer());
@@ -130,7 +130,7 @@ public void testBasicDocument() throws UnsupportedEncodingException, RedPenExcep
         assertEquals(2, firstSection.getNumberOfParagraphs());
         assertEquals(0, firstSection.getNumberOfSubsections());
 
-        Configuration configuration = new Configuration.ConfigurationBuilder()
+        Configuration configuration = Configuration.builder()
                 .addValidatorConfig(new ValidatorConfiguration("Spelling"))
                 .addValidatorConfig(new ValidatorConfiguration("InvalidSymbol")).build();
 
@@ -435,7 +435,7 @@ private Document createResourceContent(String filename, String lang) {
         Document doc = null;
         try {
             InputStream in = new FileInputStream(this.getClass().getClassLoader().getResource("asciidoc/" + filename).getFile());
-            Configuration configuration = new Configuration.ConfigurationBuilder().setLanguage(lang).build();
+            Configuration configuration = Configuration.builder().setLanguage(lang).build();
             doc = parser.parse(
                     in,
                     new SentenceExtractor(configuration.getSymbolTable()),
@@ -464,7 +464,7 @@ private Document createFileContent(String inputDocumentString) {
         DocumentParser parser = DocumentParser.ASCIIDOC;
         Document doc = null;
         try {
-            Configuration configuration = new Configuration.ConfigurationBuilder().build();
+            Configuration configuration = Configuration.builder().build();
             doc = parser.parse(
                     inputDocumentString,
                     new SentenceExtractor(configuration.getSymbolTable()),