Add spellchecking

bench/__init__.py

@@ -34,7 +34,6 @@ def __init__(self, lang_code: str, ids: Set[str]):
             lt_code, remote_server="http://localhost:8081/"
         )
         self.tool.disabled_rules = {
-            "MORFOLOGIK_RULE_EN_US",
             "GERMAN_SPELLER_RULE",
             "COMMA_PARENTHESIS_WHITESPACE",
             "DOUBLE_PUNCTUATION",
@@ -117,6 +116,7 @@ class NLPRule:
     def __init__(self, lang_code: str):
         self.tokenizer = nlprule.Tokenizer(f"storage/{lang_code}_tokenizer.bin")
         self.rules = nlprule.Rules(f"storage/{lang_code}_rules.bin", self.tokenizer)
+        self.rules.spell.options.variant = "en_US"
 
     def suggest(self, sentence: str) -> Set[Suggestion]:
         suggestions = {

build/README.md

@@ -79,6 +79,7 @@ python build/make_build_dir.py \
     --chunker_token_model=$HOME/Downloads/nlprule/en-token.bin \
     --chunker_pos_model=$HOME/Downloads/nlprule/en-pos-maxent.bin \
     --chunker_chunk_model=$HOME/Downloads/nlprule/en-chunker.bin \
+    --spell_map_path=$LT_PATH/org/languagetool/rules/en/contractions.txt \
     --out_dir=data/en
 ```
 

build/make_build_dir.py

@@ -6,6 +6,7 @@
 from zipfile import ZipFile
 import lxml.etree as ET
 import wordfreq
+from glob import glob
 from chardet.universaldetector import UniversalDetector
 
 from chunker import write_chunker  # type: ignore
@@ -59,7 +60,7 @@ def copy_lt_files(out_dir, lt_dir, lang_code):
         canonicalize(out_dir / xmlfile)
 
 
-def dump_dictionary(out_path, lt_dir, tag_dict_path, tag_info_path):
+def dump_dict(out_path, lt_dir, tag_dict_path, tag_info_path):
     # dump dictionary, see https://dev.languagetool.org/developing-a-tagger-dictionary
     os.system(
         f"java -cp {lt_dir / 'languagetool.jar'} org.languagetool.tools.DictionaryExporter "
@@ -83,7 +84,46 @@ def dump_dictionary(out_path, lt_dir, tag_dict_path, tag_info_path):
     dump_bytes = open(out_path, "rb").read()
 
     with open(out_path, "w") as f:
-        f.write(dump_bytes.decode(result["encoding"]))
+        f.write(dump_bytes.decode(result["encoding"] or "utf-8"))
+
+
+def proc_spelling_text(in_paths, out_path, lang_code):
+    with open(out_path, "w") as f:
+        for in_path in in_paths:
+            if in_path.exists():
+                for line in open(in_path):
+                    # strip comments
+                    comment_index = line.find("#")
+                    if comment_index != -1:
+                        line = line[:comment_index]
+
+                    line = line.strip()
+                    if len(line) == 0:
+                        continue
+
+                    try:
+                        word, suffix = line.split("/")
+
+                        assert lang_code == "de", "Flags are only supported for German!"
+
+                        for flag in suffix:
+                            assert flag != "Ä"
+                            if flag == "A" and word.endswith("e"):
+                                flag = "Ä"
+
+                            f.write(word + "\n")
+
+                            for ending in {
+                                "S": ["s"],
+                                "N": ["n"],
+                                "E": ["e"],
+                                "F": ["in"],
+                                "A": ["e", "er", "es", "en", "em"],
+                                "Ä": ["r", "s", "n", "m"],
+                            }[flag]:
+                                f.write(word + ending + "\n")
+                    except ValueError:
+                        f.write(line + "\n")
 
 
 if __name__ == "__main__":
@@ -138,6 +178,12 @@ def dump_dictionary(out_path, lt_dir, tag_dict_path, tag_info_path):
         default=None,
         help="Path to the OpenNLP chunker binary. See token model message for details.",
     )
+    parser.add_argument(
+        "--spell_map_path",
+        default=None,
+        action="append",
+        help="Paths to files containing a mapping from incorrect words to correct ones e.g. contractions.txt for English.",
+    )
     parser.add_argument(
         "--out_dir",
         type=lambda p: Path(p).absolute(),
@@ -149,12 +195,72 @@ def dump_dictionary(out_path, lt_dir, tag_dict_path, tag_info_path):
 
     write_freqlist(open(args.out_dir / "common.txt", "w"), args.lang_code)
     copy_lt_files(args.out_dir, args.lt_dir, args.lang_code)
-    dump_dictionary(
+
+    # tagger dictionary
+    dump_dict(
         args.out_dir / "tags" / "output.dump",
         args.lt_dir,
         args.tag_dict_path,
         args.tag_info_path,
     )
+
+    # spell dictionaries
+    (args.out_dir / "spell").mkdir()
+    for dic in glob(
+        str(
+            args.lt_dir
+            / "org"
+            / "languagetool"
+            / "resource"
+            / args.lang_code
+            / "hunspell"
+            / "*.dict"
+        )
+    ):
+        dic = Path(dic)
+        info = Path(dic).with_suffix(".info")
+
+        variant_name = dic.stem
+
+        dump_dict(
+            args.out_dir / "spell" / f"{variant_name}.dump", args.lt_dir, dic, info,
+        )
+        proc_spelling_text(
+            [
+                (
+                    dic / ".." / ("spelling_" + variant_name.replace("_", "-") + ".txt")
+                ).resolve(),
+                (
+                    dic / ".." / ("spelling-" + variant_name.replace("_", "-") + ".txt")
+                ).resolve(),
+            ],
+            args.out_dir / "spell" / f"{variant_name}.txt",
+            args.lang_code,
+        )
+
+    proc_spelling_text(
+        [
+            args.lt_dir
+            / "org"
+            / "languagetool"
+            / "resource"
+            / args.lang_code
+            / "hunspell"
+            / "spelling.txt"
+        ],
+        args.out_dir / "spell" / "spelling.txt",
+        args.lang_code,
+    )
+
+    with open(args.out_dir / "spell" / "map.txt", "w") as f:
+        for path in args.spell_map_path or []:
+            for line in open(path):
+                if line.startswith("#"):
+                    continue
+
+                assert "#" not in line
+                f.write(line)
+
     if (
         args.chunker_token_model is not None
         and args.chunker_pos_model is not None

build/src/lib.rs

@@ -5,7 +5,7 @@ use flate2::bufread::GzDecoder;
 use fs::File;
 use fs_err as fs;
 use nlprule::{compile, rules_filename, tokenizer_filename};
-use std::fs::Permissions;
+use std::{fs::Permissions, sync::Arc};
 use std::{
     io::{self, BufReader, BufWriter, Cursor, Read},
     path::{Path, PathBuf},
@@ -469,10 +469,11 @@ impl BinaryBuilder {
             let tokenizer_out = self.out_dir.join(tokenizer_filename(lang_code));
             let rules_out = self.out_dir.join(rules_filename(lang_code));
 
-            nlprule::Rules::new(rules_out)
-                .map_err(|e| Error::ValidationFailed(lang_code.to_owned(), Binary::Rules, e))?;
-            nlprule::Tokenizer::new(tokenizer_out)
+            let tokenizer = nlprule::Tokenizer::new(tokenizer_out)
                 .map_err(|e| Error::ValidationFailed(lang_code.to_owned(), Binary::Tokenizer, e))?;
+
+            nlprule::Rules::new(rules_out, Arc::new(tokenizer))
+                .map_err(|e| Error::ValidationFailed(lang_code.to_owned(), Binary::Rules, e))?;
         }
 
         Ok(())

nlprule/Cargo.toml

@@ -30,6 +30,8 @@ half = { version = "1.7", features = ["serde"] }
 srx = { version = "^0.1.2", features = ["serde"] }
 lazycell = "1"
 cfg-if = "1"
+fnv = "1"
+unicode_categories = "0.1"
 
 rayon-cond = "0.1"
 rayon = "1.5"

nlprule/build.rs

@@ -20,6 +20,7 @@ fn main() {
         ("tokenizer.json", "tokenizer_configs.json"),
         ("rules.json", "rules_configs.json"),
         ("tagger.json", "tagger_configs.json"),
+        ("spellchecker.json", "spellchecker_configs.json"),
     ] {
         let mut config_map: HashMap<String, serde_json::Value> = HashMap::new();
 

nlprule/configs/de/spellchecker.json

@@ -0,0 +1,8 @@
+{
+    "variants": [
+        "de_AT",
+        "de_DE",
+        "de_CH"
+    ],
+    "split_hyphens": true
+}

nlprule/configs/en/rules.json

@@ -3,5 +3,6 @@
     "ignore_ids": [
         "GRAMMAR/PRP_MD_NN/2",
         "TYPOS/VERB_APOSTROPHE_S/3"
-    ]
+    ],
+    "split_hyphens": true
 }

nlprule/configs/en/spellchecker.json

@@ -0,0 +1,8 @@
+{
+    "variants": [
+        "en_GB",
+        "en_US",
+        "en_AU"
+    ],
+    "split_hyphens": true
+}

nlprule/configs/es/spellchecker.json

@@ -0,0 +1,4 @@
+{
+    "variants": [],
+    "split_hyphens": true
+}

nlprule/src/bin/run.rs

@@ -1,3 +1,5 @@
+use std::sync::Arc;
+
 use clap::Clap;
 use nlprule::{rules::Rules, tokenizer::Tokenizer};
 
@@ -18,11 +20,12 @@ fn main() {
     env_logger::init();
     let opts = Opts::parse();
 
-    let tokenizer = Tokenizer::new(opts.tokenizer).unwrap();
-    let rules = Rules::new(opts.rules).unwrap();
+    let tokenizer = Arc::new(Tokenizer::new(opts.tokenizer).unwrap());
+    let mut rules = Rules::new(opts.rules, tokenizer.clone()).unwrap();
+    rules.spell_mut().options_mut().variant = Some(rules.spell().variant("en_GB").unwrap());
 
     let tokens = tokenizer.pipe(&opts.text);
 
     println!("Tokens: {:#?}", tokens);
-    println!("Suggestions: {:#?}", rules.suggest(&opts.text, &tokenizer));
+    println!("Suggestions: {:#?}", rules.suggest(&opts.text));
 }

nlprule/src/bin/test.rs

@@ -1,3 +1,5 @@
+use std::sync::Arc;
+
 use clap::Clap;
 use nlprule::{rules::Rules, tokenizer::Tokenizer};
 
@@ -19,8 +21,8 @@ fn main() {
     env_logger::init();
     let opts = Opts::parse();
 
-    let tokenizer = Tokenizer::new(opts.tokenizer).unwrap();
-    let rules_container = Rules::new(opts.rules).unwrap();
+    let tokenizer = Arc::new(Tokenizer::new(opts.tokenizer).unwrap());
+    let rules_container = Rules::new(opts.rules, tokenizer.clone()).unwrap();
     let rules = rules_container.rules();
 
     println!("Runnable rules: {}", rules.len());