Find line breaks according to the Unicode line breaking algorithm

This adds a new optional dependency on the unicode-linebreak crate,
which implements the line breaking algorithm from [Unicode Standard
Annex #14](https://www.unicode.org/reports/tr14/).

The new dependency is enabled by default since these line breaks are
more correct than what you get by splitting on whitespace.

This should help address #220 and #80, though I’m no expert on
non-Western languages. More feedback from the community would be
needed here.

Cargo.toml

@@ -21,12 +21,13 @@ harness = false
 path = "benches/linear.rs"
 
 [features]
-default = ["unicode-width", "smawk"]
+default = ["unicode-linebreak", "unicode-width", "smawk"]
 
 [dependencies]
 hyphenation = { version = "0.8", optional = true, features = ["embed_en-us"] }
 smawk = { version = "0.3", optional = true }
 terminal_size = { version = "0.1", optional = true }
+unicode-linebreak = { version = "0.1", optional = true }
 unicode-width = { version= "0.1", optional = true }
 
 [dev-dependencies]

benches/linear.rs

@@ -26,10 +26,25 @@ pub fn benchmark(c: &mut Criterion) {
 
         #[cfg(feature = "smawk")]
         {
+            #[cfg(feature = "unicode-linebreak")]
+            {
+                let options = textwrap::Options::new(LINE_LENGTH)
+                    .wrap_algorithm(textwrap::core::WrapAlgorithm::OptimalFit)
+                    .line_breaks(textwrap::UnicodeLineBreaks);
+                group.bench_with_input(
+                    BenchmarkId::new("fill_optimal_fit_unicode", length),
+                    &text,
+                    |b, text| {
+                        b.iter(|| textwrap::fill(text, &options));
+                    },
+                );
+            }
+
             let options = textwrap::Options::new(LINE_LENGTH)
-                .wrap_algorithm(textwrap::core::WrapAlgorithm::OptimalFit);
+                .wrap_algorithm(textwrap::core::WrapAlgorithm::OptimalFit)
+                .line_breaks(textwrap::AsciiSpaceLineBreaks);
             group.bench_with_input(
-                BenchmarkId::new("fill_optimal_fit", length),
+                BenchmarkId::new("fill_optimal_fit_ascii_space", length),
                 &text,
                 |b, text| {
                     b.iter(|| textwrap::fill(text, &options));

examples/interactive.rs

@@ -56,7 +56,7 @@ mod unix_only {
 
     fn draw_text<'a>(
         text: &str,
-        options: &Options<'a>,
+        options: &Options<'a, textwrap::DefaultLineBreakAlgorithm>,
         splitter_label: &str,
         stdout: &mut RawTerminal<io::Stdout>,
     ) -> Result<(), io::Error> {
@@ -256,7 +256,7 @@ mod unix_only {
         }
 
         let mut label = labels.pop().unwrap();
-        let mut options: Options = Options::new(35).splitter(Box::new(HyphenSplitter));
+        let mut options: Options<_> = Options::new(35).splitter(Box::new(HyphenSplitter));
         options.break_words = false;
         options.splitter = splitters.pop().unwrap();
 

examples/layout.rs

@@ -6,7 +6,7 @@ fn main() {
                    Zero-cost abstractions.";
     let mut prev_lines = vec![];
 
-    let mut options: Options = Options::new(0).splitter(Box::new(HyphenSplitter));
+    let mut options = Options::new(0).splitter(Box::new(HyphenSplitter));
     #[cfg(feature = "hyphenation")]
     {
         use hyphenation::Load;

examples/multi-layouts.rs

@@ -7,10 +7,16 @@ use hyphenation::{Language, Load, Standard};
 mod library {
     use textwrap::{Options, WordSplitter};
 
+    /// Default word finder.
+    #[cfg(feature = "unicode-linebreak")]
+    pub type DefaultLineBreakAlgorithm = textwrap::UnicodeLineBreaks;
+    #[cfg(not(feature = "unicode-linebreak"))]
+    pub type DefaultLineBreakAlgorithm = textwrap::AsciiSpaceLineBreaks;
+
     #[derive(Debug, Default)]
     pub struct Layout<'a> {
         // Use trait-objects which can be easily converted from any concrete `Options`
-        styles: Vec<Box<Options<'a, dyn WordSplitter + 'a>>>,
+        styles: Vec<Box<Options<'a, DefaultLineBreakAlgorithm, dyn WordSplitter + 'a>>>,
     }
 
     impl<'a> Layout<'a> {
@@ -19,7 +25,10 @@ mod library {
         }
 
         // Similar signature like `wrap` has, so it takes (nearly) everything that `warp` takes.
-        pub fn add<S: WordSplitter + 'a, T: Into<Options<'a, S>>>(&mut self, option: T) {
+        pub fn add<S: WordSplitter + 'a, T: Into<Options<'a, DefaultLineBreakAlgorithm, S>>>(
+            &mut self,
+            option: T,
+        ) {
             self.styles.push(Box::new(option.into()));
         }
 
@@ -35,7 +44,7 @@ mod library {
 
                 // Just use the textwrap functions as usual.
                 // However, we have to first coerce it into a trait-object
-                let dyn_opt: &Options<'a, dyn WordSplitter> = opt;
+                let dyn_opt: &Options<'a, DefaultLineBreakAlgorithm, dyn WordSplitter> = opt;
                 println!("{}", textwrap::fill(text, dyn_opt));
             }
         }
@@ -65,7 +74,7 @@ pub fn main() {
     layout.add(opt.clone()); // notice, here we pass opt by-value instead of by-reference
 
     // We can use boxed splitters too (however, we have to coerce the Options)
-    let opt: Options = opt.splitter(Box::new(NoHyphenation));
+    let opt: Options<_> = opt.splitter(Box::new(NoHyphenation));
     layout.add(opt);
 
     // We can also pass-in references, however, those need to outlive the local

examples/wasm/src/lib.rs

@@ -1,7 +1,7 @@
 use wasm_bindgen::prelude::*;
 use wasm_bindgen::JsCast;
 
-use textwrap::core;
+use textwrap::{core, LineBreakAlgorithm};
 
 #[wasm_bindgen]
 extern "C" {
@@ -160,7 +160,7 @@ pub fn draw_wrapped_text(
 
     let mut lineno = 0;
     for line in text.split('\n') {
-        let words = core::find_words(line);
+        let words = options.line_breaks.break_line(line);
         let split_words = core::split_words(words, &options);
 
         let canvas_words = split_words