Handle multibyte character input by KeyStroke

lib/reline.rb

@@ -307,6 +307,7 @@ def readline(prompt = '', add_hist = false)
       otio = io_gate.prep
 
       may_req_ambiguous_char_width
+      key_stroke.encoding = encoding
       line_editor.reset(prompt)
       if multiline
         line_editor.multiline_on
@@ -485,7 +486,7 @@ def self.encoding_system_needs
   def self.core
     @core ||= Core.new { |core|
       core.config = Reline::Config.new
-      core.key_stroke = Reline::KeyStroke.new(core.config)
+      core.key_stroke = Reline::KeyStroke.new(core.config, core.encoding)
       core.line_editor = Reline::LineEditor.new(core.config)
 
       core.basic_word_break_characters = " \t\n`><=;|&{("

lib/reline/key_stroke.rb

@@ -3,8 +3,11 @@ class Reline::KeyStroke
   CSI_PARAMETER_BYTES_RANGE = 0x30..0x3f
   CSI_INTERMEDIATE_BYTES_RANGE = (0x20..0x2f)
 
-  def initialize(config)
+  attr_accessor :encoding
+
+  def initialize(config, encoding)
     @config = config
+    @encoding = encoding
   end
 
   # Input exactly matches to a key sequence
@@ -21,7 +24,7 @@ def match_status(input)
     matched = key_mapping.get(input)
 
     # FIXME: Workaround for single byte. remove this after MAPPING is merged into KeyActor.
-    matched ||= input.size == 1
+    matched ||= input.size == 1 && input[0] < 0x80
     matching ||= input == [ESC_BYTE]
 
     if matching && matched
@@ -32,10 +35,14 @@ def match_status(input)
       MATCHED
     elsif input[0] == ESC_BYTE
       match_unknown_escape_sequence(input, vi_mode: @config.editing_mode_is?(:vi_insert, :vi_command))
-    elsif input.size == 1
-      MATCHED
     else
-      UNMATCHED
+      s = input.pack('c*').force_encoding(@encoding)
+      if s.valid_encoding?
+        s.size == 1 ? MATCHED : UNMATCHED
+      else
+        # Invalid string is MATCHING (part of valid string) or MATCHED (invalid bytes to be ignored)
+        MATCHING_MATCHED
+      end
     end
   end
 
@@ -45,6 +52,7 @@ def expand(input)
       bytes = input.take(i)
       status = match_status(bytes)
       matched_bytes = bytes if status == MATCHED || status == MATCHING_MATCHED
+      break if status == MATCHED || status == UNMATCHED
     end
     return [[], []] unless matched_bytes
 
@@ -53,12 +61,15 @@ def expand(input)
       keys = func.map { |c| Reline::Key.new(c, c, false) }
     elsif func
       keys = [Reline::Key.new(func, func, false)]
-    elsif matched_bytes.size == 1
-      keys = [Reline::Key.new(matched_bytes.first, matched_bytes.first, false)]
     elsif matched_bytes.size == 2 && matched_bytes[0] == ESC_BYTE
       keys = [Reline::Key.new(matched_bytes[1], matched_bytes[1] | 0b10000000, true)]
     else
-      keys = []
+      s = matched_bytes.pack('c*').force_encoding(@encoding)
+      if s.valid_encoding? && s.size == 1
+        keys = [Reline::Key.new(s.ord, s.ord, false)]
+      else
+        keys = []
+      end
     end
 
     [keys, input.drop(matched_bytes.size)]

lib/reline/line_editor.rb

@@ -265,7 +265,6 @@ def reset_line
     @line_index = 0
     @cache.clear
     @line_backup_in_history = nil
-    @multibyte_buffer = String.new(encoding: 'ASCII-8BIT')
   end
 
   def multiline_on
@@ -1036,20 +1035,11 @@ def wrap_method_call(method_symbol, method_obj, key, with_operator = false)
   end
 
   private def normal_char(key)
-    @multibyte_buffer << key.combined_char
-    if @multibyte_buffer.size > 1
-      if @multibyte_buffer.dup.force_encoding(encoding).valid_encoding?
-        process_key(@multibyte_buffer.dup.force_encoding(encoding), nil)
-        @multibyte_buffer.clear
-      else
-        # invalid
-        return
-      end
-    else # single byte
-      return if key.char >= 128 # maybe, first byte of multi byte
+    if key.char < 0x80
       method_symbol = @config.editing_mode.get_method(key.combined_char)
       process_key(key.combined_char, method_symbol)
-      @multibyte_buffer.clear
+    else
+      process_key(key.char.chr(encoding), nil)
     end
     if @config.editing_mode_is?(:vi_command) and @byte_pointer > 0 and @byte_pointer == current_line.bytesize
       byte_size = Reline::Unicode.get_prev_mbchar_size(@buffer_of_lines[@line_index], @byte_pointer)
@@ -1531,7 +1521,6 @@ def finish
 
   private def generate_searcher(search_key)
     search_word = String.new(encoding: encoding)
-    multibyte_buf = String.new(encoding: 'ASCII-8BIT')
     hit_pointer = nil
     lambda do |key|
       search_again = false
@@ -1546,11 +1535,7 @@ def finish
         search_again = true if search_key == key
         search_key = key
       else
-        multibyte_buf << key
-        if multibyte_buf.dup.force_encoding(encoding).valid_encoding?
-          search_word << multibyte_buf.dup.force_encoding(encoding)
-          multibyte_buf.clear
-        end
+        search_word << key
       end
       hit = nil
       if not search_word.empty? and @line_backup_in_history&.include?(search_word)

test/reline/helper.rb

@@ -121,17 +121,15 @@ def input_keys(input, convert = true)
           @line_editor.input_key(Reline::Key.new(byte, byte, false))
         end
       else
-        c.bytes.each do |b|
-          @line_editor.input_key(Reline::Key.new(b, b, false))
-        end
+        @line_editor.input_key(Reline::Key.new(c.ord, c.ord, false))
       end
     end
   end
 
   def input_raw_keys(input, convert = true)
     input = convert_str(input) if convert
-    input.bytes.each do |b|
-      @line_editor.input_key(Reline::Key.new(b, b, false))
+    input.chars.each do |c|
+      @line_editor.input_key(Reline::Key.new(c.ord, c.ord, false))
     end
   end
 

test/reline/test_key_stroke.rb

@@ -13,6 +13,10 @@ def to_keys
     end
   }
 
+  def encoding
+    Reline.core.encoding
+  end
+
   def test_match_status
     config = Reline::Config.new
     {
@@ -23,7 +27,7 @@ def test_match_status
     }.each_pair do |key, func|
       config.add_default_key_binding(key.bytes, func.bytes)
     end
-    stroke = Reline::KeyStroke.new(config)
+    stroke = Reline::KeyStroke.new(config, encoding)
     assert_equal(Reline::KeyStroke::MATCHING_MATCHED, stroke.match_status("a".bytes))
     assert_equal(Reline::KeyStroke::MATCHING_MATCHED, stroke.match_status("ab".bytes))
     assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status("abc".bytes))
@@ -37,7 +41,7 @@ def test_match_status
   def test_match_unknown
     config = Reline::Config.new
     config.add_default_key_binding("\e[9abc".bytes, 'x')
-    stroke = Reline::KeyStroke.new(config)
+    stroke = Reline::KeyStroke.new(config, encoding)
     sequences = [
       "\e[9abc",
       "\e[9d",
@@ -66,7 +70,7 @@ def test_expand
     }.each_pair do |key, func|
       config.add_default_key_binding(key.bytes, func.bytes)
     end
-    stroke = Reline::KeyStroke.new(config)
+    stroke = Reline::KeyStroke.new(config, encoding)
     assert_equal(['123'.bytes.map { |c| Reline::Key.new(c, c, false) }, 'de'.bytes], stroke.expand('abcde'.bytes))
     assert_equal(['456'.bytes.map { |c| Reline::Key.new(c, c, false) }, 'de'.bytes], stroke.expand('abde'.bytes))
     # CSI sequence
@@ -83,7 +87,7 @@ def test_oneshot_key_bindings
     }.each_pair do |key, func|
       config.add_default_key_binding(key.bytes, func.bytes)
     end
-    stroke = Reline::KeyStroke.new(config)
+    stroke = Reline::KeyStroke.new(config, encoding)
     assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status('zzz'.bytes))
     assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status('abc'.bytes))
   end
@@ -96,10 +100,27 @@ def test_with_reline_key
     }.each_pair do |key, func|
       config.add_oneshot_key_binding(key, func.bytes)
     end
-    stroke = Reline::KeyStroke.new(config)
+    stroke = Reline::KeyStroke.new(config, encoding)
     assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status('da'.bytes))
     assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status("\eda".bytes))
     assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status([32, 195, 164]))
     assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status([195, 164]))
   end
+
+  def test_multibyte_matching
+    config = Reline::Config.new
+    stroke = Reline::KeyStroke.new(config, encoding)
+    char = 'あ'.encode(encoding)
+    key = Reline::Key.new(char.ord, char.ord, false)
+    bytes = char.bytes
+    assert_equal(Reline::KeyStroke::MATCHED, stroke.match_status(bytes))
+    assert_equal([[key], []], stroke.expand(bytes))
+    assert_equal(Reline::KeyStroke::UNMATCHED, stroke.match_status(bytes * 2))
+    assert_equal([[key], bytes], stroke.expand(bytes * 2))
+    (1...bytes.size).each do |i|
+      partial_bytes = bytes.take(i)
+      assert_equal(Reline::KeyStroke::MATCHING_MATCHED, stroke.match_status(partial_bytes))
+      assert_equal([[], []], stroke.expand(partial_bytes))
+    end
+  end
 end

test/reline/test_line_editor.rb

@@ -8,7 +8,7 @@ class CompletionBlockTest < Reline::TestCase
     def setup
       @original_quote_characters = Reline.completer_quote_characters
       @original_word_break_characters = Reline.completer_word_break_characters
-      @line_editor = Reline::LineEditor.new(nil, Encoding::UTF_8)
+      @line_editor = Reline::LineEditor.new(nil)
     end
 
     def retrieve_completion_block(lines, line_index, byte_pointer)