Update range subtraction algorithm in dialog rendering

lib/reline/line_editor.rb

@@ -659,6 +659,45 @@ def add_dialog_proc(name, p, context = nil)
     render_dialog_changes(changes, cursor_column)
   end
 
+  private def range_subtract(base_ranges, subtract_ranges)
+    points = []
+    base_ranges.each do |range|
+      points << [range.begin, 1, 1]
+      points << [range.end, -1, 1]
+    end
+    subtract_ranges.each do |range|
+      points << [range.begin, 1, -1]
+      points << [range.end, -1, -1]
+    end
+    ranges = []
+    base_count = 0
+    subtract_count = 0
+    open = false
+    points.sort.each do |point, diff, mode|
+      if mode == 1
+        base_count += diff
+      else
+        subtract_count += diff
+      end
+      next if open == (base_count > 0 && subtract_count == 0)
+      open = !open
+      if open
+        if ranges.last&.end === point
+          ranges[-1] = (ranges.last.begin...)
+        else
+          ranges << (point...)
+        end
+      else
+        if ranges.last.begin === point
+          ranges.pop
+        else
+          ranges[-1] = ranges.last.begin...point
+        end
+      end
+    end
+    ranges
+  end
+
   private def dialog_range(dialog, dialog_y)
     x_range = dialog.column...dialog.column + dialog.width
     y_range = dialog_y + dialog.vertical_offset...dialog_y + dialog.vertical_offset + dialog.contents.size
@@ -687,24 +726,8 @@ def add_dialog_proc(name, p, context = nil)
     return if old_dialog_ranges.empty? && new_dialog_ranges.empty?
     ranges_to_restore = {}
     old_dialog_ranges.each do |y, old_x_ranges|
-      new_dialog_ranges[y]&.each do |new_x_range|
-        ranges = []
-        old_x_ranges.each do |old_x_range|
-          next if new_x_range.cover? old_x_range
-          if new_x_range.end <= old_x_range.begin || old_x_range.end <= new_x_range.begin
-            ranges << old_x_range
-          elsif new_x_range.begin <= old_x_range.begin
-            ranges << (new_x_range.end...old_x_range.end)
-          elsif old_x_range.end <= new_x_range.end
-            ranges << (old_x_range.begin...new_x_range.begin)
-          else
-            ranges << (old_x_range.begin...new_x_range.begin)
-            ranges << (new_x_range.end...old_x_range.end)
-          end
-        end
-        old_x_ranges = ranges
-      end
-      (ranges_to_restore[y] ||= []).push(*old_x_ranges) unless old_x_ranges.empty?
+      ranges = range_subtract(old_x_ranges, new_dialog_ranges[y] || [])
+      ranges_to_restore[y] = ranges if ranges.any?
     end
 
     if ranges_to_restore.any?

test/reline/test_line_editor.rb

@@ -0,0 +1,13 @@
+require_relative 'helper'
+require 'reline/line_editor'
+
+class Reline::LineEditor::Test < Reline::TestCase
+  def test_range_subtract
+    dummy_config = nil
+    editor = Reline::LineEditor.new(dummy_config, 'ascii-8bit')
+    base_ranges = [3...5, 4...10, 6...8, 12...15, 15...20]
+    subtract_ranges = [5...7, 8...9, 11...13, 17...18, 18...19]
+    expected_result = [3...5, 7...8, 9...10, 13...17, 19...20]
+    assert_equal expected_result, editor.send(:range_subtract, base_ranges, subtract_ranges)
+  end
+end