Merge 816ea00 into 37c7a29

source/NVDAObjects/UIA/winConsoleUIA.py

@@ -1,4 +1,3 @@
-# NVDAObjects/UIA/winConsoleUIA.py
 # A part of NonVisual Desktop Access (NVDA)
 # This file is covered by the GNU General Public License.
 # See the file COPYING for more details.
@@ -366,13 +365,6 @@ def _get_TextInfo(self):
 		movement."""
 		return consoleUIATextInfo if self.is21H1Plus else consoleUIATextInfoPre21H1
 
-	def _getTextLines(self):
-		# This override of _getTextLines takes advantage of the fact that
-		# the console text contains linefeeds for every line
-		# Thus a simple string splitlines is much faster than splitting by unit line.
-		ti = self.makeTextInfo(textInfos.POSITION_ALL)
-		text = ti.text or ""
-		return text.splitlines()
 
 def findExtraOverlayClasses(obj, clsList):
 	if obj.UIAElement.cachedAutomationId == "Text Area":

source/NVDAObjects/behaviors.py

@@ -1,5 +1,4 @@
 # -*- coding: UTF-8 -*-
-# NVDAObjects/behaviors.py
 # A part of NonVisual Desktop Access (NVDA)
 # This file is covered by the GNU General Public License.
 # See the file COPYING for more details.
@@ -10,9 +9,11 @@
 """
 
 import os
+import sys
 import time
 import threading
-import difflib
+import struct
+import subprocess
 import tones
 import queueHandler
 import eventHandler
@@ -29,6 +30,7 @@
 import ui
 import braille
 import nvwave
+from typing import List
 
 class ProgressBar(NVDAObject):
 
@@ -219,6 +221,9 @@ class LiveText(NVDAObject):
 	"""
 	#: The time to wait before fetching text after a change event.
 	STABILIZE_DELAY = 0
+	#: Whether this object supports Diff-Match-Patch character diffing.
+	#: Set to False to use line diffing.
+	_supportsDMP = True
 	# If the text is live, this is definitely content.
 	presentationType = NVDAObject.presType_content
 
@@ -228,6 +233,7 @@ def initOverlayClass(self):
 		self._event = threading.Event()
 		self._monitorThread = None
 		self._keepMonitoring = False
+		self._dmp = None
 
 	def startMonitoring(self):
 		"""Start monitoring for new text.
@@ -263,15 +269,19 @@ def event_textChange(self):
 		"""
 		self._event.set()
 
-	def _getTextLines(self):
-		"""Retrieve the text of this object in lines.
+	def _get_shouldUseDMP(self):
+		return self._supportsDMP and config.conf["terminals"]["useDMPWhenAvailable"]
+
+	def _getText(self) -> str:
+		"""Retrieve the text of this object.
 		This will be used to determine the new text to speak.
 		The base implementation uses the L{TextInfo}.
 		However, subclasses should override this if there is a better way to retrieve the text.
-		@return: The current lines of text.
-		@rtype: list of str
 		"""
-		return list(self.makeTextInfo(textInfos.POSITION_ALL).getTextInChunks(textInfos.UNIT_LINE))
+		if hasattr(self, "_getTextLines"):
+			log.warning("LiveText._getTextLines is deprecated, please override _getText instead.")
+			return '\n'.join(self._getTextLines())
+		return self.makeTextInfo(textInfos.POSITION_ALL).text
 
 	def _reportNewLines(self, lines):
 		"""
@@ -287,12 +297,35 @@ def _reportNewText(self, line):
 		"""
 		speech.speakText(line)
 
+	def _initializeDMP(self):
+		if hasattr(sys, "frozen"):
+			dmp_path = ("nvda_dmp.exe",)
+		else:
+			dmp_path = (sys.executable, "nvda_dmp.py")
+		self._dmp = subprocess.Popen(
+			dmp_path,
+			creationflags=subprocess.CREATE_NO_WINDOW,
+			bufsize=0,
+			stdin=subprocess.PIPE,
+			stdout=subprocess.PIPE
+		)
+
+	def _terminateDMP(self):
+		self._dmp.stdin.write(struct.pack("=II", 0, 0))
+		self._dmp = None
+
 	def _monitor(self):
+		if self.shouldUseDMP:
+			try:
+				self._initializeDMP()
+			except Exception:
+				log.exception("Error initializing DMP, falling back to difflib")
+				self._supportsDmp = False
 		try:
-			oldLines = self._getTextLines()
+			oldText = self._getText()
 		except:
-			log.exception("Error getting initial lines")
-			oldLines = []
+			log.exception("Error getting initial text")
+			oldText = ""
 
 		while self._keepMonitoring:
 			self._event.wait()
@@ -307,25 +340,65 @@ def _monitor(self):
 			self._event.clear()
 
 			try:
-				newLines = self._getTextLines()
+				newText = self._getText()
 				if config.conf["presentation"]["reportDynamicContentChanges"]:
-					outLines = self._calculateNewText(newLines, oldLines)
+					outLines = self._calculateNewText(newText, oldText)
 					if len(outLines) == 1 and len(outLines[0].strip()) == 1:
 						# This is only a single character,
 						# which probably means it is just a typed character,
 						# so ignore it.
 						del outLines[0]
 					if outLines:
 						queueHandler.queueFunction(queueHandler.eventQueue, self._reportNewLines, outLines)
-				oldLines = newLines
+				oldText = newText
 			except:
-				log.exception("Error getting lines or calculating new text")
+				log.exception("Error getting or calculating new text")
+
+		if self.shouldUseDMP:
+			try:
+				self._terminateDMP()
+			except Exception:
+				log.exception("Error stopping DMP")
 
-	def _calculateNewText(self, newLines, oldLines):
+	def _calculateNewText_dmp(self, newText: str, oldText: str) -> List[str]:
+		try:
+			if not newText and not oldText:
+				# Return an empty list here to avoid exiting
+				# nvda_dmp uses two zero-length texts as a sentinal value
+				return []
+			old = oldText.encode("utf-8")
+			new = newText.encode("utf-8")
+			tl = struct.pack("=II", len(old), len(new))
+			self._dmp.stdin.write(tl)
+			self._dmp.stdin.write(old)
+			self._dmp.stdin.write(new)
+			buf = b""
+			sizeb = b""
+			SIZELEN = 4
+			while len(sizeb) < SIZELEN:
+				sizeb = self._dmp.stdout.read(SIZELEN - len(sizeb))
+				if sizeb is None:
+					sizeb = b""
+			(size,) = struct.unpack("=I", sizeb)
+			while len(buf) < size:
+				buf += self._dmp.stdout.read(size - len(buf))
+			return [
+				line
+				for line in buf.decode("utf-8").splitlines()
+				if line and not line.isspace()
+			]
+		except Exception:
+			log.exception("Exception in DMP, falling back to difflib")
+			self._supportsDMP = False
+			return self._calculateNewText_difflib(newText, oldText)
+
+	def _calculateNewText_difflib(self, newLines: List[str], oldLines: List[str]) -> List[str]:
 		outLines = []
 
 		prevLine = None
-		for line in difflib.ndiff(oldLines, newLines):
+		from difflib import ndiff
+
+		for line in ndiff(oldLines, newLines):
 			if line[0] == "?":
 				# We're never interested in these.
 				continue
@@ -373,6 +446,16 @@ def _calculateNewText(self, newLines, oldLines):
 
 		return outLines
 
+	def _calculateNewText(self, newText: str, oldText: str) -> List[str]:
+		return (
+			self._calculateNewText_dmp(newText, oldText)
+			if self.shouldUseDMP
+			else self._calculateNewText_difflib(
+				newText.splitlines(), oldText.splitlines()
+			)
+		)
+
+
 class Terminal(LiveText, EditableText):
 	"""An object which both accepts text input and outputs text which should be reported automatically.
 	This is an L{EditableText} object,

source/NVDAObjects/window/winConsole.py

@@ -1,8 +1,7 @@
-#NVDAObjects/WinConsole.py
-#A part of NonVisual Desktop Access (NVDA)
-#This file is covered by the GNU General Public License.
-#See the file COPYING for more details.
-#Copyright (C) 2007-2019 NV Access Limited, Bill Dengler
+# A part of NonVisual Desktop Access (NVDA)
+# This file is covered by the GNU General Public License.
+# See the file COPYING for more details.
+# Copyright (C) 2007-2020 NV Access Limited, Bill Dengler
 
 import winConsoleHandler
 from . import Window
@@ -21,10 +20,14 @@ class WinConsole(Terminal, EditableTextWithoutAutoSelectDetection, Window):
 	STABILIZE_DELAY = 0.03
 
 	def initOverlayClass(self):
-		# Legacy consoles take quite a while to send textChange events.
-		# This significantly impacts typing performance, so don't queue chars.
 		if isinstance(self, KeyboardHandlerBasedTypedCharSupport):
+			# Legacy consoles take quite a while to send textChange events.
+			# This significantly impacts typing performance, so don't queue chars.
 			self._supportsTextChange = False
+		else:
+			# Use line diffing to report changes in the middle of lines
+			# in non-enhanced legacy consoles.
+			self._supportsDMP = False
 
 	def _get_windowThreadID(self):
 		# #10113: Windows forces the thread of console windows to match the thread of the first attached process.
@@ -69,8 +72,8 @@ def event_loseFocus(self):
 	def event_nameChange(self):
 		pass
 
-	def _getTextLines(self):
-		return winConsoleHandler.getConsoleVisibleLines()
+	def _getText(self):
+		return '\n'.join(winConsoleHandler.getConsoleVisibleLines())
 
 	def script_caret_backspaceCharacter(self, gesture):
 		super(WinConsole, self).script_caret_backspaceCharacter(gesture)

source/config/configSpec.py

@@ -219,6 +219,7 @@
 [terminals]
 	speakPasswords = boolean(default=false)
 	keyboardSupportInLegacy = boolean(default=True)
+	useDMPWhenAvailable = boolean(default=True)
 
 [update]
 	autoCheck = boolean(default=true)

source/gui/settingsDialogs.py

@@ -2527,6 +2527,14 @@ def __init__(self, parent):
 		self.keyboardSupportInLegacyCheckBox.defaultValue = self._getDefaultValue(["terminals", "keyboardSupportInLegacy"])
 		self.keyboardSupportInLegacyCheckBox.Enable(winVersion.isWin10(1607))
 
+		# Translators: This is the label for a checkbox in the
+		#  Advanced settings panel.
+		label = _("Detect changes by c&haracter when available")
+		self.useDMPWhenAvailableCheckBox = terminalsGroup.addItem(wx.CheckBox(self, label=label))
+		self.useDMPWhenAvailableCheckBox.SetValue(config.conf["terminals"]["useDMPWhenAvailable"])
+		self.useDMPWhenAvailableCheckBox.defaultValue = self._getDefaultValue(["terminals", "useDMPWhenAvailable"])
+		self.useDMPWhenAvailableCheckBox.Enable(winVersion.isWin10(1607))
+
 		# Translators: This is the label for a group of advanced options in the
 		#  Advanced settings panel
 		label = _("Speech")
@@ -2644,6 +2652,7 @@ def haveConfigDefaultsBeenRestored(self):
 			and self.winConsoleSpeakPasswordsCheckBox.IsChecked() == self.winConsoleSpeakPasswordsCheckBox.defaultValue
 			and self.cancelExpiredFocusSpeechCombo.GetSelection() == self.cancelExpiredFocusSpeechCombo.defaultValue
 			and self.keyboardSupportInLegacyCheckBox.IsChecked() == self.keyboardSupportInLegacyCheckBox.defaultValue
+			and self.useDMPWhenAvailableCheckBox.IsChecked() == self.useDMPWhenAvailableCheckBox.defaultValue
 			and self.caretMoveTimeoutSpinControl.GetValue() == self.caretMoveTimeoutSpinControl.defaultValue
 			and set(self.logCategoriesList.CheckedItems) == set(self.logCategoriesList.defaultCheckedItems)
 			and True  # reduce noise in diff when the list is extended.
@@ -2657,6 +2666,7 @@ def restoreToDefaults(self):
 		self.winConsoleSpeakPasswordsCheckBox.SetValue(self.winConsoleSpeakPasswordsCheckBox.defaultValue)
 		self.cancelExpiredFocusSpeechCombo.SetSelection(self.cancelExpiredFocusSpeechCombo.defaultValue)
 		self.keyboardSupportInLegacyCheckBox.SetValue(self.keyboardSupportInLegacyCheckBox.defaultValue)
+		self.useDMPWhenAvailableCheckBox.SetValue(self.useDMPWhenAvailableCheckBox.defaultValue)
 		self.caretMoveTimeoutSpinControl.SetValue(self.caretMoveTimeoutSpinControl.defaultValue)
 		self.logCategoriesList.CheckedItems = self.logCategoriesList.defaultCheckedItems
 		self._defaultsRestored = True
@@ -2673,6 +2683,7 @@ def onSave(self):
 		config.conf["terminals"]["speakPasswords"] = self.winConsoleSpeakPasswordsCheckBox.IsChecked()
 		config.conf["featureFlag"]["cancelExpiredFocusSpeech"] = self.cancelExpiredFocusSpeechCombo.GetSelection()
 		config.conf["terminals"]["keyboardSupportInLegacy"]=self.keyboardSupportInLegacyCheckBox.IsChecked()
+		config.conf["terminals"]["useDMPWhenAvailable"] = self.useDMPWhenAvailableCheckBox.IsChecked()
 		config.conf["editableText"]["caretMoveTimeoutMs"]=self.caretMoveTimeoutSpinControl.GetValue()
 		for index,key in enumerate(self.logCategories):
 			config.conf['debugLog'][key]=self.logCategoriesList.IsChecked(index)

source/nvda_dmp.py

@@ -0,0 +1,21 @@
+import struct
+import sys
+
+from diff_match_patch import diff
+
+
+if __name__ == "__main__":
+	while True:
+		oldLen, newLen = struct.unpack("=II", sys.stdin.buffer.read(8))
+		if not oldLen and not newLen:
+			break
+		oldText = sys.stdin.buffer.read(oldLen).decode("utf-8")
+		newText = sys.stdin.buffer.read(newLen).decode("utf-8")
+		res = ""
+		for op, text in diff(oldText, newText, counts_only=False):
+			if (op == "=" and text.isspace()) or op == "+":
+				res += text
+		sys.stdout.buffer.write(struct.pack("=I", len(res)))
+		sys.stdout.buffer.write(res.encode("utf-8"))
+		sys.stdin.flush()
+		sys.stdout.flush()

source/setup.py

@@ -5,6 +5,7 @@
 #This file is covered by the GNU General Public License.
 #See the file COPYING for more details.
 
+from versionInfo import formatBuildVersionString, name, version, publisher
 import os
 import copy
 import gettext
@@ -187,6 +188,20 @@ def getRecursiveDataFiles(dest,source,excludes=()):
 			"company_name": publisher,
 		},
 	],
+	console=[
+		{
+			"script": "nvda_dmp.py",
+			"uiAccess": False,
+			"icon_resources": [(1, "images/nvda.ico")],
+			"other_resources": [],  # Populated at runtime
+			"version":formatBuildVersionString(),
+			"description": "NVDA Diff-match-patch proxy",
+			"product_name": name,
+			"product_version": version,
+			"copyright": copyright,
+			"company_name": publisher,
+		},
+	],
 	options = {"py2exe": {
 		"bundle_files": 3,
 		"excludes": ["tkinter",

user_docs/en/userGuide.t2t

@@ -1835,10 +1835,16 @@ This setting controls whether characters are spoken by [speak typed characters #
 ==== Use the new typed character support in Windows Console when available ====[AdvancedSettingsKeyboardSupportInLegacy]
 This option enables an alternative method for detecting typed characters in Windows command consoles.
 While it improves performance and prevents some console output from being spelled out, it may be incompatible with some terminal programs.
-This feature is available and enabled by default on Windows 10 versions 1607 and later when UI Automation is unavailable or disabled.
+This feature is available and enabled by default on Windows 10 versions 1607and later when UI Automation is unavailable or disabled.
 Warning: with this option enabled, typed characters that do not appear onscreen, such as passwords, will not be suppressed.
 In untrusted environments, you may temporarily disable [speak typed characters #KeyboardSettingsSpeakTypedCharacters] and [speak typed words #KeyboardSettingsSpeakTypedWords] when entering passwords.
 
+==== Detect changes by character when available ====[AdvancedSettingsUseDMPWhenAvailable]
+This option enables an alternative method for detecting output changes in terminals.
+It may improve performance when large volumes of text are written to the console and allow more accurate reporting of changes made in the middle of lines.
+However, it may be incompatible with some applications.
+This feature is available and enabled by default on Windows 10 versions 1607 and later.
+
 ==== Attempt to cancel speech for expired focus events ====[CancelExpiredFocusSpeech]
 This option enables behaviour which attempts to cancel speech for expired focus events.
 In particular moving quickly through messages in Gmail with Chrome can cause NVDA to speak outdated information.