Improve keyboard support for some terminal programs (PR #9915)

Previously:
NVDA failed to announce typed characters and/or words in Mintty, and spells output close to the caret in 
legacy Windows consoles.

This commit factors out much of the code for handling typed characters in UIA consoles into a new
`NVDAObjects.behaviors.TerminalWithoutTypedCharDetection class`. The class is now used in 
Mintty (PuTTY, Git Bash) and legacy Windows consoles on Windows 10 version 1607 and later.

In legacy Windows consoles, the old keyboard handling code is disabled when the class is in use, and the
new support can be disabled in the advanced preferences in case it is incompatible with some programs 
or if suppression of passwords is critical.

Since legacy Windows consoles fire textChange rather slowly, this commit prefers faster responsiveness at 
the cost of offscreen characters (such as passwords) always being reported. Users may disable "speak typed 
characters" and/or "speak typed words" (using the existing scripts) when entering passwords to suppress 
this output.

On Windows 10 version 1607 with the new keyboard support enabled, spurious characters are reported 
when the dead key (if available) is pressed.

Fixes #513 
Fixes #1348
Related to #9614

source/NVDAObjects/IAccessible/__init__.py

@@ -542,6 +542,12 @@ def findOverlayClasses(self,clsList):
 		elif windowClassName.startswith("Chrome_"):
 			from . import chromium
 			chromium.findExtraOverlayClasses(self, clsList)
+		if (
+			windowClassName == "ConsoleWindowClass"
+			and role == oleacc.ROLE_SYSTEM_CLIENT
+		):
+			from . import winConsole
+			winConsole.findExtraOverlayClasses(self,clsList)
 
 
 		#Support for Windowless richEdit
@@ -2023,5 +2029,4 @@ def event_alert(self):
 	("NUIDialog",oleacc.ROLE_SYSTEM_CLIENT):"NUIDialogClient",
 	("_WwB",oleacc.ROLE_SYSTEM_CLIENT):"winword.ProtectedDocumentPane",
     ("MsoCommandBar",oleacc.ROLE_SYSTEM_LISTITEM):"msOffice.CommandBarListItem",
-	("ConsoleWindowClass",oleacc.ROLE_SYSTEM_CLIENT):"winConsole.WinConsole",
 }

source/NVDAObjects/IAccessible/winConsole.py

@@ -4,9 +4,19 @@
 #See the file COPYING for more details.
 #Copyright (C) 2007-2019 NV Access Limited, Bill Dengler
 
+import config
+
+from winVersion import isWin10
+
 from . import IAccessible
-from ..window.winConsole import WinConsole
+from ..window import winConsole
 
-class WinConsole(WinConsole, IAccessible):
+class WinConsole(winConsole.WinConsole, IAccessible):
 	"The legacy console implementation for situations where UIA isn't supported."
-	pass
+	pass
+
+def findExtraOverlayClasses(obj, clsList):
+	if isWin10(1607) and config.conf['terminals']['keyboardSupportInLegacy']:
+		from NVDAObjects.behaviors import KeyboardHandlerBasedTypedCharSupport
+		clsList.append(KeyboardHandlerBasedTypedCharSupport)
+	clsList.append(WinConsole)

source/NVDAObjects/UIA/winConsoleUIA.py

@@ -4,18 +4,14 @@
 # See the file COPYING for more details.
 # Copyright (C) 2019 Bill Dengler
 
-import config
 import ctypes
 import NVDAHelper
-import speech
-import time
 import textInfos
 import UIAHandler
 
-from scriptHandler import script
 from winVersion import isWin10
 from . import UIATextInfo
-from ..behaviors import Terminal
+from ..behaviors import KeyboardHandlerBasedTypedCharSupport
 from ..window import Window
 
 
@@ -223,70 +219,16 @@ def _get_focusRedirect(self):
 		return None
 
 
-class WinConsoleUIA(Terminal):
+class WinConsoleUIA(KeyboardHandlerBasedTypedCharSupport):
 	#: Disable the name as it won't be localized
 	name = ""
 	#: Only process text changes every 30 ms, in case the console is getting
 	#: a lot of text.
 	STABILIZE_DELAY = 0.03
 	_TextInfo = consoleUIATextInfo
-	#: A queue of typed characters, to be dispatched on C{textChange}.
-	#: This queue allows NVDA to suppress typed passwords when needed.
-	_queuedChars = []
-	#: Whether the console got new text lines in its last update.
-	#: Used to determine if typed character/word buffers should be flushed.
-	_hasNewLines = False
 	#: the caret in consoles can take a while to move on Windows 10 1903 and later.
 	_caretMovementTimeoutMultiplier = 1.5
 
-	def _reportNewText(self, line):
-		# Additional typed character filtering beyond that in LiveText
-		if len(line.strip()) < max(len(speech.curWordChars) + 1, 3):
-			return
-		if self._hasNewLines:
-			# Clear the queued characters buffer for new text lines.
-			self._queuedChars = []
-		super(WinConsoleUIA, self)._reportNewText(line)
-
-	def event_typedCharacter(self, ch):
-		if ch == '\t':
-			# Clear the typed word buffer for tab completion.
-			speech.clearTypedWordBuffer()
-		if (
-			(
-				config.conf['keyboard']['speakTypedCharacters']
-				or config.conf['keyboard']['speakTypedWords']
-			)
-			and not config.conf['UIA']['winConsoleSpeakPasswords']
-		):
-			self._queuedChars.append(ch)
-		else:
-			super(WinConsoleUIA, self).event_typedCharacter(ch)
-
-	def event_textChange(self):
-		while self._queuedChars:
-			ch = self._queuedChars.pop(0)
-			super(WinConsoleUIA, self).event_typedCharacter(ch)
-		super(WinConsoleUIA, self).event_textChange()
-
-	@script(gestures=[
-		"kb:enter",
-		"kb:numpadEnter",
-		"kb:tab",
-		"kb:control+c",
-		"kb:control+d",
-		"kb:control+pause"
-	])
-	def script_flush_queuedChars(self, gesture):
-		"""
-		Flushes the typed word buffer and queue of typedCharacter events if present.
-		Since these gestures clear the current word/line, we should flush the
-		queue to avoid erroneously reporting these chars.
-		"""
-		gesture.send()
-		self._queuedChars = []
-		speech.clearTypedWordBuffer()
-
 	def _get_caretMovementDetectionUsesEvents(self):
 		"""Using caret events in consoles sometimes causes the last character of the
 		prompt to be read when quickly deleting text."""
@@ -298,19 +240,6 @@ def _getTextLines(self):
 		res = [ptr.GetElement(i).GetText(-1) for i in range(ptr.length)]
 		return res
 
-	def _calculateNewText(self, newLines, oldLines):
-		self._hasNewLines = (
-			self._findNonBlankIndices(newLines)
-			!= self._findNonBlankIndices(oldLines)
-		)
-		return super(WinConsoleUIA, self)._calculateNewText(newLines, oldLines)
-
-	def _findNonBlankIndices(self, lines):
-		"""
-		Given a list of strings, returns a list of indices where the strings
-		are not empty.
-		"""
-		return [index for index, line in enumerate(lines) if line]
 
 def findExtraOverlayClasses(obj, clsList):
 	if obj.UIAElement.cachedAutomationId == "Text Area":

source/NVDAObjects/behaviors.py

@@ -24,6 +24,7 @@
 import textInfos
 import editableText
 from logHandler import log
+from scriptHandler import script
 import api
 import ui
 import braille
@@ -366,6 +367,106 @@ def event_loseFocus(self):
 		super(Terminal, self).event_loseFocus()
 		self.stopMonitoring()
 
+
+class KeyboardHandlerBasedTypedCharSupport(Terminal):
+	"""A Terminal object that also provides typed character support for
+	console applications via keyboardHandler events.
+	These events are queued from NVDA's global keyboard hook.
+	Therefore, an event is fired for every single character that is being typed,
+	even when a character is not written to the console (e.g. in read only console applications).
+	This approach is an alternative to monitoring the console output for
+	characters close to the caret, or injecting in-process with NVDAHelper.
+	This class relies on the toUnicodeEx Windows function, and in particular
+	the flag to preserve keyboard state available in Windows 10 1607
+	and later."""
+	#: Whether this object quickly and reliably sends textChange events
+	#: when its contents update.
+	#: Timely and reliable textChange events are required
+	#: to support password suppression.
+	_supportsTextChange = True
+	#: A queue of typed characters, to be dispatched on C{textChange}.
+	#: This queue allows NVDA to suppress typed passwords when needed.
+	_queuedChars = []
+	#: Whether the last typed character is a tab.
+	#: If so, we should temporarily disable filtering as completions may
+	#: be short.
+	_hasTab = False
+
+	def _reportNewText(self, line):
+		# Perform typed character filtering, as typed characters are handled with events.
+		if (
+			not self._hasTab
+			and len(line.strip()) < max(len(speech.curWordChars) + 1, 3)
+		):
+			return
+		super()._reportNewText(line)
+
+	def event_typedCharacter(self, ch):
+		if ch == '\t':
+			self._hasTab = True
+			# Clear the typed word buffer for tab completion.
+			speech.clearTypedWordBuffer()
+		else:
+			self._hasTab = False
+		if (
+			(
+				config.conf['keyboard']['speakTypedCharacters']
+				or config.conf['keyboard']['speakTypedWords']
+			)
+			and not config.conf['UIA']['winConsoleSpeakPasswords']
+			and self._supportsTextChange
+		):
+			self._queuedChars.append(ch)
+		else:
+			super().event_typedCharacter(ch)
+
+	def event_textChange(self):
+		self._dispatchQueue()
+		super().event_textChange()
+
+	@script(gestures=[
+		"kb:enter",
+		"kb:numpadEnter",
+		"kb:tab",
+		"kb:control+c",
+		"kb:control+d",
+		"kb:control+pause"
+	])
+	def script_flush_queuedChars(self, gesture):
+		"""
+		Flushes the typed word buffer and queue of typedCharacter events if present.
+		Since these gestures clear the current word/line, we should flush the
+		queue to avoid erroneously reporting these chars.
+		"""
+		self._queuedChars = []
+		speech.clearTypedWordBuffer()
+		gesture.send()
+
+	def _calculateNewText(self, newLines, oldLines):
+		hasNewLines = (
+			self._findNonBlankIndices(newLines)
+			!= self._findNonBlankIndices(oldLines)
+		)
+		if hasNewLines:
+			# Clear the typed word buffer for new text lines.
+			speech.clearTypedWordBuffer()
+			self._queuedChars = []
+		return super()._calculateNewText(newLines, oldLines)
+
+	def _dispatchQueue(self):
+		"""Sends queued typedCharacter events through to NVDA."""
+		while self._queuedChars:
+			ch = self._queuedChars.pop(0)
+			super().event_typedCharacter(ch)
+
+	def _findNonBlankIndices(self, lines):
+		"""
+		Given a list of strings, returns a list of indices where the strings
+		are not empty.
+		"""
+		return [index for index, line in enumerate(lines) if line]
+
+
 class CandidateItem(NVDAObject):
 
 	def getFormattedCandidateName(self,number,candidate):

source/NVDAObjects/window/winConsole.py

@@ -2,11 +2,11 @@
 #A part of NonVisual Desktop Access (NVDA)
 #This file is covered by the GNU General Public License.
 #See the file COPYING for more details.
-#Copyright (C) 2007-2012 NV Access Limited
+#Copyright (C) 2007-2019 NV Access Limited, Bill Dengler
 
 import winConsoleHandler
 from . import Window
-from ..behaviors import Terminal, EditableTextWithoutAutoSelectDetection
+from ..behaviors import Terminal, EditableTextWithoutAutoSelectDetection, KeyboardHandlerBasedTypedCharSupport
 import api
 import core
 from scriptHandler import script
@@ -20,6 +20,12 @@ class WinConsole(Terminal, EditableTextWithoutAutoSelectDetection, Window):
 	"""
 	STABILIZE_DELAY = 0.03
 
+	def initOverlayClass(self):
+		# Legacy consoles take quite a while to send textChange events.
+		# This significantly impacts typing performance, so don't queue chars.
+		if isinstance(self, KeyboardHandlerBasedTypedCharSupport):
+			self._supportsTextChange = False
+
 	def _get_TextInfo(self):
 		consoleObject=winConsoleHandler.consoleObject
 		if consoleObject and self.windowHandle == consoleObject.windowHandle:

source/appModules/putty.py

@@ -2,16 +2,17 @@
 #A part of NonVisual Desktop Access (NVDA)
 #This file is covered by the GNU General Public License.
 #See the file COPYING for more details.
-#Copyright (C) 2010-2014 NV Access Limited
+#Copyright (C) 2010-2019 NV Access Limited, Bill Dengler
 
 """App module for PuTTY
 """
 
 import oleacc
-from NVDAObjects.behaviors import Terminal
+from NVDAObjects.behaviors import KeyboardHandlerBasedTypedCharSupport, Terminal
 from NVDAObjects.window import DisplayModelEditableText, DisplayModelLiveText
 import appModuleHandler
 from NVDAObjects.IAccessible import IAccessible
+from winVersion import isWin10
 
 class AppModule(appModuleHandler.AppModule):
 	# Allow this to be overridden for derived applications.
@@ -23,4 +24,7 @@ def chooseNVDAObjectOverlayClasses(self, obj, clsList):
 				clsList.remove(DisplayModelEditableText)
 			except ValueError:
 				pass
-			clsList[0:0] = (Terminal, DisplayModelLiveText)
+			if isWin10(1607):
+				clsList[0:0] = (KeyboardHandlerBasedTypedCharSupport, DisplayModelLiveText)
+			else:
+				clsList[0:0] = (Terminal, DisplayModelLiveText)

source/config/configSpec.py

@@ -190,6 +190,9 @@
 	winConsoleImplementation= option("auto", "legacy", "UIA", default="auto")
 	winConsoleSpeakPasswords = boolean(default=false)
 
+[terminals]
+	keyboardSupportInLegacy = boolean(default=True)
+
 [update]
 	autoCheck = boolean(default=true)
 	startupNotification = boolean(default=true)

source/gui/settingsDialogs.py

@@ -2068,6 +2068,22 @@ def __init__(self, parent):
 		self.winConsoleSpeakPasswordsCheckBox.SetValue(config.conf["UIA"]["winConsoleSpeakPasswords"])
 		self.winConsoleSpeakPasswordsCheckBox.defaultValue = self._getDefaultValue(["UIA", "winConsoleSpeakPasswords"])
 
+		# Translators: This is the label for a group of advanced options in the
+		#  Advanced settings panel
+		label = _("Terminal programs")
+		terminalsGroup = guiHelper.BoxSizerHelper(
+			parent=self,
+			sizer=wx.StaticBoxSizer(parent=self, label=label, orient=wx.VERTICAL)
+		)
+		sHelper.addItem(terminalsGroup)
+		# Translators: This is the label for a checkbox in the
+		#  Advanced settings panel.
+		label = _("Use the new t&yped character support in legacy Windows consoles when available")
+		self.keyboardSupportInLegacyCheckBox=terminalsGroup.addItem(wx.CheckBox(self, label=label))
+		self.keyboardSupportInLegacyCheckBox.SetValue(config.conf["terminals"]["keyboardSupportInLegacy"])
+		self.keyboardSupportInLegacyCheckBox.defaultValue = self._getDefaultValue(["terminals", "keyboardSupportInLegacy"])
+		self.keyboardSupportInLegacyCheckBox.Enable(winVersion.isWin10(1607))
+
 		# Translators: This is the label for a group of advanced options in the
 		#  Advanced settings panel
 		label = _("Browse mode")
@@ -2154,6 +2170,7 @@ def haveConfigDefaultsBeenRestored(self):
 			self.UIAInMSWordCheckBox.IsChecked() == self.UIAInMSWordCheckBox.defaultValue and
 			self.ConsoleUIACheckBox.IsChecked() == (self.ConsoleUIACheckBox.defaultValue=='UIA') and
 			self.winConsoleSpeakPasswordsCheckBox.IsChecked() == self.winConsoleSpeakPasswordsCheckBox.defaultValue and
+			self.keyboardSupportInLegacyCheckBox.IsChecked() == self.keyboardSupportInLegacyCheckBox.defaultValue and
 			self.autoFocusFocusableElementsCheckBox.IsChecked() == self.autoFocusFocusableElementsCheckBox.defaultValue and
 			self.caretMoveTimeoutSpinControl.GetValue() == self.caretMoveTimeoutSpinControl.defaultValue and
 			set(self.logCategoriesList.CheckedItems) == set(self.logCategoriesList.defaultCheckedItems) and
@@ -2165,6 +2182,7 @@ def restoreToDefaults(self):
 		self.UIAInMSWordCheckBox.SetValue(self.UIAInMSWordCheckBox.defaultValue)
 		self.ConsoleUIACheckBox.SetValue(self.ConsoleUIACheckBox.defaultValue=='UIA')
 		self.winConsoleSpeakPasswordsCheckBox.SetValue(self.winConsoleSpeakPasswordsCheckBox.defaultValue)
+		self.keyboardSupportInLegacyCheckBox.SetValue(self.keyboardSupportInLegacyCheckBox.defaultValue)
 		self.autoFocusFocusableElementsCheckBox.SetValue(self.autoFocusFocusableElementsCheckBox.defaultValue)
 		self.caretMoveTimeoutSpinControl.SetValue(self.caretMoveTimeoutSpinControl.defaultValue)
 		self.logCategoriesList.CheckedItems = self.logCategoriesList.defaultCheckedItems
@@ -2179,6 +2197,7 @@ def onSave(self):
 		else:
 			config.conf['UIA']['winConsoleImplementation'] = "auto"
 		config.conf["UIA"]["winConsoleSpeakPasswords"]=self.winConsoleSpeakPasswordsCheckBox.IsChecked()
+		config.conf["terminals"]["keyboardSupportInLegacy"]=self.keyboardSupportInLegacyCheckBox.IsChecked()
 		config.conf["virtualBuffers"]["autoFocusFocusableElements"] = self.autoFocusFocusableElementsCheckBox.IsChecked()
 		config.conf["editableText"]["caretMoveTimeoutMs"]=self.caretMoveTimeoutSpinControl.GetValue()
 		for index,key in enumerate(self.logCategories):