UI Automation in Windows Console: add STABILIZE_DELAY, only enable UIA when available, limit blank lines in review, make speaking of passwords configurable, improvements to "speak typed words", and initial word movement support

source/NVDAObjects/IAccessible/__init__.py

@@ -528,6 +528,9 @@ def findOverlayClasses(self,clsList):
 		elif windowClassName.startswith("Chrome_"):
 			from . import chromium
 			chromium.findExtraOverlayClasses(self, clsList)
+		elif windowClassName == "ConsoleWindowClass":
+			from .winConsoleLegacy import winConsoleLegacy
+			clsList.append(winConsoleLegacy)
 
 
 		#Support for Windowless richEdit

source/NVDAObjects/window/winConsole.py → source/NVDAObjects/IAccessible/winConsoleLegacy.py

@@ -1,23 +1,23 @@
-#NVDAObjects/WinConsole.py
+#NVDAObjects/IAccessible/winConsoleLegacy.py
 #A part of NonVisual Desktop Access (NVDA)
 #This file is covered by the GNU General Public License.
 #See the file COPYING for more details.
-#Copyright (C) 2007-2012 NV Access Limited
+#Copyright (C) 2007-2019 NV Access Limited, Bill Dengler
 
-import winConsoleHandler
-from . import Window
+import winConsoleHandlerLegacy as winConsoleHandler
+from . import IAccessible
 from ..behaviors import Terminal, EditableTextWithoutAutoSelectDetection
 import api
 import core
 
-class WinConsole(Terminal, EditableTextWithoutAutoSelectDetection, Window):
+class winConsoleLegacy(Terminal, EditableTextWithoutAutoSelectDetection, IAccessible):
 	STABILIZE_DELAY = 0.03
 
 	def _get_TextInfo(self):
 		consoleObject=winConsoleHandler.consoleObject
 		if consoleObject and self.windowHandle == consoleObject.windowHandle:
-			return winConsoleHandler.WinConsoleTextInfo
-		return super(WinConsole,self).TextInfo
+			return winConsoleHandler.legacyConsoleTextInfo
+		return super(winConsoleLegacy,self).TextInfo
 
 	def event_becomeNavigatorObject(self, isFocus=False):
 		if winConsoleHandler.consoleObject is not self:
@@ -28,17 +28,17 @@ def event_becomeNavigatorObject(self, isFocus=False):
 				# The user is returning to the focus object with object navigation.
 				# The focused console should always be monitored if possible.
 				self.startMonitoring()
-		super(WinConsole,self).event_becomeNavigatorObject(isFocus=isFocus)
+		super(winConsoleLegacy,self).event_becomeNavigatorObject(isFocus=isFocus)
 
 	def event_gainFocus(self):
 		if winConsoleHandler.consoleObject is not self:
 			if winConsoleHandler.consoleObject:
 				winConsoleHandler.disconnectConsole()
 			winConsoleHandler.connectConsole(self)
-		super(WinConsole, self).event_gainFocus()
+		super(winConsoleLegacy, self).event_gainFocus()
 
 	def event_loseFocus(self):
-		super(WinConsole, self).event_loseFocus()
+		super(winConsoleLegacy, self).event_loseFocus()
 		if winConsoleHandler.consoleObject is self:
 			winConsoleHandler.disconnectConsole()
 
@@ -49,7 +49,7 @@ def _getTextLines(self):
 		return winConsoleHandler.getConsoleVisibleLines()
 
 	def script_caret_backspaceCharacter(self, gesture):
-		super(WinConsole, self).script_caret_backspaceCharacter(gesture)
+		super(winConsoleLegacy, self).script_caret_backspaceCharacter(gesture)
 		# #2586: We use console update events for typed characters,
 		# so the typedCharacter event is never fired for the backspace key.
 		# Call it here so that speak typed words works as expected.

source/NVDAObjects/UIA/winConsoleUIA.py

@@ -4,6 +4,10 @@
 # See the file COPYING for more details.
 # Copyright (C) 2019 Bill Dengler
 
+import config
+import ctypes
+import NVDAHelper
+import speech
 import time
 import textInfos
 import UIAHandler
@@ -16,10 +20,13 @@
 
 class consoleUIATextInfo(UIATextInfo):
 	_expandCollapseBeforeReview = False
+	_isCaret = False
+	_expandedToWord = False
 
 	def __init__(self, obj, position, _rangeObj=None):
 		super(consoleUIATextInfo, self).__init__(obj, position, _rangeObj)
 		if position == textInfos.POSITION_CARET:
+			self._isCaret = True
 			if isAtLeastWin10(1903):
 				# The UIA implementation in 1903 causes the caret to be
 				# off-by-one, so move it one position to the right
@@ -30,33 +37,172 @@ def __init__(self, obj, position, _rangeObj=None):
 					1
 				)
 
+	def move(self, unit, direction, endPoint=None):
+		if not self._isCaret:
+			# Insure we haven't gone beyond the visible text.
+			# UIA adds thousands of blank lines to the end of the console.
+			visiRanges = self.obj.UIATextPattern.GetVisibleRanges()
+			lastVisiRange = visiRanges.GetElement(visiRanges.length - 1)
+			if self._rangeObj.CompareEndPoints(
+				UIAHandler.TextPatternRangeEndpoint_Start,
+				lastVisiRange,
+				UIAHandler.TextPatternRangeEndpoint_End
+			) >= 0:
+				return 0
+		if unit == textInfos.UNIT_WORD and direction != 0:
+			# UIA doesn't implement word movement, so we need to do it manually.
+			offset = self._getCurrentOffset()
+			index = 1 if direction > 0 else 0
+			start, end = self._getWordOffsets(offset)
+			wordMoveDirections = (
+				(offset - start + 1) * -1,
+				end - offset
+			)
+			res = self.move(
+				textInfos.UNIT_CHARACTER,
+				wordMoveDirections[index],
+				endPoint=endPoint
+			)
+			if res != 0:
+				return direction
+			else:
+				return res
+		return super(consoleUIATextInfo, self).move(unit, direction, endPoint)
+
+	def expand(self, unit):
+		if unit == textInfos.UNIT_WORD:
+			self._expandedToWord = True
+		else:
+			self._expandedToWord = False
+			return super(consoleUIATextInfo, self).expand(unit)
+
+	def collapse(self):
+		self._expandedToWord = False
+		return super(consoleUIATextInfo, self).collapse()
+
+	def getTextWithFields(self, formatConfig=None):
+		if self._expandedToWord:
+			return [self.text]
+		return super(consoleUIATextInfo, self).getTextWithFields(
+			formatConfig=formatConfig
+		)
+
+	def _get_text(self):
+		if self._expandedToWord:
+			return self._getCurrentWord()
+		return super(consoleUIATextInfo, self)._get_text()
+
+	def _getCurrentOffset(self):
+		lineInfo = self.copy()
+		lineInfo.expand(textInfos.UNIT_LINE)
+		charInfo = self.copy()
+		res = 0
+		chars = None
+		while True:
+			charInfo.expand(textInfos.UNIT_CHARACTER)
+			chars = charInfo.move(textInfos.UNIT_CHARACTER, -1) * -1
+			if chars != 0 and charInfo.compareEndPoints(
+				lineInfo,
+				"startToStart"
+			) >= 0:
+				res += chars
+			else:
+				break
+		# Subtract 1 from res since UIA seems to wrap around
+		res -= 1
+		return res
+
+	def _getWordOffsets(self, offset):
+		lineInfo = self.copy()
+		lineInfo.expand(textInfos.UNIT_LINE)
+		lineText = lineInfo.text
+		# Convert NULL and non-breaking space to space to make sure
+		# that words will break on them
+		lineText = lineText.translate({0: u' ', 0xa0: u' '})
+		start = ctypes.c_int()
+		end = ctypes.c_int()
+		# Uniscribe does some strange things when you give it a string  with
+		# not more than two alphanumeric chars in a row.
+		# Inject two alphanumeric characters at the end to fix this.
+		lineText += "xx"
+		NVDAHelper.localLib.calculateWordOffsets(
+			lineText,
+			len(lineText),
+			offset,
+			ctypes.byref(start),
+			ctypes.byref(end)
+		)
+		return (
+			start.value,
+			min(end.value, len(lineText))
+		)
+
+	def _getCurrentWord(self):
+		lineInfo = self.copy()
+		lineInfo.expand(textInfos.UNIT_LINE)
+		lineText = lineInfo.text
+		offset = self._getCurrentOffset()
+		start, end = self._getWordOffsets(offset)
+		return lineText[start:end]
+
 
 class winConsoleUIA(Terminal):
+	STABILIZE_DELAY = 0.03
 	_TextInfo = consoleUIATextInfo
 	_isTyping = False
 	_lastCharTime = 0
+	_queuedChars = []
 	_TYPING_TIMEOUT = 1
 
 	def _reportNewText(self, line):
 		# Additional typed character filtering beyond that in LiveText
-		if self._isTyping and time.time() - self._lastCharTime <= self._TYPING_TIMEOUT:
+		if (
+			self._isTyping
+			and time.time() - self._lastCharTime <= self._TYPING_TIMEOUT
+		):
 			return
 		super(winConsoleUIA, self)._reportNewText(line)
 
 	def event_typedCharacter(self, ch):
 		if not ch.isspace():
 			self._isTyping = True
+		if ch in ('\r', '\t'):
+			# Clear the typed word buffer for tab and return.
+			# This will need to be changed once #8110 is merged.
+			speech.curWordChars = []
 		self._lastCharTime = time.time()
-		super(winConsoleUIA, self).event_typedCharacter(ch)
+		if (
+			(
+				config.conf['keyboard']['speakTypedCharacters']
+				or config.conf['keyboard']['speakTypedWords']
+			)
+			and not config.conf['UIA']['winConsoleSpeakPasswords']
+		):
+			self._queuedChars.append(ch)
+		else:
+			super(winConsoleUIA, self).event_typedCharacter(ch)
+
+	def event_textChange(self):
+		while self._queuedChars:
+			ch = self._queuedChars.pop(0)
+			super(winConsoleUIA, self).event_typedCharacter(ch)
+		super(winConsoleUIA, self).event_textChange()
 
-	@script(gestures=["kb:enter", "kb:numpadEnter", "kb:tab"])
+	@script(gestures=[
+		"kb:enter",
+		"kb:numpadEnter",
+		"kb:tab",
+		"kb:control+c",
+		"kb:control+d",
+		"kb:control+pause"
+	])
 	def script_clear_isTyping(self, gesture):
 		gesture.send()
 		self._isTyping = False
+		self._queuedChars = []
 
 	def _getTextLines(self):
 		# Filter out extraneous empty lines from UIA
-		# Todo: do this (also) somewhere else so they aren't in document review either
 		ptr = self.UIATextPattern.GetVisibleRanges()
 		res = [ptr.GetElement(i).GetText(-1) for i in range(ptr.length)]
 		return res

source/NVDAObjects/window/__init__.py

@@ -1,6 +1,6 @@
 #NVDAObjects/window.py
 #A part of NonVisual Desktop Access (NVDA)
-#Copyright (C) 2006-2019 NV Access Limited, Babbage B.V.
+#Copyright (C) 2006-2019 NV Access Limited, Babbage B.V., Bill Dengler
 #This file is covered by the GNU General Public License.
 #See the file COPYING for more details.
 
@@ -124,8 +124,6 @@ def findOverlayClasses(self,clsList):
 			from .scintilla import Scintilla as newCls
 		elif windowClassName in ("AkelEditW", "AkelEditA"):
 			from .akelEdit import AkelEdit as newCls
-		elif windowClassName=="ConsoleWindowClass" and config.conf['UIA']['winConsoleImplementation'] != "UIA":
-			from .winConsole import WinConsole as newCls
 		elif windowClassName=="EXCEL7":
 			from .excel import Excel7Window as newCls
 		if newCls:

source/config/configSpec.py

@@ -1,6 +1,6 @@
 # -*- coding: UTF-8 -*-
 #A part of NonVisual Desktop Access (NVDA)
-#Copyright (C) 2006-2018 NV Access Limited, Babbage B.V., Davy Kager
+#Copyright (C) 2006-2019 NV Access Limited, Babbage B.V., Davy Kager, Bill Dengler
 #This file is covered by the GNU General Public License.
 #See the file COPYING for more details.
 
@@ -188,6 +188,7 @@
 	enabled = boolean(default=true)
 	useInMSWordWhenAvailable = boolean(default=false)
 	winConsoleImplementation= option("auto", "legacy", "UIA", default="auto")
+	winConsoleSpeakPasswords = boolean(default=false)
 
 [update]
 	autoCheck = boolean(default=true)

source/core.py

@@ -416,9 +416,9 @@ def handlePowerStatusChange(self):
 		log.warning("Java Access Bridge not available")
 	except:
 		log.error("Error initializing Java Access Bridge support", exc_info=True)
-	import winConsoleHandler
-	log.debug("Initializing winConsole support")
-	winConsoleHandler.initialize()
+	import winConsoleHandlerLegacy
+	log.debug("Initializing legacy winConsole support")
+	winConsoleHandlerLegacy.initialize()
 	import UIAHandler
 	log.debug("Initializing UIA support")
 	try:
@@ -542,7 +542,7 @@ def run(self):
 	_terminate(treeInterceptorHandler)
 	_terminate(IAccessibleHandler, name="IAccessible support")
 	_terminate(UIAHandler, name="UIA support")
-	_terminate(winConsoleHandler, name="winConsole support")
+	_terminate(winConsoleHandlerLegacy, name="Legacy winConsole support")
 	_terminate(JABHandler, name="Java Access Bridge support")
 	_terminate(appModuleHandler, name="app module handler")
 	_terminate(NVDAHelper)

source/gui/settingsDialogs.py

@@ -2052,12 +2052,19 @@ def __init__(self, parent):
 
 		# Translators: This is the label for a checkbox in the
 		#  Advanced settings panel.
-		label = _("Force UI Automation in the Windows Console")
+		label = _("Use UI Automation to access the Windows C&onsole when available")
 		consoleUIADevMap = True if config.conf['UIA']['winConsoleImplementation'] == 'UIA' else False
 		self.ConsoleUIACheckBox=UIAGroup.addItem(wx.CheckBox(self, label=label))
 		self.ConsoleUIACheckBox.SetValue(consoleUIADevMap)
 		self.ConsoleUIACheckBox.defaultValue = self._getDefaultValue(["UIA", "winConsoleImplementation"])
 
+		# Translators: This is the label for a checkbox in the
+		#  Advanced settings panel.
+		label = _("Speak &passwords in UIA consoles (may improve performance)")
+		self.winConsoleSpeakPasswordsCheckBox=UIAGroup.addItem(wx.CheckBox(self, label=label))
+		self.winConsoleSpeakPasswordsCheckBox.SetValue(config.conf["UIA"]["winConsoleSpeakPasswords"])
+		self.winConsoleSpeakPasswordsCheckBox.defaultValue = self._getDefaultValue(["UIA", "winConsoleSpeakPasswords"])
+
 		# Translators: This is the label for a group of advanced options in the
 		#  Advanced settings panel
 		label = _("Browse mode")
@@ -2166,6 +2173,7 @@ def onSave(self):
 			config.conf['UIA']['winConsoleImplementation'] = "UIA"
 		else:
 			config.conf['UIA']['winConsoleImplementation'] = "auto"
+		config.conf["UIA"]["winConsoleSpeakPasswords"]=self.winConsoleSpeakPasswordsCheckBox.IsChecked()
 		config.conf["virtualBuffers"]["autoFocusFocusableElements"] = self.autoFocusFocusableElementsCheckBox.IsChecked()
 		config.conf["editableText"]["caretMoveTimeoutMs"]=self.caretMoveTimeoutSpinControl.GetValue()
 		for index,key in enumerate(self.logCategories):

source/winConsoleHandler.py → source/winConsoleHandlerLegacy.py

@@ -1,13 +1,13 @@
-#winConsoleHandler.py
+#winConsoleHandlerLegacy.py
 #A part of NonVisual Desktop Access (NVDA)
 #This file is covered by the GNU General Public License.
 #See the file COPYING for more details.
-#Copyright (C) 2009-2018 NV Access Limited, Babbage B.V.
+#Copyright (C) 2009-2019 NV Access Limited, Babbage B.V., Bill Dengler
 
 import gui
 import winUser
 import winKernel
-import wincon
+import winconLegacy as wincon
 from colors import RGB
 import eventHandler
 from logHandler import log
@@ -150,7 +150,7 @@ def terminate():
 	if consoleObject:
 		disconnectConsole()
 
-class WinConsoleTextInfo(textInfos.offsets.OffsetsTextInfo):
+class legacyConsoleTextInfo(textInfos.offsets.OffsetsTextInfo):
 
 	_cache_consoleScreenBufferInfo=True
 	def _get_consoleScreenBufferInfo(self):

user_docs/en/userGuide.t2t

@@ -1671,9 +1671,12 @@ This includes in Microsoft Word itself, and also the Microsoft Outlook message v
 However, There may be some information which is either not exposed, or exposed incorrectly in some versions of Microsoft Office, which means this UI automation support cannot always be relied upon.
 We still do not recommend that  the majority of users turn this on by default, though we do welcome users of Office 2016/365 to test this feature and provide feedback.
 
-==== Force UI Automation in the Windows Console====[AdvancedSettingsConsoleUIA]
+==== Use UI Automation to access the Windows Console when available ====[AdvancedSettingsConsoleUIA]
 When this option is enabled, NVDA will use a new, work in progress version of its support for Windows Console which takes advantage of [accessibility improvements made by Microsoft https://devblogs.microsoft.com/commandline/whats-new-in-windows-console-in-windows-10-fall-creators-update/]. This feature is highly experimental and is still incomplete, so its use is not yet recommended. However, once completed, it is anticipated that this new support will become the default, improving NVDA's performance and stability in Windows command consoles.
 
+==== Speak passwords in UIA consoles ====[AdvancedSettingsWinConsoleSpeakPasswords]
+This setting controls whether characters are spoken by [speak typed characters #KeyboardSettingsSpeakTypedCharacters] or [speak typed words #KeyboardSettingsSpeakTypedWords] in situations where the screen does not update (such as password entry) in the Windows Console with UI automation support enabled. For security purposes, this setting should be left disabled. However, you may wish to enable it if you experience performance issues or instability with typed character and/or word reporting while using NVDA's new experimental console support.
+
 ==== Automatically set system focus to focusable elements in Browse Mode ====[BrowseModeSettingsAutoFocusFocusableElements]
 Key: NVDA+8