From 7e31f30f57bf0f0e04e7a65dec9acffb4b88ec18 Mon Sep 17 00:00:00 2001 From: Michael Curran Date: Wed, 29 May 2024 10:43:20 +1000 Subject: [PATCH] Implement support for writing UI Automation Remote Operations natively in NVDA using Python (#16214) Several years ago Microsoft added a low-level API in Windows to allow the caller to execute multiple actions in a UI Automation remote provider via only one cross-process call. Limiting the amount of cross-process calls is necessary for providing a responsive experience, especially for interacting with applications hosted in Windows Defender Application Guard (WDAG) or as a Remote Application Integrated Locally (RAIL). There are also scenarios such as searching for / listing headings in large documents, which can bennifit from limiting cross-process calls, even when the application is running on the local machine. the Windows.UI.UIAutomation.Core.CoreAutomationRemoteOperation is a winRT interface that allows the caller to specify the actions as a string of byte code representing a specific set of instructions for a conceptual virtual machine which is executed within the remote provider. The instructions allow you to manipulate UI Automation elements, patterns and text ranges, as well as perform basic logic and collect information using ints, floats, strings, GUIDs and arrays created within the conceptual virtual machine. Although these instructions are well-defined, public documentation is limited and programming with these raw instructions can be slow and complicated to read. Microsoft released the Microsoft-UI-UIAutomation Remote Operations Library which provides some higher level ways of building and executing these instructions, including intermediate winRT interfaces, and high-level pure c++/winRT interfaces. NV Access has made many prototypes with this library, and has made significant contributions to that project over the years. Today, NVDA includes this library as a depedency, and currently uses it to access the new UI Automation Custom Extensions feature, which is required to access certain information in Microsoft Word. In theory, NVDA could make a much greater uuse of UI automation Remote Operations to improve performance across many parts of Windows and other applications. However, there are several issues with the Microsoft UI Automation Remote Operations Library which slow down or limit greater usage in NVDA, including: All remote operations logic must be written in C++ and not Python. This creates a large learning curve, if not a quite uncomfortable context switch at very least. the Remote Operations Library seems to be no longer maintained by Microsoft. The Remote Operations Library currently contributes an extra massive 16 minutes to NvDA's build time, as it must compile the whole of the C++/winRT API set. Solution This PR introduces a new remote operations library written in pure Python, which wraps the low-level CoreAutomationRemoteOperation winRT interfaces in Windows. This new library replaces the need for the older Microsoft Remote Operations Library, and therefore is no longer a dependency. I had several initial goals in mind with this new library: To support our current usage of remote operations (namely UI Automation custom extensions: isExtension and CallExtension). If I could not achieve our current requirements, there would be no point continuing. To make a significant performance improvement to at least one extra action or scenario in NvDA (E.g. quick navigation by heading in Microsoft Word). This would prove that implementation of a fix / feature was easier with the new library, and would make it worth spending the initial time on it. I was able to achieve both of these goals. Description of user facing changes In browse mode in Microsoft Word with UIA enabled, quick nav to headings, and listing headings in Elements list, is now up to 30 times faster. E.g. Jumping to a heading at the bottom of a 160 page document all the way from the top previously took ~4 seconds, now it takes ~0.13 seconds. Description of development approach A new UIAHandler._remoteOps package has been added, which interfaces with the low-level Windows Remote Operations API, providing higher-level APIs and types to aide in building and executing easy to read remote algorithms. See The NVDA remote operations readme for explanations and example code for all features. UIAHandler.remote's msWord_getCustomAttributeValue has been re-written to use the new NVDA remote operations library. Some extra functions were added to UIAHandler.remote, including findFirstHeadingInTextRange and collectAllHeadingsInTextRange, built on the NVDA remote ops library. UIA browse mode makes use of these functions on Windows 11, to majorly speed up jumping to / listing headings. --- .gitmodules | 3 - include/microsoft-ui-uiautomation | 1 - include/readme.md | 6 - nvdaHelper/UIARemote/UIARemote.cpp | 174 ----- nvdaHelper/UIARemote/lowLevel.cpp | 192 +++++ nvdaHelper/UIARemote/remoteLog.h | 77 -- nvdaHelper/UIARemote/sconscript | 17 +- nvdaHelper/archBuild_sconscript | 3 - .../Microsoft.UI.UIAutomation.dll.manifest | 12 - .../microsoft-ui-uiautomation/sconscript | 138 ---- nvdaHelper/readme.md | 5 - projectDocs/dev/createDevEnvironment.md | 3 - source/NVDAObjects/UIA/wordDocument.py | 2 +- source/UIAHandler/_remoteOps/__init__.py | 0 source/UIAHandler/_remoteOps/builder.py | 299 +++++++ .../_remoteOps/instructions/__init__.py | 111 +++ .../_remoteOps/instructions/_base.py | 17 + .../_remoteOps/instructions/arithmetic.py | 110 +++ .../_remoteOps/instructions/array.py | 93 +++ .../_remoteOps/instructions/bool.py | 70 ++ .../_remoteOps/instructions/controlFlow.py | 67 ++ .../_remoteOps/instructions/element.py | 79 ++ .../_remoteOps/instructions/extension.py | 39 + .../_remoteOps/instructions/float.py | 37 + .../_remoteOps/instructions/general.py | 56 ++ .../_remoteOps/instructions/guid.py | 37 + .../UIAHandler/_remoteOps/instructions/int.py | 59 ++ .../_remoteOps/instructions/null.py | 35 + .../_remoteOps/instructions/status.py | 27 + .../_remoteOps/instructions/string.py | 63 ++ .../_remoteOps/instructions/textRange.py | 200 +++++ source/UIAHandler/_remoteOps/localExecute.py | 228 ++++++ source/UIAHandler/_remoteOps/lowLevel.py | 376 +++++++++ source/UIAHandler/_remoteOps/operation.py | 354 +++++++++ source/UIAHandler/_remoteOps/readme.md | 556 +++++++++++++ source/UIAHandler/_remoteOps/remoteAPI.py | 379 +++++++++ .../UIAHandler/_remoteOps/remoteAlgorithms.py | 40 + .../_remoteOps/remoteFuncWrapper.py | 114 +++ .../_remoteOps/remoteTypes/__init__.py | 739 ++++++++++++++++++ .../_remoteOps/remoteTypes/element.py | 95 +++ .../_remoteOps/remoteTypes/extensionTarget.py | 74 ++ .../_remoteOps/remoteTypes/intEnum.py | 95 +++ .../_remoteOps/remoteTypes/textRange.py | 273 +++++++ source/UIAHandler/browseMode.py | 33 +- source/UIAHandler/remote.py | 168 +++- tests/unit/test_UIARemoteOps/__init__.py | 0 .../test_highLevel/__init__.py | 0 .../test_highLevel/test_bool.py | 91 +++ .../test_highLevel/test_element.py | 36 + .../test_highLevel/test_errorHandling.py | 61 ++ .../test_highLevel/test_float.py | 119 +++ .../test_highLevel/test_if.py | 100 +++ .../test_highLevel/test_instructionLimit.py | 54 ++ .../test_highLevel/test_int.py | 145 ++++ .../test_highLevel/test_iterable.py | 97 +++ .../test_highLevel/test_numericComparison.py | 84 ++ .../test_highLevel/test_string.py | 28 + .../test_highLevel/test_while.py | 61 ++ 58 files changed, 5970 insertions(+), 462 deletions(-) delete mode 160000 include/microsoft-ui-uiautomation delete mode 100644 nvdaHelper/UIARemote/UIARemote.cpp create mode 100644 nvdaHelper/UIARemote/lowLevel.cpp delete mode 100644 nvdaHelper/UIARemote/remoteLog.h delete mode 100644 nvdaHelper/microsoft-ui-uiautomation/Microsoft.UI.UIAutomation.dll.manifest delete mode 100644 nvdaHelper/microsoft-ui-uiautomation/sconscript create mode 100644 source/UIAHandler/_remoteOps/__init__.py create mode 100644 source/UIAHandler/_remoteOps/builder.py create mode 100644 source/UIAHandler/_remoteOps/instructions/__init__.py create mode 100644 source/UIAHandler/_remoteOps/instructions/_base.py create mode 100644 source/UIAHandler/_remoteOps/instructions/arithmetic.py create mode 100644 source/UIAHandler/_remoteOps/instructions/array.py create mode 100644 source/UIAHandler/_remoteOps/instructions/bool.py create mode 100644 source/UIAHandler/_remoteOps/instructions/controlFlow.py create mode 100644 source/UIAHandler/_remoteOps/instructions/element.py create mode 100644 source/UIAHandler/_remoteOps/instructions/extension.py create mode 100644 source/UIAHandler/_remoteOps/instructions/float.py create mode 100644 source/UIAHandler/_remoteOps/instructions/general.py create mode 100644 source/UIAHandler/_remoteOps/instructions/guid.py create mode 100644 source/UIAHandler/_remoteOps/instructions/int.py create mode 100644 source/UIAHandler/_remoteOps/instructions/null.py create mode 100644 source/UIAHandler/_remoteOps/instructions/status.py create mode 100644 source/UIAHandler/_remoteOps/instructions/string.py create mode 100644 source/UIAHandler/_remoteOps/instructions/textRange.py create mode 100644 source/UIAHandler/_remoteOps/localExecute.py create mode 100644 source/UIAHandler/_remoteOps/lowLevel.py create mode 100644 source/UIAHandler/_remoteOps/operation.py create mode 100644 source/UIAHandler/_remoteOps/readme.md create mode 100644 source/UIAHandler/_remoteOps/remoteAPI.py create mode 100644 source/UIAHandler/_remoteOps/remoteAlgorithms.py create mode 100644 source/UIAHandler/_remoteOps/remoteFuncWrapper.py create mode 100644 source/UIAHandler/_remoteOps/remoteTypes/__init__.py create mode 100644 source/UIAHandler/_remoteOps/remoteTypes/element.py create mode 100644 source/UIAHandler/_remoteOps/remoteTypes/extensionTarget.py create mode 100644 source/UIAHandler/_remoteOps/remoteTypes/intEnum.py create mode 100644 source/UIAHandler/_remoteOps/remoteTypes/textRange.py create mode 100644 tests/unit/test_UIARemoteOps/__init__.py create mode 100644 tests/unit/test_UIARemoteOps/test_highLevel/__init__.py create mode 100644 tests/unit/test_UIARemoteOps/test_highLevel/test_bool.py create mode 100644 tests/unit/test_UIARemoteOps/test_highLevel/test_element.py create mode 100644 tests/unit/test_UIARemoteOps/test_highLevel/test_errorHandling.py create mode 100644 tests/unit/test_UIARemoteOps/test_highLevel/test_float.py create mode 100644 tests/unit/test_UIARemoteOps/test_highLevel/test_if.py create mode 100644 tests/unit/test_UIARemoteOps/test_highLevel/test_instructionLimit.py create mode 100644 tests/unit/test_UIARemoteOps/test_highLevel/test_int.py create mode 100644 tests/unit/test_UIARemoteOps/test_highLevel/test_iterable.py create mode 100644 tests/unit/test_UIARemoteOps/test_highLevel/test_numericComparison.py create mode 100644 tests/unit/test_UIARemoteOps/test_highLevel/test_string.py create mode 100644 tests/unit/test_UIARemoteOps/test_highLevel/test_while.py diff --git a/.gitmodules b/.gitmodules index 103fabc4231..a006510f0b7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -30,9 +30,6 @@ [submodule "include/nsis"] path = include/nsis url = https://github.com/nvaccess/nsis-build -[submodule "include/microsoft-ui-uiautomation"] - path = include/microsoft-ui-uiautomation - url = https://github.com/michaeldcurran/microsoft-ui-uiautomation [submodule "include/wil"] path = include/wil url = https://github.com/microsoft/wil diff --git a/include/microsoft-ui-uiautomation b/include/microsoft-ui-uiautomation deleted file mode 160000 index 224b22f3bf9..00000000000 --- a/include/microsoft-ui-uiautomation +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 224b22f3bf9e6bcd7326e471d1e134fe739bf4ff diff --git a/include/readme.md b/include/readme.md index 627570f65bb..5065d024677 100644 --- a/include/readme.md +++ b/include/readme.md @@ -24,12 +24,6 @@ Fetch latest from master. ### liblouis TODO -### microsoft-ui-uiautomation -https://www.github.com/michaeldcurran/microsoft-ui-uiautomation/ - -We are stuck on our own fork. -The fork specifically adds support for connection bound objects to the high-level API, see PR microsoft/microsoft-ui-uiautomation#95. - ### nsis [NSIS-build readme](https://github.com/nvaccess/NSIS-build) diff --git a/nvdaHelper/UIARemote/UIARemote.cpp b/nvdaHelper/UIARemote/UIARemote.cpp deleted file mode 100644 index dbc67209e60..00000000000 --- a/nvdaHelper/UIARemote/UIARemote.cpp +++ /dev/null @@ -1,174 +0,0 @@ -/* -This file is a part of the NVDA project. -URL: http://www.nvaccess.org/ -Copyright 2021-2022 NV Access Limited - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License version 2.0, as published by - the Free Software Foundation. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -This license can be found at: -http://www.gnu.org/licenses/old-licenses/gpl-2.0.html -*/ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace UiaOperationAbstraction; - -#include "remoteLog.h" - -wchar_t dllDirectory[MAX_PATH]; - -// Several custom extension GUIDs specific to Microsoft Word -winrt::guid guid_msWord_extendedTextRangePattern{ 0x93514122, 0xff04, 0x4b2c, { 0xa4, 0xad, 0x4a, 0xb0, 0x45, 0x87, 0xc1, 0x29 } }; -winrt::guid guid_msWord_getCustomAttributeValue{ 0x81aca91, 0x32f2, 0x46f0, { 0x9f, 0xb9, 0x1, 0x70, 0x38, 0xbc, 0x45, 0xf8 } }; - -bool _isInitialized {false}; - -// Fetches a custom attribute value from a range of text in Microsoft Word. -// this function uses the UI automation Operation Abstraction API to call the Microsoft Word specific custom extension -extern "C" __declspec(dllexport) bool __stdcall msWord_getCustomAttributeValue(IUIAutomationElement* docElementArg, IUIAutomationTextRange* pTextRangeArg, int customAttribIDArg, VARIANT* pCustomAttribValueArg) { - if(!_isInitialized) { - LOG_ERROR(L"UIARemote not initialized!"); - return false; - } - try { - auto scope=UiaOperationScope::StartNew(); - RemoteableLogger logger{scope}; - // Here starts declarative code which will be executed remotely - logger<vt = VT_I4; - pCustomAttribValueArg->lVal = customAttribValue.AsInt(); - return true; - } else if(customAttribValue.IsString()) { - pCustomAttribValueArg->vt = VT_BSTR; - pCustomAttribValueArg->bstrVal = customAttribValue.AsString().get(); - return true; - } else { - LOG_ERROR(L"Unknown data type"); - return false; - } - } else { - LOG_DEBUG(L"Extension not supported"); - } - } catch (std::exception& e) { - auto wideWhat = stringToWstring(e.what()); - LOG_ERROR(L"msWord_getCustomAttributeValue exception: "<()) { - LOG_ERROR(L"Unable to get Microsoft.UI.UIAutomation activation factory"); - return false; - } - LOG_INFO(L"Microsoft.UI.UIAutomation is available"); - UiaOperationAbstraction::Initialize(doRemote,client); - _isInitialized = true; - return true; -} - -// Cleans up the remote operations library. -extern "C" __declspec(dllexport) void __stdcall cleanup() { - if (_isInitialized) { - LOG_INFO(L"Cleaning up the UIA Remote Operations abstraction library...") - UiaOperationAbstraction::Cleanup(); - _isInitialized = false; - LOG_INFO(L"Done") - } -} - - -BOOL WINAPI DllMain(HINSTANCE hModule,DWORD reason,LPVOID lpReserved) { - if(reason==DLL_PROCESS_ATTACH) { - GetModuleFileName(hModule,dllDirectory,MAX_PATH); - PathRemoveFileSpec(dllDirectory); - } - return true; -} - diff --git a/nvdaHelper/UIARemote/lowLevel.cpp b/nvdaHelper/UIARemote/lowLevel.cpp new file mode 100644 index 00000000000..e28d51bee18 --- /dev/null +++ b/nvdaHelper/UIARemote/lowLevel.cpp @@ -0,0 +1,192 @@ +#include +#include +#include +#include + +using namespace winrt::Windows::Foundation; +using namespace winrt::Windows::UI::UIAutomation; +using namespace winrt::Windows::UI::UIAutomation::Core; + +extern "C" __declspec(dllexport) HRESULT __stdcall remoteOp_create(void** ppRemoteOp) { + *ppRemoteOp = winrt::detach_abi(CoreAutomationRemoteOperation()); + return S_OK; +} + +extern "C" __declspec(dllexport) HRESULT __stdcall remoteOp_free(void* arg_pRemoteOp) { + CoreAutomationRemoteOperation operation {nullptr}; + winrt::attach_abi(operation, arg_pRemoteOp); + return S_OK; +} + +extern "C" __declspec(dllexport) HRESULT __stdcall remoteOp_importElement(void* arg_pRemoteOp, int arg_registerID, IUIAutomationElement* arg_pElement) { + CoreAutomationRemoteOperation operation {nullptr}; + winrt::copy_from_abi(operation, arg_pRemoteOp); + if(!operation) { + return E_FAIL; + } + AutomationElement element {nullptr}; + winrt::copy_from_abi(element, arg_pElement); + if(!element) { + return E_FAIL; + } + operation.ImportElement(AutomationRemoteOperationOperandId{arg_registerID}, element); + return S_OK; +} + +extern "C" __declspec(dllexport) HRESULT __stdcall remoteOp_importTextRange(void* arg_pRemoteOp, int arg_registerID, IUIAutomationTextRange* arg_pTextRange) { + CoreAutomationRemoteOperation operation {nullptr}; + winrt::copy_from_abi(operation, arg_pRemoteOp); + if(!operation) { + return E_FAIL; + } + AutomationTextRange textRange {nullptr}; + winrt::copy_from_abi(textRange, arg_pTextRange); + if(!textRange) { + return E_FAIL; + } + operation.ImportTextRange(AutomationRemoteOperationOperandId{arg_registerID}, textRange); + return S_OK; +} + +extern "C" __declspec(dllexport) HRESULT __stdcall remoteOp_addToResults(void* arg_pRemoteOp, int arg_registerID) { + CoreAutomationRemoteOperation operation {nullptr}; + winrt::copy_from_abi(operation, arg_pRemoteOp); + if(!operation) { + return E_FAIL; + } + operation.AddToResults(AutomationRemoteOperationOperandId{arg_registerID}); + return S_OK; +} + +extern "C" __declspec(dllexport) HRESULT __stdcall remoteOp_isOpcodeSupported(void* arg_pRemoteOp, uint32_t arg_opcode, bool* pIsSupported) { + CoreAutomationRemoteOperation operation {nullptr}; + winrt::copy_from_abi(operation, arg_pRemoteOp); + if(!operation) { + return E_FAIL; + } + *pIsSupported = operation.IsOpcodeSupported(arg_opcode); + return S_OK; +} + +extern "C" __declspec(dllexport) HRESULT __stdcall remoteOp_execute(void* arg_pRemoteOp, uint8_t arg_byteCodeBuffer[], int arg_byteCodeBufferLength, void** ppResults) { + CoreAutomationRemoteOperation operation {nullptr}; + winrt::copy_from_abi(operation, arg_pRemoteOp); + if(!operation) { + return E_FAIL; + } + auto results = operation.Execute(winrt::array_view(arg_byteCodeBuffer, arg_byteCodeBufferLength)); + *ppResults = winrt::detach_abi(results); + return S_OK; +} + +extern "C" __declspec(dllexport) HRESULT __stdcall remoteOpResult_getErrorLocation(void* arg_pResults, int* pErrorLocation) { + AutomationRemoteOperationResult results {nullptr}; + winrt::copy_from_abi(results, arg_pResults); + if(!results) { + return E_INVALIDARG; + } + *pErrorLocation = results.ErrorLocation(); + return S_OK; +} + +extern "C" __declspec(dllexport) HRESULT __stdcall remoteOpResult_getExtendedError(void* arg_pResults, HRESULT* pExtendedError) { + AutomationRemoteOperationResult results {nullptr}; + winrt::copy_from_abi(results, arg_pResults); + if(!results) { + return E_INVALIDARG; + } + *pExtendedError = results.ExtendedError(); + return S_OK; +} + +extern "C" __declspec(dllexport) HRESULT __stdcall remoteOpResult_getStatus(void* arg_pResults, int* pStatus) { + AutomationRemoteOperationResult results {nullptr}; + winrt::copy_from_abi(results, arg_pResults); + if(!results) { + return E_INVALIDARG; + } + auto status = results.Status(); + *pStatus = static_cast(status); + return S_OK; +} + +extern "C" __declspec(dllexport) HRESULT __stdcall remoteOpResult_hasOperand(void* arg_pResults, int arg_registerID, bool* pHasOperand) { + AutomationRemoteOperationResult results {nullptr}; + winrt::copy_from_abi(results, arg_pResults); + if(!results) { + return E_INVALIDARG; + } + *pHasOperand = results.HasOperand(AutomationRemoteOperationOperandId{arg_registerID}); + return S_OK; +} + +extern "C" __declspec(dllexport) HRESULT __stdcall remoteOpResult_free(void* arg_pResults) { + AutomationRemoteOperationResult results {nullptr}; + winrt::attach_abi(results, arg_pResults); + return S_OK; +} + +HRESULT IInspectableToVariant(winrt::Windows::Foundation::IInspectable result, VARIANT* arg_pVariant) { + if(!result) { + // operand is NULL. + return S_OK; + } + auto propVal = result.try_as(); + if(propVal) { + // Unbox property value into VARIANT + auto propType = propVal.Type(); + switch(propVal.Type()) { + case PropertyType::Int32: + arg_pVariant->vt = VT_I4; + arg_pVariant->lVal = propVal.GetInt32(); + break; + case PropertyType::String: + arg_pVariant->vt = VT_BSTR; + arg_pVariant->bstrVal = SysAllocString(propVal.GetString().c_str()); + break; + case PropertyType::Boolean: + arg_pVariant->vt = VT_BOOL; + arg_pVariant->boolVal = propVal.GetBoolean() ? VARIANT_TRUE : VARIANT_FALSE; + break; + case PropertyType::Inspectable: + arg_pVariant->vt = VT_UNKNOWN; + arg_pVariant->punkVal = static_cast<::IUnknown*>(winrt::detach_abi(propVal.as())); + break; + default: + return E_NOTIMPL; + } + return S_OK; + } + auto vec = result.try_as>(); + if(vec) { + // Unbox vector into VARIANT array. + auto vecSize = vec.Size(); + arg_pVariant->vt = VT_ARRAY | VT_VARIANT; + arg_pVariant->parray = SafeArrayCreateVector(VT_VARIANT, 0, vecSize); + if(!arg_pVariant->parray) { + return E_OUTOFMEMORY; + } + for(ULONG i = 0; i < vecSize; i++) { + auto vecItem = vec.GetAt(i); + auto hr = IInspectableToVariant(vecItem, &static_cast(arg_pVariant->parray->pvData)[i]); + if(FAILED(hr)) { + return hr; + } + } + return S_OK; + } + // Just treat it as an IUnknown. + arg_pVariant->vt = VT_UNKNOWN; + arg_pVariant->punkVal = static_cast<::IUnknown*>(winrt::detach_abi(result.as())); + return S_OK; +} + +extern "C" __declspec(dllexport) HRESULT __stdcall remoteOpResult_getOperand(void* arg_pResults, int arg_registerID, VARIANT* arg_pVariant) { + AutomationRemoteOperationResult results {nullptr}; + winrt::copy_from_abi(results, arg_pResults); + if(!results) { + return E_INVALIDARG; + } + auto result = results.GetOperand(AutomationRemoteOperationOperandId{arg_registerID}); + return IInspectableToVariant(result, arg_pVariant); +} diff --git a/nvdaHelper/UIARemote/remoteLog.h b/nvdaHelper/UIARemote/remoteLog.h deleted file mode 100644 index b3bb9856dbd..00000000000 --- a/nvdaHelper/UIARemote/remoteLog.h +++ /dev/null @@ -1,77 +0,0 @@ -/* -This file is a part of the NVDA project. -URL: http://www.nvaccess.org/ -Copyright 2021-2022 NV Access Limited - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License version 2.0, as published by - the Free Software Foundation. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -This license can be found at: -http://www.gnu.org/licenses/old-licenses/gpl-2.0.html -*/ - -#pragma once - -// Converts a utf8 encoded string into a utf16 encoded wstring -std::wstring stringToWstring(const std::string& from) { - int wideLen = MultiByteToWideChar(CP_UTF8, 0, from.c_str(), from.length(), nullptr, 0); - std::wstring wideBuf (wideLen, L'\0'); - MultiByteToWideChar(CP_UTF8, 0, from.c_str(), from.length(), wideBuf.data(), wideLen); - return wideBuf; -} - -const std::wstring endl{L"\n"}; - -class RemoteableLogger; - -// A class for logging messages from within a remote ops call. -// Push messages to the object with << just like an ostream. -// Currently standard strings, UiaStrings, and UiaInt instances are supported. -// After remote execution is complete, call dumpLog to write the content to our standard logging framework. -class RemoteableLogger { - public: - - RemoteableLogger(UiaOperationScope& scope): _log{} { - scope.BindResult(_log); - } - - RemoteableLogger& operator <<(UiaInt& message) { - _log.Append(message.Stringify()); - return *this; - } - - RemoteableLogger& operator <<(UiaString& message) { - _log.Append(message); - return *this; - } - - RemoteableLogger& operator <<(const std::wstring message) { - _log.Append(message); - return *this; - } - - void dumpLog() { - assert(!UiaOperationAbstraction::ShouldUseRemoteApi()); - std::wstring messageBlock{L"Dump log start:\n"}; - try { - // locally, a UiaArray is a shared_ptr to a vector of will_shared_bstr - const std::vector& v = *_log; - for(const auto& message: v) { - messageBlock+=message.get(); - } - } catch (std::exception& e) { - auto wideWhat = stringToWstring(e.what()); - messageBlock += L"dumpLog exception: "; - messageBlock += wideWhat; - messageBlock += L"\n"; - } - messageBlock+=L"Dump log end"; - LOG_DEBUG(messageBlock); - } - - private: - UiaArray _log; - -}; diff --git a/nvdaHelper/UIARemote/sconscript b/nvdaHelper/UIARemote/sconscript index 3695a96f2a8..2faea1118fc 100644 --- a/nvdaHelper/UIARemote/sconscript +++ b/nvdaHelper/UIARemote/sconscript @@ -15,32 +15,19 @@ Import([ 'env', 'localLib', - 'MSUIA_lib_outDir', - 'MSUIA_include_outDir', -]) - -env = env.Clone() -env.Append(CPPPATH=Dir('#include/wil/include')) -env.Append(CPPPATH=MSUIA_include_outDir) -# Re-enable permissive mode as disabling it isn't supported for UiaOperationAbstraction.h -env['CCFLAGS'].remove('/permissive-') -env.Append(CCFLAGS=[ - '/MD', - # Swap C++20 for C++17 as C++20 is not yet supported by UiaOperationAbstraction.h - '/std:c++17', ]) UIARemoteLib=env.SharedLibrary( target="UIARemote", source=[ env['projectResFile'], - "UIARemote.cpp", + "lowLevel.cpp", ], LIBS=[ "runtimeobject", "UIAutomationCore", + "oleaut32", localLib[2], - MSUIA_lib_outDir.File('UiaOperationAbstraction.lib'), ], ) diff --git a/nvdaHelper/archBuild_sconscript b/nvdaHelper/archBuild_sconscript index d81b4ae6fd2..9deea2d081a 100644 --- a/nvdaHelper/archBuild_sconscript +++ b/nvdaHelper/archBuild_sconscript @@ -206,9 +206,6 @@ if TARGET_ARCH=='x86': if signExec: env.AddPostAction(win10localLib[0],[signExec]) env.Install(libInstallDir,win10localLib) - MSUIA_lib_outDir,MSUIA_include_outDir = thirdPartyEnv.SConscript('microsoft-ui-uiautomation/sconscript') - Export('MSUIA_lib_outDir') - Export('MSUIA_include_outDir') UIARemoteLib=env.SConscript('UIARemote/sconscript') if signExec: env.AddPostAction(UIARemoteLib[0],[signExec]) diff --git a/nvdaHelper/microsoft-ui-uiautomation/Microsoft.UI.UIAutomation.dll.manifest b/nvdaHelper/microsoft-ui-uiautomation/Microsoft.UI.UIAutomation.dll.manifest deleted file mode 100644 index f8b20a28579..00000000000 --- a/nvdaHelper/microsoft-ui-uiautomation/Microsoft.UI.UIAutomation.dll.manifest +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - diff --git a/nvdaHelper/microsoft-ui-uiautomation/sconscript b/nvdaHelper/microsoft-ui-uiautomation/sconscript deleted file mode 100644 index a63d7ce5844..00000000000 --- a/nvdaHelper/microsoft-ui-uiautomation/sconscript +++ /dev/null @@ -1,138 +0,0 @@ -### -#This file is a part of the NVDA project. -#URL: http://www.nvaccess.org/ -#Copyright 2021 NV Access Limited -#This program is free software: you can redistribute it and/or modify -#it under the terms of the GNU General Public License version 2.0, as published by -#the Free Software Foundation. -#This program is distributed in the hope that it will be useful, -#but WITHOUT ANY WARRANTY; without even the implied warranty of -#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -#This license can be found at: -#http://www.gnu.org/licenses/old-licenses/gpl-2.0.html -### - -import os -import glob - -""" -Builds the open source Microsoft-UI-UIAutomation Remote Operations library from https://github.com/microsoft/microsoft-ui-uiautomation. -This library contains both a low-level winrt API, and a higher-level pure C++ API. -The outputs of this sconscript are: -* a 'lib' directory, containing: - * microsoft.ui.uiAutomation.dll and .lib: the dll containing the low-level winrt implementation. - To use in other code, link against the .lib, and also load the included microsoft.ui.uiAutomation.dll.manifest file into an activation context and activate it. - * UiaOperationAbstraction.lib: a static library containing runtime code for the higher-level pure C++ API. - This should be linked into any compiled executable or library that needs to use the higher-level C++ API. -* an 'include' directory, containing: - * a 'UIAOperationAbstraction' directory containing all the public headers for the high-level C++ API - * a 'winrt' directory containing the generated C++/winrt language bindings of the low-level API, required by the high-level C++ API headers -""" - - -Import( - 'env', - 'sourceDir', - 'sourceLibDir', -) - -env = env.Clone() -# Building with msbuild requires windir to be set as environment variable, -# otherwise the build fails with Visual Studio 2022 -env['ENV']['windir'] = os.environ['windir'] - - -# We must ensure that msbuild uses the same platform toolset as the rest of NVDA. -# Thus we must fetch this info from the MSVC_VERSION SCons variable, -# Formatting it as vMajorMinor (e.g. v143 for 14.3). -platformToolset = "v" + "".join(env.get('MSVC_VERSION').split('.')[:2]) - -MSUIA_sourceDir = Dir('#include/microsoft-ui-uiautomation/src/uiAutomation') -MSUIA_lib_outDir = Dir('lib') -MSUIA_include_outDir = Dir('include') -MSUIA_solutionFile = MSUIA_sourceDir.File('UIAutomation.sln') - -MSUIA_wrapper_libs = env.Command( - target = [ - MSUIA_lib_outDir.File('Microsoft.UI.UIAutomation.dll'), - MSUIA_lib_outDir.File('Microsoft.UI.UIAutomation.exp'), - MSUIA_lib_outDir.File('Microsoft.UI.UIAutomation.pdb'), - MSUIA_lib_outDir.File('Microsoft.UI.UIAutomation.lib'), - MSUIA_lib_outDir.File('winmd/Microsoft.UI.UIAutomation.winmd'), - MSUIA_sourceDir.File('microsoft.ui.uiautomation/Generated Files/winrt/microsoft.ui.uiautomation.h'), - MSUIA_sourceDir.File('microsoft.ui.uiautomation/Generated Files/winrt/impl/microsoft.ui.uiautomation.0.h'), - MSUIA_sourceDir.File('microsoft.ui.uiautomation/Generated Files/winrt/impl/microsoft.ui.uiautomation.1.h'), - MSUIA_sourceDir.File('microsoft.ui.uiautomation/Generated Files/winrt/impl/microsoft.ui.uiautomation.2.h'), - ], - source = [ - MSUIA_sourceDir.File('microsoft.ui.uiautomation/Microsoft.UI.UIAutomation.vcxproj'), - MSUIA_sourceDir.File('microsoft.ui.uiautomation/Microsoft.UI.UIAutomation.idl'), - glob.glob(os.path.join(MSUIA_sourceDir.abspath, 'microsoft.ui.uiautomation', '*.cpp')), - glob.glob(os.path.join(MSUIA_sourceDir.abspath, 'microsoft.ui.uiautomation', '*.h')), - ], - action = [ - # Fetch any required NuGet packages - f"msbuild {MSUIA_solutionFile} /t:Restore /p:RestorePackagesConfig=true,Configuration=Release,Platform=x86,PlatformToolset={platformToolset}", - # Remove any old generated files - Delete(MSUIA_sourceDir.Dir('microsoft.ui.uiautomation/Generated Files')), - # Do the actual build - "msbuild /t:Build /p:Configuration=Release,Platform=x86,PlatformToolset={platformToolset},OutDir={outDir}/ $SOURCE".format( - outDir=MSUIA_lib_outDir.abspath, - platformToolset=platformToolset - ) - ], -) -env.Ignore(MSUIA_wrapper_libs,MSUIA_sourceDir.File('microsoft.ui.uiautomation/Microsoft.UI.UIAutomation_h.h')) - -env.Install(sourceLibDir,MSUIA_wrapper_libs[0]) -env.Install(sourceLibDir, "Microsoft.UI.UIAutomation.dll.manifest") - -MSUIA_abstraction_libs = env.Command( - target = [ - MSUIA_lib_outDir.File('UiaOperationAbstraction.lib'), - ], - source = [ - MSUIA_sourceDir.File('UiaOperationAbstraction/UiaOperationAbstraction.vcxproj'), - glob.glob(os.path.join(MSUIA_sourceDir.abspath, 'UiaOperationAbstraction', '*.cpp')), - glob.glob(os.path.join(MSUIA_sourceDir.abspath, 'UiaOperationAbstraction', '*.h')), - ], - action = [ - "msbuild /t:Build /p:Configuration=Release,Platform=x86,PlatformToolset={platformToolset},OutDir={outDir}/ $SOURCE".format( - outDir=MSUIA_lib_outDir.abspath, - platformToolset=platformToolset - ), - ] -) - -env.Depends(MSUIA_abstraction_libs,MSUIA_wrapper_libs) - -MSUIA_wrapper_header = env.Install( - MSUIA_include_outDir.Dir('winrt'), - MSUIA_sourceDir.File('microsoft.ui.uiautomation/Generated Files/winrt/microsoft.ui.uiautomation.h'), -) -env.Depends(MSUIA_wrapper_header,MSUIA_wrapper_libs) - -MSUIA_wrapper_impl = env.Install( - MSUIA_include_outDir.Dir('winrt/impl'), - [ - MSUIA_sourceDir.File('microsoft.ui.uiautomation/Generated Files/winrt/impl/microsoft.ui.uiautomation.0.h'), - MSUIA_sourceDir.File('microsoft.ui.uiautomation/Generated Files/winrt/impl/microsoft.ui.uiautomation.1.h'), - MSUIA_sourceDir.File('microsoft.ui.uiautomation/Generated Files/winrt/impl/microsoft.ui.uiautomation.2.h'), - ] -) -env.Depends(MSUIA_wrapper_impl,MSUIA_wrapper_libs) - - -MSUIA_abstraction_headers = env.Install( - MSUIA_include_outDir.Dir('UiaOperationAbstraction'), - [ - MSUIA_sourceDir.File('UiaOperationAbstraction/UiaOperationAbstraction.h'), - MSUIA_sourceDir.File('UiaOperationAbstraction/UiaTypeAbstractionEnums.g.h'), - MSUIA_sourceDir.File('UiaOperationAbstraction/UiaTypeAbstraction.g.h'), - MSUIA_sourceDir.File('UiaOperationAbstraction/SafeArrayUtil.h'), - ] -) - -env.Depends(MSUIA_abstraction_headers,[MSUIA_wrapper_header,MSUIA_wrapper_impl]) - -Return(['MSUIA_lib_outDir','MSUIA_include_outDir']) diff --git a/nvdaHelper/readme.md b/nvdaHelper/readme.md index 4d6ca74e253..97e7ea907e7 100644 --- a/nvdaHelper/readme.md +++ b/nvdaHelper/readme.md @@ -43,11 +43,6 @@ It provides the following features: This dll is loaded by NVDA, providing utility functions that perform certain tasks or batch procedures on Microsoft UI Automation elements. It makes use of the UI Automation Remote Operations capabilities in Windows 11, allowing to declaratively define code to access and manipulate UI Automation elements, that will be Just-In-Time compiled by Windows and executed in the process providing the UI Automation elements. -##### microsoft-ui-uiAutomation remote ops library -As a dependency of UIARemote.dll, the open source [Microsoft-UI-UIAutomation Remote Operations library](https://github.com/microsoft/microsoft-ui-uiautomation) is also built. -This library contains both a low-level winrt API, and a higher-level pure C++ API which depends on the lower-level winrt API. UIARemote.dll tends to use the higher-level Operations abstraction API for its work. -In order for the winrt API to be available, UIARemote must register it with the Windows winrt platform. this involves loading a manifest file (See `microsoft-ui-uiautomation/Microsoft.UI.UIAutomation.dll.manifest`) and activating an activation context. - ### Configuring Visual Studio The following steps won't prepare a buildable solution, but it will enable intellisense. You should still build on the command line to verify errors. diff --git a/projectDocs/dev/createDevEnvironment.md b/projectDocs/dev/createDevEnvironment.md index 22c2652ec5f..57de4da07b2 100644 --- a/projectDocs/dev/createDevEnvironment.md +++ b/projectDocs/dev/createDevEnvironment.md @@ -85,9 +85,6 @@ For reference, the following run time dependencies are included in Git submodule * [Nullsoft Install System](https://nsis.sourceforge.io), version 3.08 * [Java Access Bridge 32 bit, from Zulu Community OpenJDK build 17.0.9+8Zulu (17.46.19)](https://github.com/nvaccess/javaAccessBridge32-bin) * [wil](https://github.com/microsoft/wil/) -* [Microsoft UI Automation Remote Operations Library, forked from @microsoft by @michaeldcurran](https://www.github.com/michaeldcurran/microsoft-ui-uiautomation/) - * Commit 224b22f3bf9e - * The fork specifically adds support for CallExtension / IsExtensionSupported to the high-level API, see pr microsoft/microsoft-ui-uiautomation#84 and #95. * [NVDA DiffMatchPatch](https://github.com/codeofdusk/nvda_dmp) Additionally, the following build time dependencies are included in the miscDeps git submodule: diff --git a/source/NVDAObjects/UIA/wordDocument.py b/source/NVDAObjects/UIA/wordDocument.py index 3f0b197752a..30aef221723 100644 --- a/source/NVDAObjects/UIA/wordDocument.py +++ b/source/NVDAObjects/UIA/wordDocument.py @@ -426,7 +426,7 @@ def _getFormatFieldAtRange(self, textRange, formatConfig, ignoreMixedValues=Fals formatField = super()._getFormatFieldAtRange(textRange, formatConfig, ignoreMixedValues=ignoreMixedValues) if not formatField: return formatField - if winVersion.getWinVer() >= winVersion.WIN11: + if UIARemote.isSupported(): docElement = self.obj.UIAElement if formatConfig['reportLineNumber']: lineNumber = UIARemote.msWord_getCustomAttributeValue( diff --git a/source/UIAHandler/_remoteOps/__init__.py b/source/UIAHandler/_remoteOps/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/source/UIAHandler/_remoteOps/builder.py b/source/UIAHandler/_remoteOps/builder.py new file mode 100644 index 00000000000..0c966057db2 --- /dev/null +++ b/source/UIAHandler/_remoteOps/builder.py @@ -0,0 +1,299 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + + +from __future__ import annotations +from abc import ABCMeta, abstractproperty +from typing import ( + Self, + ClassVar, + Any, + Iterable +) +import ctypes +from ctypes import ( + _SimpleCData, + c_char, + c_long, + c_wchar +) +import enum +from dataclasses import dataclass +import weakref +import struct +import itertools +import contextlib +from . import lowLevel +from .lowLevel import OperandId + + +class _RemoteBase: + + _robRef: weakref.ReferenceType[RemoteOperationBuilder] | None = None + _mutable: bool = True + + @property + def rob(self) -> RemoteOperationBuilder: + if self._robRef is None: + raise RuntimeError("Object not bound yet") + rob = self._robRef() + if rob is None: + raise RuntimeError("Builder has died") + return rob + + def __init__(self, rob: RemoteOperationBuilder): + self._robRef = weakref.ref(rob) + + def isBound(self, toBuilder: RemoteOperationBuilder) -> bool: + if self._robRef is None: + return False + rob = self._robRef() + if rob is None: + raise RuntimeError("Builder has died") + if rob is not toBuilder: + raise RuntimeError("Builder mismatch") + return True + + def __eq__(self, other: object) -> bool: + if not isinstance(other, type(self)): + return False + rob = self._robRef() if self._robRef is not None else None + otherBuilder = other._robRef() if other._robRef is not None else None + if rob != otherBuilder: + return False + if self._mutable != other._mutable: + return False + return True + + +class Operand(_RemoteBase): + + _operandId: OperandId | None = None + _sectionForInitInstructions: str | None = None + _defaultSectionForInitInstructions: str = "main" + + def __init__(self, rob: RemoteOperationBuilder, operandId: OperandId): + super().__init__(rob) + self._operandId = operandId + + @property + def operandId(self) -> OperandId: + if self._operandId is None: + raise RuntimeError("Object not bound yet") + return self._operandId + + @property + def sectionForInitInstructions(self) -> str: + return self._sectionForInitInstructions or self._defaultSectionForInitInstructions + + def __eq__(self, other: Self | object) -> bool: + if super().__eq__(other) is False: + return False + if type(other) is not Operand: + return False + if self._operandId != other._operandId: + return False + return True + + def __repr__(self) -> str: + output = "" + if self._sectionForInitInstructions == "static": + output += "static " + if not self._mutable: + output += "const " + output += f"{self.__class__.__name__} at {self.operandId}" + return output + + +class InstructionBase(metaclass=ABCMeta): + opCode: ClassVar[lowLevel.InstructionType] + + @abstractproperty + def params(self) -> dict[str, Any]: + raise NotImplementedError() + + def getByteCode(self) -> bytes: + byteCode = struct.pack('l', self.opCode.value) + params = list(self.params.values()) + if len(params) > 0 and isinstance(params[-1], list): + # If the last parameter is a list, it is a variable length parameter. + params[-1:] = params[-1] + for param in params: + if isinstance(param, enum.IntEnum): + param = c_long(param.value) + elif isinstance(param, Operand): + param = param.operandId + paramBytes = (c_char * ctypes.sizeof(param)).from_address(ctypes.addressof(param)).raw + byteCode += paramBytes + return byteCode + + def dumpInstruction(self) -> str: + output = f"{self.opCode.name}" + paramOutputList = [] + for paramName, param in self.params.items(): + paramOutput = f"{paramName}=" + if isinstance(param, ctypes.Array) and param._type_ == c_wchar: + paramOutput += f"c_wchar_array({repr(param.value)})" + else: + paramOutput += f"{repr(param)}" + paramOutputList.append(paramOutput) + output += "(" + ", ".join(paramOutputList) + ")" + return output + + def localExecute(self, registers: dict[OperandId, object]): + raise NotImplementedError() + + +@dataclass +class GenericInstruction(InstructionBase): + opCode: lowLevel.InstructionType + _params: dict[str, Operand | _SimpleCData | ctypes.Array | ctypes.Structure] + + def __init__( + self, + opCode: lowLevel.InstructionType, + **kwargs: Operand | _SimpleCData | ctypes.Array | ctypes.Structure + ): + self.opCode = opCode + self._params = kwargs + + @property + def params(self) -> dict[str, Operand | _SimpleCData | ctypes.Array | ctypes.Structure]: + return self._params + + +class InstructionList: + + _all: list[InstructionBase | str] + _instructions: list[InstructionBase] + _modified = False + _byteCodeCache: bytes | None = None + + def __init__(self): + super().__init__() + self._all = [] + self._instructions = [] + + def _addItem(self, item: InstructionBase | str): + self._all.append(item) + if isinstance(item, InstructionBase): + self._instructions.append(item) + self._modified = True + + def addComment(self, comment: str): + self._addItem(f"# {comment}") + + def addInstruction(self, instruction: InstructionBase) -> int: + self._addItem(instruction) + return self.getInstructionCount() - 1 + + def addGenericInstruction( + self, + opCode: lowLevel.InstructionType, + **params: Operand | _SimpleCData | ctypes.Array | ctypes.Structure, + ) -> int: + return self.addInstruction(GenericInstruction(opCode, **params)) + + def addMetaCommand(self, command: str): + self._addItem(f"[{command}]") + + def getByteCode(self) -> bytes: + if self._byteCodeCache is not None and not self._modified: + return self._byteCodeCache + byteCode = b'' + for instruction in self._instructions: + byteCode += instruction.getByteCode() + self._byteCodeCache = byteCode + self._modified = False + return byteCode + + def getInstruction(self, index) -> InstructionBase: + return self._instructions[index] + + def getInstructionCount(self) -> int: + return len(self._instructions) + + def iterItems(self) -> Iterable[InstructionBase | str]: + return iter(self._all) + + def dumpInstructions(self) -> str: + output = "" + for item in self.iterItems(): + if isinstance(item, InstructionBase): + output += item.dumpInstruction() + elif isinstance(item, str): + output += item + output += "\n" + return output + + def clear(self): + self._all.clear() + self._instructions.clear() + self._modified = False + self._byteCodeCache = None + + +class RemoteOperationBuilder: + + _versionBytes: bytes = struct.pack('l', 0) + _sectionNames = ["static", "const", "main"] + _lastOperandIdRequested = OperandId(1) + _defaultSection: str = "main" + + @property + def lastOperandIdRequested(self) -> OperandId: + return self._lastOperandIdRequested + + def __init__(self): + self._instructionListBySection: dict[str, InstructionList] = { + sectionName: InstructionList() for sectionName in self._sectionNames + } + self._remotedArgCache: dict[object, Operand] = {} + self.operandIdGen = itertools.count(start=1) + self._results = None + + def requestNewOperandId(self) -> OperandId: + operandID = self.lastOperandIdRequested + self._lastOperandIdRequested = OperandId(operandID.value + 1) + return operandID + + def getInstructionList(self, section) -> InstructionList: + return self._instructionListBySection[section] + + def getDefaultInstructionList(self) -> InstructionList: + return self.getInstructionList(self._defaultSection) + + @contextlib.contextmanager + def overrideDefaultSection(self, section: str): + oldDefaultSection = self._defaultSection + self._defaultSection = section + yield + self._defaultSection = oldDefaultSection + + def getAllInstructions(self) -> list[InstructionBase]: + return list(itertools.chain.from_iterable( + instructionList._instructions for instructionList in self._instructionListBySection.values() + )) + + def getByteCode(self) -> bytes: + byteCode = self._versionBytes + for sectionName in self._sectionNames: + byteCode += self._instructionListBySection[sectionName].getByteCode() + return byteCode + + def dumpInstructions(self) -> str: + output = "" + globalInstructionIndex = 0 + for sectionName, instructionList in self._instructionListBySection.items(): + output += f"{sectionName}:\n" + for item in instructionList.iterItems(): + if isinstance(item, InstructionBase): + output += f"{globalInstructionIndex}: " + globalInstructionIndex += 1 + output += item.dumpInstruction() + elif isinstance(item, str): + output += item + output += "\n" + return output diff --git a/source/UIAHandler/_remoteOps/instructions/__init__.py b/source/UIAHandler/_remoteOps/instructions/__init__.py new file mode 100644 index 00000000000..1ac80713240 --- /dev/null +++ b/source/UIAHandler/_remoteOps/instructions/__init__.py @@ -0,0 +1,111 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +""" +This package contains all the instructions that can be executed by the remote ops framework. +Each instruction contains the appropriate op code and parameter types. +Most instructions also contain a `localExecute` method, +which provides an implementation of the instruction that can be executed locally. +""" + + +# Import all instructions so that they can be accessed as attributes of this module. +# flake8: noqa: F401 + + +from ..builder import InstructionBase +from .arithmetic import ( + BinaryAdd, + BinarySubtract, + BinaryMultiply, + BinaryDivide, + InplaceAdd, + InplaceSubtract, + InplaceMultiply, + InplaceDivide, +) +from .array import ( + NewArray, + IsArray, + ArrayAppend, + ArrayGetAt, + ArrayRemoveAt, + ArraySetAt, + ArraySize, +) +from .bool import ( + NewBool, + IsBool, + BoolNot, + BoolAnd, + BoolOr, +) +from .controlFlow import ( + Halt, + Fork, + ForkIfFalse, + NewLoopBlock, + EndLoopBlock, + NewTryBlock, + EndTryBlock, + BreakLoop, + ContinueLoop, +) +from .element import ( + IsElement, + ElementGetPropertyValue, + ElementNavigate, +) +from .extension import ( + IsExtensionSupported, + CallExtension, +) +from .float import ( + NewFloat, + IsFloat, +) +from .general import ( + Set, + Compare, +) +from .guid import ( + NewGuid, + IsGuid, +) +from .int import ( + NewInt, + IsInt, + NewUint, + IsUint, +) +from .null import ( + NewNull, + IsNull, +) +from .status import ( + SetOperationStatus, + GetOperationStatus, +) +from .string import ( + NewString, + IsString, + StringConcat, + Stringify, +) +from .textRange import ( + TextRangeGetText, + TextRangeMove, + TextRangeMoveEndpointByUnit, + TextRangeCompare, + TextRangeClone, + TextRangeFindAttribute, + TextRangeFindText, + TextRangeGetAttributeValue, + TextRangeGetBoundingRectangles, + TextRangeGetEnclosingElement, + TextRangeExpandToEnclosingUnit, + TextRangeMoveEndpointByRange, + TextRangeCompareEndpoints, +) diff --git a/source/UIAHandler/_remoteOps/instructions/_base.py b/source/UIAHandler/_remoteOps/instructions/_base.py new file mode 100644 index 00000000000..b33b2061466 --- /dev/null +++ b/source/UIAHandler/_remoteOps/instructions/_base.py @@ -0,0 +1,17 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + + +from __future__ import annotations +from ..builder import ( + InstructionBase, +) + + +class _TypedInstruction(InstructionBase): + + @property + def params(self) -> dict[str, object]: + return vars(self) diff --git a/source/UIAHandler/_remoteOps/instructions/arithmetic.py b/source/UIAHandler/_remoteOps/instructions/arithmetic.py new file mode 100644 index 00000000000..24e06af412a --- /dev/null +++ b/source/UIAHandler/_remoteOps/instructions/arithmetic.py @@ -0,0 +1,110 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +""" +This module contains the instructions that perform arithmetic operations, +such as addition, subtraction, multiplication, and division. +Both binary and in-place operations are supported. +""" +from __future__ import annotations +from dataclasses import dataclass +from .. import lowLevel +from .. import builder +from ._base import _TypedInstruction + + +@dataclass +class BinaryAdd(_TypedInstruction): + opCode = lowLevel.InstructionType.BinaryAdd + result: builder.Operand + left: builder.Operand + right: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = registers[self.left.operandId] + registers[self.right.operandId] + + +@dataclass +class BinarySubtract(_TypedInstruction): + opCode = lowLevel.InstructionType.BinarySubtract + result: builder.Operand + left: builder.Operand + right: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = registers[self.left.operandId] - registers[self.right.operandId] + + +@dataclass +class BinaryMultiply(_TypedInstruction): + opCode = lowLevel.InstructionType.BinaryMultiply + result: builder.Operand + left: builder.Operand + right: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = registers[self.left.operandId] * registers[self.right.operandId] + + +@dataclass +class BinaryDivide(_TypedInstruction): + opCode = lowLevel.InstructionType.BinaryDivide + result: builder.Operand + left: builder.Operand + right: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + left = registers[self.left.operandId] + right = registers[self.right.operandId] + if isinstance(left, int) and isinstance(right, int): + result = left // right + else: + result = left / right + registers[self.result.operandId] = result + + +@dataclass +class InplaceAdd(_TypedInstruction): + opCode = lowLevel.InstructionType.Add + target: builder.Operand + value: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.target.operandId] += registers[self.value.operandId] + + +@dataclass +class InplaceSubtract(_TypedInstruction): + opCode = lowLevel.InstructionType.Subtract + target: builder.Operand + value: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.target.operandId] -= registers[self.value.operandId] + + +@dataclass +class InplaceMultiply(_TypedInstruction): + opCode = lowLevel.InstructionType.Multiply + target: builder.Operand + value: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.target.operandId] *= registers[self.value.operandId] + + +@dataclass +class InplaceDivide(_TypedInstruction): + opCode = lowLevel.InstructionType.Divide + target: builder.Operand + value: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + target = registers[self.target.operandId] + value = registers[self.value.operandId] + if isinstance(target, int) and isinstance(value, int): + registers[self.target.operandId] //= value + else: + registers[self.target.operandId] /= value diff --git a/source/UIAHandler/_remoteOps/instructions/array.py b/source/UIAHandler/_remoteOps/instructions/array.py new file mode 100644 index 00000000000..05173e87dc0 --- /dev/null +++ b/source/UIAHandler/_remoteOps/instructions/array.py @@ -0,0 +1,93 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +""" +This module contains the instructions that operate on arrays. +Including to create new arrays, append, get, set, and remove elements from arrays, +and check if an object is an array. +""" + +from __future__ import annotations +from typing import cast +from dataclasses import dataclass +from .. import lowLevel +from .. import builder +from ._base import _TypedInstruction + + +@dataclass +class NewArray(_TypedInstruction): + opCode = lowLevel.InstructionType.NewArray + result: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = [] + + +@dataclass +class IsArray(_TypedInstruction): + opCode = lowLevel.InstructionType.IsArray + result: builder.Operand + target: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = isinstance(registers[self.target.operandId], list) + + +@dataclass +class ArrayAppend(_TypedInstruction): + opCode = lowLevel.InstructionType.RemoteArrayAppend + target: builder.Operand + value: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + array = cast(list, registers[self.target.operandId]) + array.append(registers[self.value.operandId]) + + +@dataclass +class ArrayGetAt(_TypedInstruction): + opCode = lowLevel.InstructionType.RemoteArrayGetAt + result: builder.Operand + target: builder.Operand + index: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + array = cast(list, registers[self.target.operandId]) + registers[self.result.operandId] = array[cast(int, registers[self.index.operandId])] + + +@dataclass +class ArrayRemoveAt(_TypedInstruction): + opCode = lowLevel.InstructionType.RemoteArrayRemoveAt + target: builder.Operand + index: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + array = cast(list, registers[self.target.operandId]) + del array[cast(int, registers[self.index.operandId])] + + +@dataclass +class ArraySetAt(_TypedInstruction): + opCode = lowLevel.InstructionType.RemoteArraySetAt + target: builder.Operand + index: builder.Operand + value: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + array = cast(list, registers[self.target.operandId]) + array[cast(int, registers[self.index.operandId])] = registers[self.value.operandId] + + +@dataclass +class ArraySize(_TypedInstruction): + opCode = lowLevel.InstructionType.RemoteArraySize + result: builder.Operand + target: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + array = cast(list, registers[self.target.operandId]) + registers[self.result.operandId] = len(array) diff --git a/source/UIAHandler/_remoteOps/instructions/bool.py b/source/UIAHandler/_remoteOps/instructions/bool.py new file mode 100644 index 00000000000..b7d9ce44130 --- /dev/null +++ b/source/UIAHandler/_remoteOps/instructions/bool.py @@ -0,0 +1,70 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +""" +This module contains the instructions that operate on boolean values. +Including to create new boolean values, check if an object is a boolean, +and perform boolean operations such as and, or and not. +""" + + +from __future__ import annotations +from dataclasses import dataclass +import ctypes +from .. import lowLevel +from .. import builder +from ._base import _TypedInstruction + + +@dataclass +class NewBool(_TypedInstruction): + opCode = lowLevel.InstructionType.NewBool + result: builder.Operand + value: ctypes.c_bool + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = self.value.value + + +@dataclass +class IsBool(_TypedInstruction): + opCode = lowLevel.InstructionType.IsBool + result: builder.Operand + target: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = isinstance(registers[self.target.operandId], bool) + + +@dataclass +class BoolNot(_TypedInstruction): + opCode = lowLevel.InstructionType.BoolNot + result: builder.Operand + target: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = not registers[self.target.operandId] + + +@dataclass +class BoolAnd(_TypedInstruction): + opCode = lowLevel.InstructionType.BoolAnd + result: builder.Operand + left: builder.Operand + right: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = registers[self.left.operandId] and registers[self.right.operandId] + + +@dataclass +class BoolOr(_TypedInstruction): + opCode = lowLevel.InstructionType.BoolOr + result: builder.Operand + left: builder.Operand + right: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = registers[self.left.operandId] or registers[self.right.operandId] diff --git a/source/UIAHandler/_remoteOps/instructions/controlFlow.py b/source/UIAHandler/_remoteOps/instructions/controlFlow.py new file mode 100644 index 00000000000..650254b4bcb --- /dev/null +++ b/source/UIAHandler/_remoteOps/instructions/controlFlow.py @@ -0,0 +1,67 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +""" +This module contains the instructions that control the flow of execution. +Including to halt execution, fork execution, and manage loops and try blocks. +""" + + +from __future__ import annotations +from dataclasses import dataclass +from .. import lowLevel +from .. import builder +from ._base import _TypedInstruction + + +@dataclass +class Halt(_TypedInstruction): + opCode = lowLevel.InstructionType.Halt + + +@dataclass +class Fork(_TypedInstruction): + opCode = lowLevel.InstructionType.Fork + jumpTo: lowLevel.RelativeOffset + + +@dataclass +class ForkIfFalse(_TypedInstruction): + opCode = lowLevel.InstructionType.ForkIfFalse + condition: builder.Operand + branch: lowLevel.RelativeOffset + + +@dataclass +class NewLoopBlock(_TypedInstruction): + opCode = lowLevel.InstructionType.NewLoopBlock + breakBranch: lowLevel.RelativeOffset + continueBranch: lowLevel.RelativeOffset + + +@dataclass +class EndLoopBlock(_TypedInstruction): + opCode = lowLevel.InstructionType.EndLoopBlock + + +@dataclass +class NewTryBlock(_TypedInstruction): + opCode = lowLevel.InstructionType.NewTryBlock + catchBranch: lowLevel.RelativeOffset + + +@dataclass +class EndTryBlock(_TypedInstruction): + opCode = lowLevel.InstructionType.EndTryBlock + + +@dataclass +class BreakLoop(_TypedInstruction): + opCode = lowLevel.InstructionType.BreakLoop + + +@dataclass +class ContinueLoop(_TypedInstruction): + opCode = lowLevel.InstructionType.ContinueLoop diff --git a/source/UIAHandler/_remoteOps/instructions/element.py b/source/UIAHandler/_remoteOps/instructions/element.py new file mode 100644 index 00000000000..e94e34cf51b --- /dev/null +++ b/source/UIAHandler/_remoteOps/instructions/element.py @@ -0,0 +1,79 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +""" +This module contains the instructions that operate on UI Automation elements. +Including to check if an object is an element, +get a property value of an element, +and navigate the UI Automation tree. +""" + + +from __future__ import annotations +from typing import cast +from dataclasses import dataclass +from ctypes import POINTER +import UIAHandler +from UIAHandler import UIA +from .. import lowLevel +from .. import builder +from ._base import _TypedInstruction + + +@dataclass +class IsElement(_TypedInstruction): + opCode = lowLevel.InstructionType.IsElement + result: builder.Operand + target: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = isinstance( + registers[self.target.operandId], POINTER(UIA.IUIAutomationElement) + ) + + +@dataclass +class ElementGetPropertyValue(_TypedInstruction): + opCode = lowLevel.InstructionType.GetPropertyValue + result: builder.Operand + target: builder.Operand + propertyId: builder.Operand + ignoreDefault: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + element = cast(UIA.IUIAutomationElement, registers[self.target.operandId]) + propertyId = cast(int, registers[self.propertyId.operandId]) + ignoreDefault = cast(bool, registers[self.ignoreDefault.operandId]) + value = element.GetCurrentPropertyValueEx(propertyId, ignoreDefault) + registers[self.result.operandId] = value + + +@dataclass +class ElementNavigate(_TypedInstruction): + opCode = lowLevel.InstructionType.Navigate + result: builder.Operand + target: builder.Operand + direction: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + element = cast(UIA.IUIAutomationElement, registers[self.target.operandId]) + if not UIAHandler.handler: + raise RuntimeError("UIAHandler not initialized") + client = cast(UIA.IUIAutomation, UIAHandler.handler.clientObject) + treeWalker = client.RawViewWalker + direction = cast(lowLevel.NavigationDirection, registers[self.direction.operandId]) + match direction: + case lowLevel.NavigationDirection.Parent: + registers[self.result.operandId] = treeWalker.GetParentElement(element) + case lowLevel.NavigationDirection.FirstChild: + registers[self.result.operandId] = treeWalker.GetFirstChildElement(element) + case lowLevel.NavigationDirection.LastChild: + registers[self.result.operandId] = treeWalker.GetLastChildElement(element) + case lowLevel.NavigationDirection.NextSibling: + registers[self.result.operandId] = treeWalker.GetNextSiblingElement(element) + case lowLevel.NavigationDirection.PreviousSibling: + registers[self.result.operandId] = treeWalker.GetPreviousSiblingElement(element) + case _: + raise ValueError(f"Unknown navigation direction {direction}") diff --git a/source/UIAHandler/_remoteOps/instructions/extension.py b/source/UIAHandler/_remoteOps/instructions/extension.py new file mode 100644 index 00000000000..dd788899f2d --- /dev/null +++ b/source/UIAHandler/_remoteOps/instructions/extension.py @@ -0,0 +1,39 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +""" +This module contains the instructions that check for and call UI Automation custom extensions. +""" + + +from __future__ import annotations +from dataclasses import dataclass +import ctypes +from .. import lowLevel +from .. import builder +from ._base import _TypedInstruction + + +@dataclass +class IsExtensionSupported(_TypedInstruction): + opCode = lowLevel.InstructionType.IsExtensionSupported + result: builder.Operand + target: builder.Operand + extensionId: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + raise NotImplementedError("Extension support is not implemented") + + +@dataclass +class CallExtension(_TypedInstruction): + opCode = lowLevel.InstructionType.CallExtension + target: builder.Operand + extensionId: builder.Operand + argCount: ctypes.c_ulong + arguments: list[builder.Operand] + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + raise NotImplementedError("Extension support is not implemented") diff --git a/source/UIAHandler/_remoteOps/instructions/float.py b/source/UIAHandler/_remoteOps/instructions/float.py new file mode 100644 index 00000000000..5705b7c3e0a --- /dev/null +++ b/source/UIAHandler/_remoteOps/instructions/float.py @@ -0,0 +1,37 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +""" +This module contains the instructions that operate on floating point numbers. +Including to create new floating point numbers, and check if an object is a floating point number. +""" + + +from __future__ import annotations +from dataclasses import dataclass +import ctypes +from .. import lowLevel +from .. import builder +from ._base import _TypedInstruction + + +@dataclass +class NewFloat(_TypedInstruction): + opCode = lowLevel.InstructionType.NewDouble + result: builder.Operand + value: ctypes.c_double + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = self.value.value + + +@dataclass +class IsFloat(_TypedInstruction): + opCode = lowLevel.InstructionType.IsDouble + result: builder.Operand + target: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = isinstance(registers[self.target.operandId], float) diff --git a/source/UIAHandler/_remoteOps/instructions/general.py b/source/UIAHandler/_remoteOps/instructions/general.py new file mode 100644 index 00000000000..d945769f905 --- /dev/null +++ b/source/UIAHandler/_remoteOps/instructions/general.py @@ -0,0 +1,56 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +""" +This module contains the instructions that operate on all object types. +Including to set a value, or compare two values. +""" + + +from __future__ import annotations +from dataclasses import dataclass +from .. import lowLevel +from .. import builder +from ._base import _TypedInstruction + + +@dataclass +class Set(_TypedInstruction): + opCode = lowLevel.InstructionType.Set + target: builder.Operand + value: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + value = registers[self.value.operandId] + registers[self.target.operandId] = value + + +@dataclass +class Compare(_TypedInstruction): + opCode = lowLevel.InstructionType.Compare + result: builder.Operand + left: builder.Operand + right: builder.Operand + comparisonType: lowLevel.ComparisonType + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + localLeft = registers[self.left.operandId] + localRight = registers[self.right.operandId] + match self.comparisonType: + case lowLevel.ComparisonType.Equal: + localResult = (localLeft == localRight) + case lowLevel.ComparisonType.NotEqual: + localResult = (localLeft != localRight) + case lowLevel.ComparisonType.LessThan: + localResult = (localLeft < localRight) + case lowLevel.ComparisonType.LessThanOrEqual: + localResult = (localLeft <= localRight) + case lowLevel.ComparisonType.GreaterThan: + localResult = (localLeft > localRight) + case lowLevel.ComparisonType.GreaterThanOrEqual: + localResult = (localLeft >= localRight) + case _: + raise NotImplementedError(f"Unknown comparison type {self.comparisonType}") + registers[self.result.operandId] = localResult diff --git a/source/UIAHandler/_remoteOps/instructions/guid.py b/source/UIAHandler/_remoteOps/instructions/guid.py new file mode 100644 index 00000000000..404859176f7 --- /dev/null +++ b/source/UIAHandler/_remoteOps/instructions/guid.py @@ -0,0 +1,37 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +""" +This module contains the instructions that operate on GUID values. +Including to create new GUID values, and check if an object is a GUID. +""" + + +from __future__ import annotations +from dataclasses import dataclass +from comtypes import GUID +from .. import lowLevel +from .. import builder +from ._base import _TypedInstruction + + +@dataclass +class NewGuid(_TypedInstruction): + opCode = lowLevel.InstructionType.NewGuid + result: builder.Operand + value: GUID + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = self.value + + +@dataclass +class IsGuid(_TypedInstruction): + opCode = lowLevel.InstructionType.IsGuid + result: builder.Operand + target: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = isinstance(registers[self.target.operandId], GUID) diff --git a/source/UIAHandler/_remoteOps/instructions/int.py b/source/UIAHandler/_remoteOps/instructions/int.py new file mode 100644 index 00000000000..2b77da5c3bc --- /dev/null +++ b/source/UIAHandler/_remoteOps/instructions/int.py @@ -0,0 +1,59 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +""" +This module contains the instructions that operate on integers. +Including to create new integers, and check if an object is an integer. +Both signed and unsigned integers are supported. +""" + + +from __future__ import annotations +from dataclasses import dataclass +import ctypes +from .. import lowLevel +from .. import builder +from ._base import _TypedInstruction + + +@dataclass +class NewInt(_TypedInstruction): + opCode = lowLevel.InstructionType.NewInt + result: builder.Operand + value: ctypes.c_long + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = self.value.value + + +@dataclass +class IsInt(_TypedInstruction): + opCode = lowLevel.InstructionType.IsInt + result: builder.Operand + target: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = isinstance(registers[self.target.operandId], int) + + +@dataclass +class NewUint(_TypedInstruction): + opCode = lowLevel.InstructionType.NewUint + result: builder.Operand + value: ctypes.c_ulong + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = self.value.value + + +@dataclass +class IsUint(_TypedInstruction): + opCode = lowLevel.InstructionType.IsUint + result: builder.Operand + target: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + val = registers[self.target.operandId] + registers[self.result.operandId] = isinstance(val, int) and val >= 0 diff --git a/source/UIAHandler/_remoteOps/instructions/null.py b/source/UIAHandler/_remoteOps/instructions/null.py new file mode 100644 index 00000000000..1b3be87c615 --- /dev/null +++ b/source/UIAHandler/_remoteOps/instructions/null.py @@ -0,0 +1,35 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +""" +This module contains the instructions that operate on null values. +Including to create new null values, and check if an object is null. +""" + + +from __future__ import annotations +from dataclasses import dataclass +from .. import lowLevel +from .. import builder +from ._base import _TypedInstruction + + +@dataclass +class NewNull(_TypedInstruction): + opCode = lowLevel.InstructionType.NewNull + result: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = None + + +@dataclass +class IsNull(_TypedInstruction): + opCode = lowLevel.InstructionType.IsNull + result: builder.Operand + target: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = registers[self.target.operandId] is None diff --git a/source/UIAHandler/_remoteOps/instructions/status.py b/source/UIAHandler/_remoteOps/instructions/status.py new file mode 100644 index 00000000000..c6803013511 --- /dev/null +++ b/source/UIAHandler/_remoteOps/instructions/status.py @@ -0,0 +1,27 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +""" +This module contains the instructions that operate on the status of operations. +""" + + +from __future__ import annotations +from dataclasses import dataclass +from .. import lowLevel +from .. import builder +from ._base import _TypedInstruction + + +@dataclass +class SetOperationStatus(_TypedInstruction): + opCode = lowLevel.InstructionType.SetOperationStatus + status: builder.Operand + + +@dataclass +class GetOperationStatus(_TypedInstruction): + opCode = lowLevel.InstructionType.GetOperationStatus + result: builder.Operand diff --git a/source/UIAHandler/_remoteOps/instructions/string.py b/source/UIAHandler/_remoteOps/instructions/string.py new file mode 100644 index 00000000000..c82163183f5 --- /dev/null +++ b/source/UIAHandler/_remoteOps/instructions/string.py @@ -0,0 +1,63 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +""" +This module contains the instructions that operate on strings. +Including to create new strings, check if an object is a string, and concatenate strings. +""" + + +from __future__ import annotations +from typing import cast +from dataclasses import dataclass +import ctypes +from .. import lowLevel +from .. import builder +from ._base import _TypedInstruction + + +@dataclass +class NewString(_TypedInstruction): + opCode = lowLevel.InstructionType.NewString + result: builder.Operand + length: ctypes.c_ulong + value: ctypes.Array[ctypes.c_wchar] + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = self.value.value + + +@dataclass +class IsString(_TypedInstruction): + opCode = lowLevel.InstructionType.IsString + result: builder.Operand + target: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = isinstance(registers[self.target.operandId], str) + + +@dataclass +class StringConcat(_TypedInstruction): + opCode = lowLevel.InstructionType.RemoteStringConcat + result: builder.Operand + left: builder.Operand + right: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + localLeft = cast(str, registers[self.left.operandId]) + localRight = cast(str, registers[self.right.operandId]) + localResult = localLeft + localRight + registers[self.result.operandId] = localResult + + +@dataclass +class Stringify(_TypedInstruction): + opCode = lowLevel.InstructionType.Stringify + result: builder.Operand + target: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + registers[self.result.operandId] = str(registers[self.target.operandId]) diff --git a/source/UIAHandler/_remoteOps/instructions/textRange.py b/source/UIAHandler/_remoteOps/instructions/textRange.py new file mode 100644 index 00000000000..f97929d077f --- /dev/null +++ b/source/UIAHandler/_remoteOps/instructions/textRange.py @@ -0,0 +1,200 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +""" +This module contains the instructions that operate on UI Automation text ranges. +""" + + +from __future__ import annotations +from typing import cast +from dataclasses import dataclass +from UIAHandler import UIA +from .. import lowLevel +from .. import builder +from ._base import _TypedInstruction + + +@dataclass +class TextRangeGetText(_TypedInstruction): + opCode = lowLevel.InstructionType.TextRangeGetText + result: builder.Operand + target: builder.Operand + maxLength: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + textRange = cast(UIA.IUIAutomationTextRange, registers[self.target.operandId]) + maxLength = cast(int, registers[self.maxLength.operandId]) + registers[self.result.operandId] = textRange.GetText(maxLength) + + +@dataclass +class TextRangeMove(_TypedInstruction): + opCode = lowLevel.InstructionType.TextRangeMove + result: builder.Operand + target: builder.Operand + unit: builder.Operand + count: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + textRange = cast(UIA.IUIAutomationTextRange, registers[self.target.operandId]) + unit = cast(int, registers[self.unit.operandId]) + count = cast(int, registers[self.count.operandId]) + registers[self.result.operandId] = textRange.Move(unit, count) + + +@dataclass +class TextRangeMoveEndpointByUnit(_TypedInstruction): + opCode = lowLevel.InstructionType.TextRangeMoveEndpointByUnit + result: builder.Operand + target: builder.Operand + endpoint: builder.Operand + unit: builder.Operand + count: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + textRange = cast(UIA.IUIAutomationTextRange, registers[self.target.operandId]) + endPoint = cast(int, registers[self.endpoint.operandId]) + unit = cast(int, registers[self.unit.operandId]) + count = cast(int, registers[self.count.operandId]) + registers[self.result.operandId] = textRange.MoveEndpointByUnit(endPoint, unit, count) + + +@dataclass +class TextRangeCompare(_TypedInstruction): + opCode = lowLevel.InstructionType.TextRangeCompare + result: builder.Operand + left: builder.Operand + right: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + left = cast(UIA.IUIAutomationTextRange, registers[self.left.operandId]) + right = cast(UIA.IUIAutomationTextRange, registers[self.right.operandId]) + registers[self.result.operandId] = left.Compare(right) + + +@dataclass +class TextRangeClone(_TypedInstruction): + opCode = lowLevel.InstructionType.TextRangeClone + result: builder.Operand + target: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + textRange = cast(UIA.IUIAutomationTextRange, registers[self.target.operandId]) + registers[self.result.operandId] = textRange.Clone() + + +@dataclass +class TextRangeFindAttribute(_TypedInstruction): + opCode = lowLevel.InstructionType.TextRangeFindAttribute + result: builder.Operand + target: builder.Operand + attributeId: builder.Operand + value: builder.Operand + reverse: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + textRange = cast(UIA.IUIAutomationTextRange, registers[self.target.operandId]) + attributeId = cast(int, registers[self.attributeId.operandId]) + value = cast(object, registers[self.value.operandId]) + reverse = cast(bool, registers[self.reverse.operandId]) + registers[self.result.operandId] = textRange.FindAttribute(attributeId, value, reverse) + + +@dataclass +class TextRangeFindText(_TypedInstruction): + opCode = lowLevel.InstructionType.TextRangeFindText + result: builder.Operand + target: builder.Operand + value: builder.Operand + reverse: builder.Operand + ignoreCase: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + textRange = cast(UIA.IUIAutomationTextRange, registers[self.target.operandId]) + value = cast(str, registers[self.value.operandId]) + reverse = cast(bool, registers[self.reverse.operandId]) + ignoreCase = cast(bool, registers[self.ignoreCase.operandId]) + registers[self.result.operandId] = textRange.FindText(value, reverse, ignoreCase) + + +@dataclass +class TextRangeGetAttributeValue(_TypedInstruction): + opCode = lowLevel.InstructionType.TextRangeGetAttributeValue + result: builder.Operand + target: builder.Operand + attributeId: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + textRange = cast(UIA.IUIAutomationTextRange, registers[self.target.operandId]) + attributeId = cast(int, registers[self.attributeId.operandId]) + registers[self.result.operandId] = textRange.GetAttributeValue(attributeId) + + +@dataclass +class TextRangeGetBoundingRectangles(_TypedInstruction): + opCode = lowLevel.InstructionType.TextRangeGetBoundingRectangles + result: builder.Operand + target: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + textRange = cast(UIA.IUIAutomationTextRange, registers[self.target.operandId]) + registers[self.result.operandId] = textRange.GetBoundingRectangles() + + +@dataclass +class TextRangeGetEnclosingElement(_TypedInstruction): + opCode = lowLevel.InstructionType.TextRangeGetEnclosingElement + result: builder.Operand + target: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + textRange = cast(UIA.IUIAutomationTextRange, registers[self.target.operandId]) + registers[self.result.operandId] = textRange.GetEnclosingElement() + + +@dataclass +class TextRangeExpandToEnclosingUnit(_TypedInstruction): + opCode = lowLevel.InstructionType.TextRangeExpandToEnclosingUnit + target: builder.Operand + unit: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + textRange = cast(UIA.IUIAutomationTextRange, registers[self.target.operandId]) + unit = cast(int, registers[self.unit.operandId]) + textRange.ExpandToEnclosingUnit(unit) + + +@dataclass +class TextRangeMoveEndpointByRange(_TypedInstruction): + opCode = lowLevel.InstructionType.TextRangeMoveEndpointByRange + target: builder.Operand + srcEndpoint: builder.Operand + otherRange: builder.Operand + otherEndpoint: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + textRange = cast(UIA.IUIAutomationTextRange, registers[self.target.operandId]) + srcEndpoint = cast(int, registers[self.srcEndpoint.operandId]) + otherRange = cast(UIA.IUIAutomationTextRange, registers[self.otherRange.operandId]) + otherEndpoint = cast(int, registers[self.otherEndpoint.operandId]) + textRange.MoveEndpointByRange(srcEndpoint, otherRange, otherEndpoint) + + +@dataclass +class TextRangeCompareEndpoints(_TypedInstruction): + opCode = lowLevel.InstructionType.TextRangeCompareEndpoints + result: builder.Operand + target: builder.Operand + thisEndpoint: builder.Operand + otherRange: builder.Operand + otherEndpoint: builder.Operand + + def localExecute(self, registers: dict[lowLevel.OperandId, object]): + textRange = cast(UIA.IUIAutomationTextRange, registers[self.target.operandId]) + thisEndpoint = cast(int, registers[self.thisEndpoint.operandId]) + otherRange = cast(UIA.IUIAutomationTextRange, registers[self.otherRange.operandId]) + otherEndpoint = cast(int, registers[self.otherEndpoint.operandId]) + registers[self.result.operandId] = textRange.CompareEndpoints(thisEndpoint, otherRange, otherEndpoint) diff --git a/source/UIAHandler/_remoteOps/localExecute.py b/source/UIAHandler/_remoteOps/localExecute.py new file mode 100644 index 00000000000..d2b629fbce8 --- /dev/null +++ b/source/UIAHandler/_remoteOps/localExecute.py @@ -0,0 +1,228 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +from __future__ import annotations +from typing import Type, cast +from dataclasses import dataclass +from comtypes import COMError +from . import lowLevel +from UIAHandler import UIA +from . import builder +from . import instructions +from . import operation + + +@dataclass +class LocalExecutionResult(operation.ExecutionResult): + results: dict[lowLevel.OperandId, object] + + def hasOperand(self, operandId: lowLevel.OperandId) -> bool: + return operandId in self.results + + def getOperand(self, operandId: lowLevel.OperandId) -> object: + return self.results[operandId] + + +class HaltException(Exception): + pass + + +class BreakLoopException(Exception): + pass + + +class BadOperationStatusException(Exception): + pass + + +class InstructionLimitExceededException(Exception): + pass + + +@dataclass +class LocalOperationResultSet: + _registers: dict[lowLevel.OperandId, object] + status: int + errorLocation: int + extendedError: int + + def hasOperand(self, operandId: lowLevel.OperandId) -> bool: + return operandId in self._registers + + def getOperand(self, operandId: lowLevel.OperandId) -> object: + return self._registers[operandId] + + +class LocalExecutor(operation.Executor): + _registers: dict[lowLevel.OperandId, object] + _requestedResults: set[lowLevel.OperandId] + _operationStatus: int = 0 + _instructions: list[builder.InstructionBase] + _ip: int + _instructionLoopDepth = 0 + _instructionCounter = 0 + _maxInstructions: int + + def __init__(self, maxInstructions: int = 10000): + self._maxInstructions = maxInstructions + self._registers = {} + self._requestedResults = set() + + @property + def operationStatus(self) -> int: + return self._operationStatus + + @operationStatus.setter + def operationStatus(self, value: int): + self._operationStatus = value + if value < 0: + raise BadOperationStatusException() + + def storeRegisterValue(self, operandId: lowLevel.OperandId, value: object): + self._registers[operandId] = value + + def fetchRegisterValue(self, operandId: lowLevel.OperandId) -> object: + return self._registers[operandId] + + def _operationStatusFromException(self, e: Exception) -> int: + if isinstance(e, COMError): + return e.hresult + elif isinstance(e, ZeroDivisionError): + return -805306220 + else: + return 0 + + def _execute_ForkIfFalse(self, instruction: instructions.ForkIfFalse): + condition = self._registers[instruction.condition.operandId] + if not isinstance(condition, bool): + raise RuntimeError(f"Expected bool, got {type(condition)}") + if not condition: + self._ip += instruction.branch.value + else: + self._ip += 1 + + def _execute_NewLoopBlock(self, instruction: instructions.NewLoopBlock): + breakAddress = self._ip + instruction.breakBranch.value + continueAddress = self._ip + instruction.continueBranch.value + self._ip += 1 + self._instructionLoop( + stopInstruction=instructions.EndLoopBlock, + breakAddress=breakAddress, + continueAddress=continueAddress + ) + + def _execute_NewTryBlock(self, instruction: instructions.NewTryBlock): + self._ip += 1 + catchAddress = self._ip + instruction.catchBranch.value + self._instructionLoop(instructions.EndTryBlock, catchAddress=catchAddress) + + def _execute_ContinueLoop(self, instruction: instructions.ContinueLoop, continueAddress: int | None): + if continueAddress is not None: + self._ip = continueAddress + else: + raise RuntimeError("ContinueLoop instruction encountered outside of loop") + + def _execute_SetOperationStatus(self, instruction: instructions.SetOperationStatus): + self.operationStatus = cast(int, self._registers[instruction.status.operandId]) + self._ip += 1 + + def _execute_GetOperationStatus(self, instruction: instructions.GetOperationStatus): + self._registers[instruction.result.operandId] = self.operationStatus + self._ip += 1 + + def _executeInstruction( + self, + instruction: builder.InstructionBase, + breakAddress: int | None = None, + continueAddress: int | None = None + ): + match instruction: + case instructions.Halt(): + raise HaltException() + case instructions.Fork(): + self._ip += instruction.jumpTo.value + case instructions.ForkIfFalse(): + self._execute_ForkIfFalse(instruction) + case instructions.NewLoopBlock(): + self._execute_NewLoopBlock(instruction) + case instructions.NewTryBlock(): + self._execute_NewTryBlock(instruction) + case instructions.BreakLoop(): + raise BreakLoopException() + case instructions.ContinueLoop(): + self._execute_ContinueLoop(instruction, continueAddress) + case instructions.SetOperationStatus(): + self._execute_SetOperationStatus(instruction) + case instructions.GetOperationStatus(): + self._execute_GetOperationStatus(instruction) + case _: + instruction.localExecute(self._registers) + self._ip += 1 + + def _instructionLoop( + self, + stopInstruction: Type[builder.InstructionBase] | None = None, + breakAddress: int | None = None, + continueAddress: int | None = None, + catchAddress: int | None = None + ): + self._instructionLoopDepth += 1 + try: + while self._ip < len(self._instructions): + instruction = self._instructions[self._ip] + self._instructionCounter += 1 + if self._maxInstructions is not None and self._instructionCounter > self._maxInstructions: + raise InstructionLimitExceededException() + if stopInstruction is not None and type(instruction) is stopInstruction: + self._ip += 1 + break + try: + self._executeInstruction(instruction, breakAddress, continueAddress) + except Exception as e: + self.operationStatus = self._operationStatusFromException(e) + raise + except BreakLoopException: + if breakAddress is not None: + self._ip = breakAddress + else: + raise RuntimeError("BreakLoop instruction encountered outside of loop") + except BadOperationStatusException: + if catchAddress is not None: + self._ip = catchAddress + else: + raise + finally: + self._instructionLoopDepth -= 1 + + def importElement(self, operandId: lowLevel.OperandId, element: UIA.IUIAutomationElement): + self._registers[operandId] = element + + def importTextRange(self, operandId: lowLevel.OperandId, textRange: UIA.IUIAutomationTextRange): + self._registers[operandId] = textRange + + def addToResults(self, operandId: lowLevel.OperandId): + self._requestedResults.add(operandId) + + def loadInstructions(self, rob: builder.RemoteOperationBuilder): + self._instructions = rob.getAllInstructions() + + def execute(self) -> LocalExecutionResult: + self._ip = 0 + self._instructionCounter = 0 + status = lowLevel.RemoteOperationStatus.Success + try: + self._instructionLoop() + except HaltException: + pass + except InstructionLimitExceededException: + status = lowLevel.RemoteOperationStatus.InstructionLimitExceeded + except BadOperationStatusException: + status = lowLevel.RemoteOperationStatus.UnhandledException + return LocalExecutionResult( + results={k: v for k, v in self._registers.items() if k in self._requestedResults}, + status=status, + errorLocation=self._ip, + extendedError=self._operationStatus + ) diff --git a/source/UIAHandler/_remoteOps/lowLevel.py b/source/UIAHandler/_remoteOps/lowLevel.py new file mode 100644 index 00000000000..effa9524189 --- /dev/null +++ b/source/UIAHandler/_remoteOps/lowLevel.py @@ -0,0 +1,376 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +from __future__ import annotations +from ctypes import ( + oledll, + byref, + c_void_p, + c_long, + c_ulong, + c_bool +) +from comtypes.automation import VARIANT +import os +import enum +from UIAHandler import UIA +import NVDAHelper + + +""" +This module contains classes and constants for the low-level Windows UI Automation Remote Operations API, +i.e. Windows.UI.UIAutomation.core. +The low-level UI Automation Remote Operations API is a binary API +that allows for the execution of special byte code specific to UI Automation, +allowing for the execution of multiple UI Automation operations in a remote provider, +via one cross-process call. +""" + + +class OperandId(c_ulong): + """ + An operand ID is a unique identifier for an operand (or register) in the remote operation VM. + It is an unsigned 32 bit integer. + """ + + def __eq__(self, other: object) -> bool: + if type(other) is OperandId: + return self.value == other.value + return False + + def __repr__(self) -> str: + return f"OperandId {self.value}" + + def __hash__(self) -> int: + return hash(self.value) + + +class RelativeOffset(c_long): + """ + A relative offset is a signed 32 bit integer that represents an offset from the current instruction pointer. + """ + + def __repr__(self) -> str: + return f"RelativeOffset {self.value}" + + +_dll = oledll[os.path.join(NVDAHelper.versionedLibPath, "UIARemote.dll")] + + +class RemoteOperationResultSet: + """ + Wraps a Windows.UI.UIAutomation.Core.AutomationRemoteOperationResultSet. + """ + + def __init__(self, pResults: c_void_p): + if not pResults or not isinstance(pResults, c_void_p): + raise RuntimeError("Invalid results pointer") + self._pResults = pResults + + @property + def errorLocation(self) -> int: + """The index of the instruction where the error occurred.""" + val = c_long() + _dll.remoteOpResult_getErrorLocation(self._pResults, byref(val)) + return val.value + + @property + def extendedError(self) -> int: + """ + The error HRESULT produced by the instruction that caused the error. + """ + val = c_long() + _dll.remoteOpResult_getExtendedError(self._pResults, byref(val)) + return val.value + + @property + def status(self) -> RemoteOperationStatus: + """ + The status of the remote operation. + E.g. success, malformed bytecode, etc. + """ + val = c_long() + _dll.remoteOpResult_getStatus(self._pResults, byref(val)) + return RemoteOperationStatus(val.value) + + def hasOperand(self, operandId: OperandId) -> bool: + """ + Returns true if the result set contains an operand with the given ID. + I.e. The operand was requested as a result before execution, + and the remote operation successfully produced a value for it. + """ + val = c_bool() + _dll.remoteOpResult_hasOperand(self._pResults, operandId, byref(val)) + return val.value + + def getOperand(self, operandId: OperandId) -> object: + """ + Returns the value of the operand with the given ID. + In order to succeed, + the operand must have been requested as a result before execution, + and the remote operation must have successfully produced a value for it. + """ + val = VARIANT() + res = _dll.remoteOpResult_getOperand(self._pResults, operandId, byref(val)) + if res != 0: + raise LookupError(f"Operand {operandId} not found in results") + return val.value + + def __del__(self): + _dll.remoteOpResult_free(self._pResults) + + +class RemoteOperation: + """ + Creates and wraps a Windows.UI.UIAutomation.Core.CoreAutomationRemoteOperation. + """ + + def __init__(self): + self._pRemoteOperation = c_void_p() + _dll.remoteOp_create(byref(self._pRemoteOperation)) + + def importElement(self, operandId: OperandId, element: UIA.IUIAutomationElement): + """ + Imports a UI automation element into the remote operation VM at the given operand ID. + :param operandId: The operand ID to import the element into. + :param element: The element to import. + """ + _dll.remoteOp_importElement(self._pRemoteOperation, operandId, element) + + def importTextRange(self, operandId: OperandId, textRange: UIA.IUIAutomationTextRange): + """ + Imports a UI automation text range into the remote operation VM at the given operand ID. + :param operandId: The operand ID to import the text range into. + :param textRange: The text range to import. + """ + _dll.remoteOp_importTextRange(self._pRemoteOperation, operandId, textRange) + + def addToResults(self, operandId: OperandId): + """ + Requests that an operand be made available after execution in the results set. + :param operandId: The operand ID to add to the results. + """ + _dll.remoteOp_addToResults(self._pRemoteOperation, operandId) + + def isOpcodeSupported(self, opcode: InstructionType) -> bool: + """ + Returns true if the given opcode (instruction) is supported by the remote operation VM. + :param opcode: The opcode to check. + """ + val = c_bool() + _dll.remoteOp_isOpcodeSupported(self._pRemoteOperation, opcode, byref(val)) + return val.value + + def execute(self, byteCode: bytes) -> RemoteOperationResultSet: + """ + Executes the given byte code in the remote operation VM. + :param byteCode: The byte code array to execute. + """ + pResults = c_void_p() + _dll.remoteOp_execute(self._pRemoteOperation, byteCode, len(byteCode), byref(pResults)) + return RemoteOperationResultSet(pResults) + + def __del__(self): + _dll.remoteOp_free(self._pRemoteOperation) + + +class RemoteOperationStatus(enum.IntEnum): + Success = 0 + MalformedBytecode = 1 + InstructionLimitExceeded = 2 + UnhandledException = 3 + ExecutionFailure = 4 + + +class InstructionType(enum.IntEnum): + Nop = 0x00 + Set = 0x01 + + # Control flow + ForkIfTrue = 0x02 + ForkIfFalse = 0x03 + Fork = 0x04 + Halt = 0x05 + + # Loops + NewLoopBlock = 0x06 + EndLoopBlock = 0x07 + BreakLoop = 0x08 + ContinueLoop = 0x09 + + # Error handling + NewTryBlock = 0x0a + EndTryBlock = 0x0b + SetOperationStatus = 0x0c + GetOperationStatus = 0x0d + + # Arithmetic + Add = 0x0e + Subtract = 0x0f + Multiply = 0x10 + Divide = 0x11 + BinaryAdd = 0x12 + BinarySubtract = 0x13 + BinaryMultiply = 0x14 + BinaryDivide = 0x15 + + # Boolean operators + InPlaceBoolNot = 0x16 + InPlaceBoolAnd = 0x17 + InPlaceBoolOr = 0x18 + + BoolNot = 0x19 + BoolAnd = 0x1a + BoolOr = 0x1b + + # Generic comparison + Compare = 0x1c + + # New object constructors + NewInt = 0x1d + NewUint = 0x1e + NewBool = 0x1f + NewDouble = 0x20 + NewChar = 0x21 + NewString = 0x22 + NewPoint = 0x23 + NewRect = 0x24 + NewArray = 0x25 + NewStringMap = 0x26 + NewNull = 0x27 + + # Point and Rect methods + GetPointProperty = 0x28 + GetRectProperty = 0x29 + + # RemoteArray methods + RemoteArrayAppend = 0x2a + RemoteArraySetAt = 0x2b + RemoteArrayRemoveAt = 0x2c + RemoteArrayGetAt = 0x2d + RemoteArraySize = 0x2e + + # RemoteStringMap methods + RemoteStringMapInsert = 0x2f + RemoteStringMapRemove = 0x30 + RemoteStringMapHasKey = 0x31 + RemoteStringMapLookup = 0x32 + RemoteStringMapSize = 0x33 + + # RemoteString methods + RemoteStringGetAt = 0x34 + RemoteStringSubstr = 0x35 + RemoteStringConcat = 0x36 + RemoteStringSize = 0x37 + + # UIA element methods + GetPropertyValue = 0x38 + Navigate = 0x39 + + # Type interrogation methods + IsNull = 0x3a + IsNotSupported = 0x3b + IsMixedAttribute = 0x3c + IsBool = 0x3d + IsInt = 0x3e + IsUint = 0x3f + IsDouble = 0x40 + IsChar = 0x41 + IsString = 0x42 + IsPoint = 0x43 + IsRect = 0x44 + IsArray = 0x45 + IsStringMap = 0x46 + IsElement = 0x47 + + # GUID support + NewGuid = 0x48 + IsGuid = 0x49 + LookupId = 0x4a + LookupGuid = 0x4b + + # Cache requests + NewCacheRequest = 0x4c + IsCacheRequest = 0x4d + CacheRequestAddProperty = 0x4e + CacheRequestAddPattern = 0x4f + PopulateCache = 0x50 + + Stringify = 0x51 + GetMetadataValue = 0x52 + + # Extensibility + CallExtension = 0x53 + IsExtensionSupported = 0x54 + + # text ranges + TextRangeClone = 0x271e0103 + TextRangeCompare = 0x271e0104 + TextRangeCompareEndpoints = 0x271e0105 + TextRangeExpandToEnclosingUnit = 0x271e0106 + TextRangeFindAttribute = 0x271e0107 + TextRangeFindText = 0x271e0108 + TextRangeGetAttributeValue = 0x271e0109 + TextRangeGetBoundingRectangles = 0x271e010a + TextRangeGetEnclosingElement = 0x271e010b + TextRangeGetText = 0x271e010c + TextRangeMove = 0x271e010d + TextRangeMoveEndpointByUnit = 0x271e010e + TextRangeMoveEndpointByRange = 0x271e010f + TextRangeSelect = 0x271e0110 + TextRangeAddToSelection = 0x271e0111 + TextRangeRemoveFromSelection = 0x271e0112 + TextRangeScrollIntoView = 0x271e0113 + TextRangeGetChildren = 0x271e0114 + TextRangeShowContextMenu = 0x271e0115 + + +class ComparisonType(enum.IntEnum): + Equal = 0 + NotEqual = 1 + GreaterThan = 2 + LessThan = 3 + GreaterThanOrEqual = 4 + LessThanOrEqual = 5 + + +class NavigationDirection(enum.IntEnum): + Parent = 0 + NextSibling = 1 + PreviousSibling = 2 + FirstChild = 3 + LastChild = 4 + + +class TextUnit(enum.IntEnum): + Character = 0 + Format = 1 + Word = 2 + Line = 3 + Paragraph = 4 + Page = 5 + Document = 6 + + +class TextPatternRangeEndpoint(enum.IntEnum): + Start = 0 + End = 1 + + +PropertyId = enum.IntEnum( + "PropertyId", + {k[4:-10]: v for k, v in vars(UIA).items() if k.endswith("PropertyId")} +) + + +AttributeId = enum.IntEnum( + "AttributeId", + {k[4:-11]: v for k, v in vars(UIA).items() if k.endswith("AttributeId")} +) + +StyleId = enum.IntEnum( + "StyleId", + {k[8:]: v for k, v in vars(UIA).items() if k.startswith("StyleId")} +) diff --git a/source/UIAHandler/_remoteOps/operation.py b/source/UIAHandler/_remoteOps/operation.py new file mode 100644 index 00000000000..ecb52fcfcec --- /dev/null +++ b/source/UIAHandler/_remoteOps/operation.py @@ -0,0 +1,354 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +from __future__ import annotations +import contextlib +from typing import ( + Type, + Any, + Generator, + Callable, +) +from dataclasses import dataclass +from logHandler import log +from UIAHandler import UIA +from . import lowLevel +from . import builder +from . import remoteAPI +from . import instructions + + +@dataclass +class ExecutionResult: + status: int + errorLocation: int + extendedError: int + + def hasOperand(self, operandId: lowLevel.OperandId) -> bool: + raise NotImplementedError + + def getOperand(self, operandId: lowLevel.OperandId) -> remoteAPI.RemoteBaseObject: + raise NotImplementedError + + +class Executor: + + def importElement(self, operandId: lowLevel.OperandId, element: UIA.IUIAutomationElement): + raise NotImplementedError + + def importTextRange(self, operandId: lowLevel.OperandId, textRange: UIA.IUIAutomationTextRange): + raise NotImplementedError + + def addToResults(self, operandId: lowLevel.OperandId): + raise NotImplementedError + + def loadInstructions(self, rob: builder.RemoteOperationBuilder): + raise NotImplementedError + + def execute(self) -> ExecutionResult: + raise NotImplementedError + + +class OperationException(RuntimeError): + operation: Operation + executionResult: ExecutionResult + errorLocation: int | None = None + extendedError: int | None = None + instructionRecord: instructions.InstructionBase | None = None + + def __init__(self, operation: Operation, executionResult: ExecutionResult): + self.operation = operation + self.executionResult = executionResult + self.errorLocation = executionResult.errorLocation + self.extendedError = executionResult.extendedError + if self.errorLocation >= 0: + instructions = operation._rob.getAllInstructions() + try: + self.instructionRecord = instructions[self.errorLocation] + except (IndexError, RuntimeError): + self.instructionRecord = None + super().__init__(f"Operation failed with status {executionResult.status}") + + def __str__(self) -> str: + message = "" + if self.errorLocation: + message += f"\nat instruction {self.errorLocation}" + if self.instructionRecord is not None: + message += f": {self.instructionRecord.dumpInstruction()}" + if self.extendedError: + message += f"\nextendedError {self.extendedError}" + return message + + +class ExecutionFailureException(OperationException): + pass + + +class MalformedBytecodeException(OperationException): + pass + + +class InstructionLimitExceededException(OperationException): + pass + + +class UnhandledException(OperationException): + pass + + +class NoReturnException(Exception): + pass + + +@dataclass +class RemoteExecutionResult(ExecutionResult): + resultSet: lowLevel.RemoteOperationResultSet + + def hasOperand(self, operandId: lowLevel.OperandId) -> bool: + return self.resultSet.hasOperand(operandId) + + def getOperand(self, operandId: lowLevel.OperandId) -> object: + return self.resultSet.getOperand(operandId) + + +class RemoteExecutor(Executor): + _ro: lowLevel.RemoteOperation + _isConnectionBound = False + _byteCode: bytes = b"" + + def __init__(self): + self._ro = lowLevel.RemoteOperation() + + def importElement(self, operandId: lowLevel.OperandId, element: UIA.IUIAutomationElement): + self._ro.importElement(operandId, element) + self._isConnectionBound = True + + def importTextRange(self, operandId: lowLevel.OperandId, textRange: UIA.IUIAutomationTextRange): + self._ro.importTextRange(operandId, textRange) + self._isConnectionBound = True + + def addToResults(self, operandId: lowLevel.OperandId): + self._ro.addToResults(operandId) + + def loadInstructions(self, rob: builder.RemoteOperationBuilder): + self._byteCode = rob.getByteCode() + + def execute(self) -> ExecutionResult: + if not self._isConnectionBound: + raise RuntimeError("RemoteExecutor must be bound to a connection before execution") + resultSet = self._ro.execute(self._byteCode) + return RemoteExecutionResult( + status=resultSet.status, + errorLocation=resultSet.errorLocation, + extendedError=resultSet.extendedError, + resultSet=resultSet + ) + + +class Operation: + _executorClass: Type[Executor] = RemoteExecutor + _compiletimeLoggingEnabled: bool + _runtimeLoggingEnabled: bool + _remoteLog: remoteAPI.RemoteString | None = None + _rob: builder.RemoteOperationBuilder + _importedElements: dict[lowLevel.OperandId, UIA.IUIAutomationElement] + _importedTextRanges: dict[lowLevel.OperandId, UIA.IUIAutomationTextRange] + _requestedResults: dict[lowLevel.OperandId, remoteAPI.RemoteBaseObject] + _staticOperands: list[remoteAPI.RemoteBaseObject] + _returnIdOperand: remoteAPI.RemoteInt | None = None + _yieldListOperand: remoteAPI.RemoteArray | None = None + _built = False + _executionCount = 0 + + def __init__( + self, + enableCompiletimeLogging: bool = False, + enableRuntimeLogging: bool = False, + localMode: bool = False + ): + self._compiletimeLoggingEnabled = enableCompiletimeLogging + self._runtimeLoggingEnabled = enableRuntimeLogging + self._localMode = localMode + if localMode: + from .localExecute import LocalExecutor + self._executorClass = LocalExecutor + self._rob = builder.RemoteOperationBuilder() + self._importedElements = {} + self._importedTextRanges = {} + self._requestedResults = {} + self._staticOperands = [] + + def importElement( + self, + element: UIA.IUIAutomationElement, + operandId: lowLevel.OperandId | None = None + ) -> remoteAPI.RemoteElement: + if operandId is None: + operandId = self._rob.requestNewOperandId() + self._importedElements[operandId] = element + self._rob.getDefaultInstructionList().addMetaCommand( + f"importElement into {operandId}, value {element}" + ) + return remoteAPI.RemoteElement(self._rob, operandId) + + def importTextRange( + self, + textRange: UIA.IUIAutomationTextRange, + operandId: lowLevel.OperandId | None = None + ) -> remoteAPI.RemoteTextRange: + if operandId is None: + operandId = self._rob.requestNewOperandId() + self._importedTextRanges[operandId] = textRange + self._rob.getDefaultInstructionList().addMetaCommand( + f"importTextRange into {operandId}, value {textRange}" + ) + return remoteAPI.RemoteTextRange(self._rob, operandId) + + def addToResults(self, *operands: remoteAPI.RemoteBaseObject): + for operand in operands: + self._requestedResults[operand.operandId] = operand + + def _registerStaticOperand(self, operand: remoteAPI.RemoteBaseObject): + self._staticOperands.append(operand) + self.addToResults(operand) + + def _refreshStaticInstructions(self): + with self._rob.overrideDefaultSection('static'): + self._rob.getDefaultInstructionList().clear() + for operand in self._staticOperands: + if isinstance(operand, remoteAPI.RemoteElement): + localElement = operand.localValue + if not localElement: + remoteAPI.RemoteNull.createNew(self._rob, operandId=operand.operandId) + else: + self.importElement(operand.localValue, operandId=operand.operandId) + elif isinstance(operand, remoteAPI.RemoteTextRange): + localTextRange = operand.localValue + if not localTextRange: + remoteAPI.RemoteNull.createNew(self._rob, operandId=operand.operandId) + else: + self.importTextRange(operand.localValue, operandId=operand.operandId) + else: + type(operand).createNew(self._rob, initialValue=operand.localValue, operandId=operand.operandId) + + @contextlib.contextmanager + def buildContext(self): + if self._built: + raise RuntimeError("RemoteOperation cannot be built more than once") + ra = remoteAPI.RemoteAPI(self, enableRemoteLogging=self._runtimeLoggingEnabled) + self._remoteLog = logObj = ra.getLogObject() + if logObj is not None: + self.addToResults(logObj) + yield ra + ra.halt() + if self._compiletimeLoggingEnabled: + self._dumpCompiletimeLog() + self._built = True + + def buildFunction( + self, + func: Callable[[remoteAPI.RemoteAPI], None] + ) -> Operation: + with self.buildContext() as ra: + self._returnIdOperand = ra.newInt(-1) + self.addToResults(self._returnIdOperand) + func(ra) + return self + + def buildIterableFunction( + self, + func: Callable[[remoteAPI.RemoteAPI], None] + ) -> Operation: + with self.buildContext() as ra: + self._yieldListOperand = ra.newArray() + self.addToResults(self._yieldListOperand) + func(ra) + return self + + def _execute(self) -> ExecutionResult: + if not self._built: + raise RuntimeError("RemoteOperation must be built before execution") + executor = self._executorClass() + for operandId, element in self._importedElements.items(): + executor.importElement(operandId, element) + for operandId, textRange in self._importedTextRanges.items(): + executor.importTextRange(operandId, textRange) + for operandId in self._requestedResults: + executor.addToResults(operandId) + executor.loadInstructions(self._rob) + executionResult = executor.execute() + for operand in self._requestedResults.values(): + operand._setExecutionResult(executionResult) + if executionResult.status == lowLevel.RemoteOperationStatus.ExecutionFailure: + raise ExecutionFailureException(self, executionResult) + elif executionResult.status == lowLevel.RemoteOperationStatus.MalformedBytecode: + raise MalformedBytecodeException(self, executionResult) + return executionResult + + def _dumpRemoteLog(self): + if self._remoteLog is not None: + logOutput = self._remoteLog.localValue + if logOutput: + log.info( + f"Remote log for execution {self._executionCount}\n" + "--- Begin ---\n" + f"{logOutput}" + "--- end ---" + ) + + def _dumpCompiletimeLog(self): + log.info( + "Dumping instructions:\n" + "--- Begin ---\n" + f"{self._rob.dumpInstructions()}" + "--- End ---" + ) + + def _executeUntilSuccess(self, maxTries: int) -> Generator[ExecutionResult, None, None]: + self._executionCount = 0 + try: + while self._executionCount < maxTries: + self._executionCount += 1 + if self._executionCount > 1: + self._refreshStaticInstructions() + executionResult = self._execute() + self._dumpRemoteLog() + yield executionResult + if executionResult.status == lowLevel.RemoteOperationStatus.InstructionLimitExceeded: + if self._executionCount == maxTries: + raise InstructionLimitExceededException(self, executionResult) + else: + continue + if executionResult.status == lowLevel.RemoteOperationStatus.UnhandledException: + raise UnhandledException(self, executionResult) + break + except Exception as e: + e.add_note( + f"Error occured on execution try {self._executionCount}" + ) + e.add_note( + "Dumping instructions:\n" + "--- Begin ---\n" + f"{self._rob.dumpInstructions()}" + "--- End ---" + ) + raise + + def execute(self, maxTries: int = 1) -> Any: + if self._returnIdOperand is None: + raise RuntimeError("RemoteOperation has no return operand") + for executionResult in self._executeUntilSuccess(maxTries): + pass + returnId = self._returnIdOperand.localValue + if returnId < 0: + raise NoReturnException() + return self._requestedResults[lowLevel.OperandId(returnId)].localValue + + def iterExecute(self, maxTries: int = 1) -> Generator[Any, None, None]: + if self._yieldListOperand is None: + raise RuntimeError("RemoteOperation has no yield list operand") + for executionResult in self._executeUntilSuccess(maxTries): + for value in self._yieldListOperand.localValue: + yield value diff --git a/source/UIAHandler/_remoteOps/readme.md b/source/UIAHandler/_remoteOps/readme.md new file mode 100644 index 00000000000..90def0ec1f6 --- /dev/null +++ b/source/UIAHandler/_remoteOps/readme.md @@ -0,0 +1,556 @@ +# UI Automation Remote Operations for NVDA + +## Introduction +If you have code that makes more than a few UI Automation calls all in a row, you may want to consider rewriting the code as a Remote Operation. +This will allow Windows to execute the code all in one cross-process call, thereby significantly speeding up execution. + +Following is a simple example of a remote operation. +### Example 1: Fetching the names of all ancestors of a UI Automation element +```py +import api +from UIAHandler import UIA +from UIAHandler._remoteOps.operation import Operation +from UIAHandler._remoteOps.remoteAPI import RemoteAPI + +# Fetch the UI Automation element for the current focus in NVDA +focusElement = api.getFocusObject().UIAElement + +# Create a new Remote Operation +op = Operation() + +# Build the instructions for the remote operation. +@op.buildFunction +def code(ra: RemoteAPI): + # Create a new remote element initializing it to the focus element. + element = ra.newElement(focusElement) + # Create a new array to hold the names we collect. + names = ra.newArray() + # Declare a while loop that will walk up the ancestors and collect the names. + with ra.whileBlock(lambda: element.isNull().inverse()): + # Fetch the name property of this element and store it in the array. + name = element.getPropertyValue(UIA.UIA_NamePropertyId) + names.append(name) + # Fetch the element's parent and point element to it. + parent = element.getParentElement() + element.set(parent) + # Now back outside the while loop. + # Return the names array from the remote operation. + ra.Return(names) +# Now the operation is built. + +# Actually execute the remote operation, which will return the names array to NVDA. +names = op.execute() +# Print the names we got back. +print(f"{names=}") +``` + +## Building a remote operation +To build a remote operation, you define a function decorated by the `Operation.buildFunction` decorator. +This function must take a `RemoteAPI` object as its one and only argument. +The function will use methods on the `RemoteAPI` object to declare all its actions and logic. +Avoid using any other APIs, function calls or control flow. +```py +op = Operation() +@op.buildFunction +def code(ra: RemoteAPI): + # Call some methods on ra... +``` + +### Returning values +To return a value or values from a remote operation, use the `ra.Return` method, passing one or more remote values as arguments: +```py +op = Operation() +@op.buildFunction +def code(ra: RemoteAPI): + i = ra.newInt(10) + div = i / 7 + mod = j % 7 + ra.Return(div, mod) +``` + +Note that all build functions must return at least one value. +Otherwise, `Operation.execute` will raise a `NoReturnException`. + +### All operations require at least one element or text range +The primary reason for writing a remote operation is to perform actions upon one or more UI automation elements or text ranges. +And as a remote operation is executed in a remote provider, it needs to be connection bound, meaning that it needs to be associated with at least one element or text range from that provider process. +Therefore, all remote operations require at least one element (via ra.newElement) or text range (via ra.newTextRange) to be declared. +Also, all elements and text ranges in the operation must be from the same provider process. + +### Declarative style and control flow +When building a remote operation, the actions are specified in a declarative style. +In other words, the code internally builds up a set of low-level instructions under the hood which will later be executed remotely. +This is most evident when specifying control flow such as a while loop: +```py +counter = ra.newInt(0) +with ra.whileBlock(lambda: counter < 5): + counter += 1 +``` +From Python's point of view, the body of the declared while loop is only run once, as it is only being declared, not executed. +Similarly, for if-else blocks: +```py +condition = ra.newBool(True) +with ra.ifblock(condition): + # Do stuff if condition is true... +with ra.elseBlock(): + # do stuff if condition is false... +``` + +From Python's point of view, the code inside both the if and else blocks will be run, as as it is declaring (not executing) the code here. +This will be covered more in further sections about control flow. +But the most important thing to remember here is that you should avoid using any of Python's own control flow (such as if or while, as it most likely will not do what you expected). +The remote API has all the control flow you need, such as ifBlock, elseBlock, whileBlock, tryBlock, again covered in later sections. + +### Method arguments +When providing arguments to methods, you can use remote values previously declared in the operation, or you can use literal Python values. +When using literal Python values, these will be automatically remoted as special constant values for you. +For example: +```py +textRange = ra.newTextRange(UIATextRange) +textRange.move(TextUnit_Word, 1) +``` + +In this example, the values TextUnit_Word and 1 will be automatically remoted. + +### Basic remote types +#### Equality checks +All types support equality checks: +```py +i = ra.newBool(True) +j = ra.newBool(False) +k = i == j +``` + +#### Setting a specific value +All types have a `set` method which allows you to set the remote variable to a specific value: +```py +i = ra.newBool(False) +# i is initialised as false. +# But now set it to true +i.set(True) +``` + +For most types, `set` will copy the value, i.e. setting `a` to `b` and then manipulating `b` will not change `a`. +However, for certain types such as elements, text ranges and arrays, these are held by reference and therefore manipulating the value it was set two will change the underlying object for both variables. + +#### Booleans +```py +a = ra.newBool(True) +b = ra.newBool(False) +``` + +##### Logical operators +Booleans support logical operations: +* and: `a & b` +* or: `a | b` +* Inverse: `a.inverse()` + +Unfortunately the Python language does not allow overriding `!=`, `and` and `or` to return custom types. +Thus why the above operators were chosen. + +#### Ints and floats +Remote operations support declaring and manipulating int and float types. +```py +myInt = ra.newInt(5) +myFloat = ra.newFloat(7.2) +``` + +There is also unsigned int (`ra.newUint`) but the only place you may need to use this is for interacting with the size of a remote array (covered later). + +Please note that remote operations do not allow ints and floats to be converted to one another. + +##### Arithmetic +Ints, uints and floats all support the standard arithmetic operations: add, subtract, multiply, divide and modulo. +There are both binary and in-place operators for these. +```py +i = ra.newInt(5) +j = ra.newInt(6) +# addition +k = i + j +k += j +# subtraction +l = i - j +l -= j +# multiplication +m = i * j +m *= j +# division +n = i / j +n /= j +# modulo +o = i % j +o %= j +``` + +##### Comparison operators +Ints and floats support comparisons, returning a boolean: less, less equals, equals, greater equals and greater. +```py +i = ra.newInt(5) +j = ra.newInt(6) +k = i < j +l = i <= j +m = i == j +n = i >= j +o = i > j +``` + +#### Strings +```py +s = ra.newString("Hello") +``` + +##### Concatenation +Strings can be concatenated to create a new string: +```py +s = ra.newString("Hello ") +t = ra.newString("world") +u = s + t +``` + +Or they can be concatenated in-place: +```py +s = ra.newString("Hello ") +t = ra.newString("world") +s += t +``` + +### UIA elements +#### Declaring an element +To create a new remote element, call `ra.newElement`, giving it an existing `IUIAutomationElement` comtypes pointer as its argument: +```py +element = ra.newElement(UIAElement) +``` + +#### Fetching element properties +To fetch properties from an element, use `getPropertyValue`: +```py +name = element.getPropertyValue(UIA_NamePropertyId) +controlType = element.getPropertyValue(UIA_ControlTypePropertyId) +``` + +Any of the standard UI Automation property IDs can be used here. + +#### Navigating the element tree +To navigate to other elements in the tree from this element, use the following methods on the element: +* `getParentElement` +* `getFirstchildElement` +* `getLastChildElement` +* `getPreviousSiblingElement` +* `getNextSiblingElement` + +#### Pointing to another element +To make an element point to another physical UI Automation element, call its `set` method with another remote element as an argument: +```py +parent = element.getParentElement() +element.set(parent) +``` + +This is useful when walking the element tree in a loop. + +### UI Automation text ranges +#### Declaring a text range +To create a new remote text range, call `ra.newTextRange`, giving it an existing IUIAutomationTextRange comtypes pointer as its argument: +```py +textRange = ra.newElement(UIATextRange) +``` + +Note that under the hood the text range is automatically cloned after it has been remoted, so that any manipulation of the remote text range (such as moving its ends) is not reflected in the original IUIAutomationTextRange you gave it. + +#### Retrieving text, comparison and manipulation +The majority of methods found on `IUIAutomationTextRange` are available on remote text ranges, including: +* `getText` +* `compareEndpoints` +* `moveEndpointByUnit` +* `moveEndpointByRange` +* `expandToEnclosingUnit` +* `getEnclosingElement` +* ... + +Refer to the `RemoteTextRange` class in remoteAPI.py, or official [IUIAutomationTextRange documentation](https://learn.microsoft.com/en-us/windows/win32/api/uiautomationclient/nn-uiautomationclient-iuiautomationtextrange) for all the call signatures. +But as an example, here is an algorithm that can count the number of words in a text range: +```py +wordCount = ra.newInt(0) +textRange = ra.newTextRange(UIATextRange) +tempRange = textRange.clone() +# Collapse the range to the start +tempRange.moveEndpointByRange(TextPatternRangeEndpoint_End, tempRange, TextPatternRangeEndpoint_Start) +with ra.whileBlock(lambda: tempRange.move(TextUnit_word, 1) == 1): + with ra.ifblock(tempRange.compareEndpoints(textPatternRangeEndpoint_Start, textRange, TextPatternRangeEndpoint_End) >= 0): + ra.breakLoop() + wordCount += 1 +ra.Return(wordcount) +``` + +#### Text range logical adapter to improve logic and readability +The verboseness of many of the compare and move text range methods can make it hard to quickly read the code and gain a good idea of what the algorithm is actually doing. +It is also quite tricky to write an algorithm that is easily reversed. +Therefore remote text ranges have a `getLogicalAdapter` method, taking a single boolean `reverse` argument which returns a special object which wraps a remote text range, and provides friendly start and end properties, which take the reversal into account. +Here is an example of how you could write an algorithm to fetch the first 20 words in the text range, either from the start or end: +```py +textRange = ra.newTextRange(UIATextRange) +words = ra.newArray() +counter = ra.newInt(0) +logicalTextRange = textRange.getLogicalAdapter(reverse=False) # Change to True to reverse the algorithm. +logicalTempRange = logicalTextRange.clone() +# Collapse the range to the start +logicalTempRange.end = logicalTempRange.start +# Loop up to 20 times +with ra.whileBlock(lambda: counter < 20): + # Move the end of the text range forward by one word. + # If it fails, break out of the loop. + with ra.ifBlock(logicalTempRange.end.moveByUnit(TextUnit_Word, 1) == 0): + ra.breakLoop() + # If our temp range has passed the end of the original text range, break out of the loop. + with ra.ifBlock(logicalTempRange.end > logicalTextRange.end): + ra.breakLoop() + # collect the text and add it to the words array. + text = logicalTempRange.textRange.getText(-1) + words.append(text) + # collapse the range to the end. + logicalTempRange.start = logicalTempRange.end + # Increment the counter by 1. + counter += 1 +# Return the words array. +ra.Return(words) +``` + +By simply changing the False to True, the algorithm is automatically reversed, as the start and end properties reverse, and the `numUnits` argument on the methods have their sign flipped. +As shown above, `start` and `end` properties can be assigned to which moves the endpoint, and they can be moved by a unit with `moveByUnit`. +They can also be compared with `<`, `<=`, `==`, `>=`, and `>`. +If you still want the comparison delta (like with `compareEndpoints`), the properties also have a `compareWith` method, which takes another endpoint and gives back a number less than 0, equal to 0 or greater than 0. + +### control flow +The remoteAPI object has methods for control flow that return Python context managers, so that they can be used as `with` statements. + +#### if-else +To conditionally perform actions, place them in a `with` statement using `ra.ifBlock`. +`ifBlock` takes one argument, which is a remote boolean. +If this argument is evaluated to True during execution, then the actions within the `with` statement are executed. +An optional `with` statement using `ra.elseBlock` can directly follow the `ifBlock` `with` statement, and if the condition evaluates to False, then the actions within the `elseBlock` `with` statement will be executed instead. +```py +i = ra.newInt(5) +j = ra.newInt(6) +with ra.ifblock(i < j): + # do stuff if true... +with ra.elseBlock(): + # do stuff if false... +``` + +#### While loops +To keep performing some actions while a condition is True, place the actions in a `with` statement using `ra.whileBlock`. +`whileBlock` takes one argument, which is a lambda that will return a remote boolean. +```py +counter = ra.newInt(0) +with ra.whileBlock(lambda: counter < 5): + # do some actions + counter += 1 +``` + +`ra.breakLoop` and `ra.continueLoop` methods can be called within the loop body to break or continue respectively. + +Please note that the condition of the while loop must be placed in a lambda as it needs to be fully evaluated within the top of the loop. +If this was not done, the instructions that produced the final boolean condition would appear before the loop and never get re-run on subsequent iterations. + +#### try-catch +If an action causes an error, it is possible to catch the error by placing those actions in a `with` statement using the `ra.tryBlock` method. +When using `ra.tryBlock`, a second `with` statement using `ra.catchBlock` must follow straight after. +If an error occurs within the `tryBlock` `with` statement, then execution jumps to the `catchBlock` `with` statement. +You can capture the exact error code as the value of the `with` statement. +```py +with ra.tryBlock(): + # do stuff... +with ra.catchBlock() as errorCode: + # do stuff... +``` + +If it is an element or text range method that causes the error, the error code will be the COM HRESULT for that method E.g. `E_INVALIDARG`. +Other errors such as divide by 0 have their own error codes. + +### Higher-level algorithms +#### Looping over a range of numbers +Although you can use a while loop and a counter to loop over a range of numbers, the library provides a helper method `ra.forEachNumInRange` which takes start, stop, and optional step arguments. +This method can be used in a `with` statement to loop over a range of numbers like so: +```py +with ra.forEachNumInRange(0, 10, 2) as num: + # do something with num +``` + +#### Looping over arrays +To simplifying looping over each item in an array, the library provides `ra.forEachItemInArray` which can be used in a `with` statement like such: +```py +array = ra.newArray() +# Populate the array... + +with ra.forEachItemInArray(array) as item: + # do something with the item... +``` + +## Executing an operation +Once an operation is built, you will want to actually execute it on the remote provider. +To execute the operation, call `Operation.execute`. +This method takes no arguments, and returns any values previously returned with `ra.Return`. +These values are brought back to NVDA and converted to real Python types. + +### Instruction limits +So as to not freeze a remote provider, Microsoft has placed a limit on how many instructions can be executed for one operation. +Currently this limit is 10,000. +This seems a lot, but once you are dealing with many while loop iterations containing a lot of actions, it is very easy to to hit this limit. +If the instruction limit is reached, then `Operation.execute` will raise `InstructionLimitExceededException`. +Assuming your algorithm was written appropriately, you could then re-execute it, and the remote provider will have had a chance to run its own main loop or do what ever it needs to do between operations. + +To aide in writing algorithms that can handle this instruction limit and continue to execute where it left off, there are several features of this Remote Operations library that can be used. + +#### Automatic retry +`Operation.execute` takes a `maxTries` keyword argument which is set to 1 by default, meaning that the operation will only be executed once, and if the instruction limit is hit, then `InstructionLimitExceededException` is raised. +However, if `maxTries` is greater than 1, `Operation.execute` will automatically retry more times until the operation executes without hitting the limit, or when `maxTries` is reached. + +This in itself however is not too useful unless some other changes are made to the algorithm itself, so that it is suitable for running multiple times by remembering where it left off. + +#### Static values +Most `ra.newXXX` methods take a `static` keyword argument which is set to `False` by default. +However, if set to `True`, subsequent executions of the operation will initialize the value to what it was when the last execution finished. +```py +counter = ra.newInt(0, static=True) +with ra.whileBlock(lambda: counter < 20000): + counter += 1 +``` + +The above example will most definitely hit the instruction limit, however, because static was set to true, on the next execution counter will be re-initialized with the last value it was before the limit was hit. + +Please note though that the execution will still start again from the first instruction, so the algorithm still needs to be written to take this into account. + +It could be possible in future to implement the library to start from the exact instruction where the limit was hit, but this would mean marshalling each and every declared remote variable out and then back in, which could be costly. +This is why it currently only does it for ones marked as `static`. + +Also, currently it is impossible to mark arrays as `static`, as arrays cannot be initialized with a value in the low-level remote operations framework. +Again, in future the library could be extended to support this, but it would involve having to marshal out all items, and marshal them back in, appending them to the array one at a time, which would also be costly. + +#### Building iterable functions +A common use of remote operations is to walk a text range or element tree, and collect data which would be returned in an array. +However, as arrays can not be marked as `static`, this would involve a lot of extra code to handle execution continuation after the instruction limit is reached. +Therefore the library supports a `Operation.buildIterableFunction` decorator, which can be used in place of `Operator.buildFunction`. +Within a function that uses this decorator, rather than using ra.Return to return value and halt, you can use `ra.Yield` which will yield a value and continue to execute (until the instruction limit is reached of course). +To actually execute an iterable function though, instead of using `Operation.execute`, you use `Operation.iterExecute` as the generator to a `for` loop, which will iterate over the yielded values. +```py +op = Operation() + +@op.buildIterfunction +def code(ra: RemoteAPI): + counter = ra.newInt(0, static=True) + with ra.whileBlock(lambda: counter < 20000): + with ra.ifBlock((counter % 1000) == 0): + ra.Yield(counter) + counter += 1 + +for item in op.iterExecute(maxTries=10): + print(f"{item=}") +``` + +The above example will print 0, 1000, 2000, 3000... + +Although the `for` loop will see each yielded value separately, they will only physically yield either when the instruction limit is reached, and or when the execution finally reaches the end, which still means that as many actions as possible are executed in one cross-process call. + +## Debugging +It can be tricky to debug a remote operation as it executes in the remote provider. +Therefore the library contains several features which can help. + +### Dumping instructions +The library can dump all the instructions to NVDA's log each time an operation is built, by setting the Operation's `enableCompiletimeLogging` keyword argument to True. +Even if left as False, instructions will still be automatically dumped to NVDA's log if there is an uncaught error while executing, or the instruction limit is reached and it has run out of tries. +Following is code for a simple remote operation, followed by a dump of its instructions. +```py +counter = ra.newInt(0, static=True) +with ra.whileBlock(lambda: counter < 20000): + with ra.ifBlock((counter % 1000) == 0): + ra.Yield(counter) + counter += 1 +``` + +And now the instruction dump: +``` +--- Begin --- +static: +0: NewInt(result=RemoteInt at OperandId 2, value=c_long(0)) +const: +1: NewInt(result=const RemoteInt at OperandId 4, value=c_long(20000)) +2: NewInt(result=const RemoteInt at OperandId 6, value=c_long(1000)) +3: NewInt(result=const RemoteInt at OperandId 10, value=c_long(0)) +4: NewInt(result=const RemoteInt at OperandId 11, value=c_long(1)) +main: +5: NewArray(result=RemoteArray at OperandId 1) +6: NewLoopBlock(breakBranch=RelativeOffset 12, continueBranch=RelativeOffset 1) +# Entering RemoteNumber.__lt__(20000, ) +# Using cached const RemoteInt at OperandId 4 for constant value 20000 +7: Compare(result=RemoteBool at OperandId 3, left=RemoteInt at OperandId 2, right=const RemoteInt at OperandId 4, comparisonType=) +# Exiting RemoteNumber.__lt__ +8: ForkIfFalse(condition=RemoteBool at OperandId 3, branch=RelativeOffset 9) +# While block body +# Entering RemoteNumber.__mod__(1000, ) +# Entering RemoteNumber.__truediv__(1000, ) +# Using cached const RemoteInt at OperandId 6 for constant value 1000 +9: BinaryDivide(result=RemoteInt at OperandId 5, left=RemoteInt at OperandId 2, right=const RemoteInt at OperandId 6) +# Exiting RemoteNumber.__truediv__ +# Entering RemoteNumber.__mul__(1000, ) +# Using cached const RemoteInt at OperandId 6 for constant value 1000 +10: BinaryMultiply(result=RemoteInt at OperandId 7, left=RemoteInt at OperandId 5, right=const RemoteInt at OperandId 6) +# Exiting RemoteNumber.__mul__ +# Entering RemoteNumber.__sub__(RemoteInt at OperandId 7, ) +11: BinarySubtract(result=RemoteInt at OperandId 8, left=RemoteInt at OperandId 2, right=RemoteInt at OperandId 7) +# Exiting RemoteNumber.__sub__ +# Exiting RemoteNumber.__mod__ +# Entering RemoteBaseObject.__eq__(0, ) +# Using cached const RemoteInt at OperandId 10 for constant value 0 +12: Compare(result=RemoteBool at OperandId 9, left=RemoteInt at OperandId 8, right=const RemoteInt at OperandId 10, comparisonType=) +# Exiting RemoteBaseObject.__eq__ +13: ForkIfFalse(condition=RemoteBool at OperandId 9, branch=RelativeOffset 2) +# If block body +# Begin yield (RemoteInt at OperandId 2,) +# Yielding RemoteInt at OperandId 2 +# Entering RemoteArray.append(RemoteInt at OperandId 2, ) +14: RemoteArrayAppend(target=RemoteArray at OperandId 1, value=RemoteInt at OperandId 2) +# Exiting RemoteArray.append +# End of if block body +# Entering RemoteNumber.__iadd__(1, ) +# Using cached const RemoteInt at OperandId 11 for constant value 1 +15: Add(target=RemoteInt at OperandId 2, value=const RemoteInt at OperandId 11) +# Exiting RemoteNumber.__iadd__ +# End of while block body +16: ContinueLoop() +17: EndLoopBlock() +18: Halt() +--- End -- +``` + +Looking at the dump we can see the library stores instructions in three specific sections: +* `static`: for initializing static instructions. this section is replaced before each execution. +* `const`: This section holds values which have been automatically remoted, and are used through out the rest of the instructions. +* `main`: the instructions implementing the main logic of the operation. + +We can also see that a part from each numbered instruction and its parameters, there are also comments which help to understand the higher-level logic, such as when entering and exiting particular methods. + +We can see that yielding values is actually implemented by the library internally as an array. + +and finally, we can see that the modulo (%) operator is actually emulated by the equation: `a - (a / b) * b`, as the low-level remote operations framework does not actually support modulo. + +### Adding compiletime comments +It is possible to add comments to the instruction dump when building an operation by using `ra.addCompiletimeComment`. +It takes a single argument which is a literal string value. + +### Runtime remote logging +Setting an Operation's `enableRuntimeLogging` keyword argument to True enables remote logging at execution time. +`ra.logRuntimeMessage` can be called to log a message at runtime. +It takes one or more literal strings and or remote values, and concatinates them together remotely. +For remote values that are not strings, `logRuntimeMessage` uses the remote value's `stringify` method to produce a string representation of the value. + +After the operation is executed, the remote log is marshalled back and dumped to NvDA's log, thereby giving the ability to trace what is happening during the execution. +Though be aware that as remote logging itself involves creating and manipulating remote values, then the number of instructions can change quite significantly with remote logging enabled. + +### Local mode +When unit testing this library, or in a scenario where remote operations is unavailable but you want to use the exact same algorithm but locally, you can set the Operation's `localMode` keyword argument to True. +This causes all instructions to be executed locally, rather than in a remote provider. +This will of course be significantly slower, as every instruction that manipulates an element or text range will be itself one cross-process call. +However, it is a useful means of testing and debugging, and much care has been taken to ensure that the results and side-effects are identical to executing it remotely. + +This differs some what from Microsoft's original remote operations library which implemented its local mode so that instructions were executed locally at build time, and executing did nothing. +This library produces instructions just as it would remotely, but it is these low-level instructions that are executed locally at execution time, following all the same rules and limitations that executing remotely would. +Thus, it is more suited to debugging / testing, rather than as a means of executing where remote operations is unavailable, as code could be written much more efficiently using comtypes IUIAutomation interfaces directly. diff --git a/source/UIAHandler/_remoteOps/remoteAPI.py b/source/UIAHandler/_remoteOps/remoteAPI.py new file mode 100644 index 00000000000..260605c7ff2 --- /dev/null +++ b/source/UIAHandler/_remoteOps/remoteAPI.py @@ -0,0 +1,379 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + + +from __future__ import annotations +from typing import ( + Type, + Any, + Callable, + Generator, + TypeVar, + cast +) +import contextlib +from comtypes import ( + GUID +) +from UIAHandler import UIA +from .lowLevel import RelativeOffset +from . import instructions +from . import builder +from .remoteFuncWrapper import ( + remoteContextManager +) +from . import operation +from .remoteTypes import ( + RemoteBaseObject, + RemoteVariant, + RemoteBool, + RemoteIntBase, + RemoteInt, + RemoteUint, + RemoteFloat, + RemoteString, + RemoteGuid, + RemoteArray, + RemoteElement, + RemoteTextRange, +) + + +class RemoteAPI(builder._RemoteBase): + _op: operation.Operation + _rob: builder.RemoteOperationBuilder + _logObj: RemoteString | None = None + + def __init__(self, op: operation.Operation, enableRemoteLogging: bool = False): + super().__init__(op._rob) + self._op = op + self._logObj = self.newString() if enableRemoteLogging else None + + def Return(self, *values: RemoteBaseObject | int | float | str | bool | None): + remoteValues = [RemoteBaseObject.ensureRemote(self.rob, value) for value in values] + if len(remoteValues) == 1: + remoteValue = remoteValues[0] + else: + remoteValue = self.newArray() + self.addCompiletimeComment( + f"Created {remoteValue} for returning values {remoteValues}" + ) + for value in remoteValues: + remoteValue.append(value) + if self._op._returnIdOperand is None: + raise RuntimeError("ReturnIdOperand not set not created") + self._op.addToResults(remoteValue) + self.addCompiletimeComment( + f"Returning {remoteValue}" + ) + self._op._returnIdOperand.set(remoteValue.operandId.value) + self.halt() + + def Yield(self, *values: RemoteBaseObject | int | float | str | bool | None): + self.addCompiletimeComment(f"Begin yield {values}") + remoteValues = [RemoteBaseObject.ensureRemote(self.rob, value) for value in values] + if len(remoteValues) == 1: + remoteValue = remoteValues[0] + else: + remoteValue = self.newArray() + self.addCompiletimeComment( + f"Created {remoteValue} for yielding values {remoteValues}" + ) + for value in remoteValues: + remoteValue.append(value) + if self._op._yieldListOperand is None: + raise RuntimeError("YieldIdOperand not set not created") + self._op.addToResults(remoteValue) + self.addCompiletimeComment(f"Yielding {remoteValue}") + self._op._yieldListOperand.append(remoteValue) + + _newObject_RemoteType = TypeVar('_newObject_RemoteType', bound=RemoteBaseObject) + + def _newObject( + self, + RemoteType: Type[_newObject_RemoteType], + value: Any, + static: bool = False + ) -> _newObject_RemoteType: + section = "static" if static else "main" + with self.rob.overrideDefaultSection(section): + obj = RemoteType.createNew(self.rob, value) + if static: + self._op._registerStaticOperand(obj) + return obj + + def newUint(self, value: int = 0, static: bool = False) -> RemoteUint: + return self._newObject(RemoteUint, value, static=static) + + def newInt(self, value: int = 0, static: bool = False) -> RemoteInt: + return self._newObject(RemoteInt, value, static=static) + + def newFloat(self, value: float = 0.0, static: bool = False) -> RemoteFloat: + return self._newObject(RemoteFloat, value, static=static) + + def newString(self, value: str = "", static: bool = False) -> RemoteString: + return self._newObject(RemoteString, value, static=static) + + def newBool(self, value: bool = False, static: bool = False) -> RemoteBool: + return self._newObject(RemoteBool, value, static=static) + + def newGuid(self, value: GUID | str | None = None, static: bool = False) -> RemoteGuid: + if value is None: + realValue = GUID() + elif isinstance(value, str): + realValue = GUID(value) + else: + realValue = value + return self._newObject(RemoteGuid, realValue, static=static) + + def newVariant(self) -> RemoteVariant: + return RemoteVariant.createNew(self.rob) + + def newArray(self) -> RemoteArray: + return RemoteArray.createNew(self.rob) + + def newElement( + self, + value: UIA.IUIAutomationElement | None = None, + static: bool = False + ) -> RemoteElement: + section = "static" if static else "main" + with self.rob.overrideDefaultSection(section): + if value is not None: + obj = self._op.importElement(value) + if static: + self._op._registerStaticOperand(obj) + return obj + else: + return self._newObject(RemoteElement, value, static=static) + + def newTextRange( + self, + value: UIA.IUIAutomationTextRange | None = None, + static: bool = False + ) -> RemoteTextRange: + section = "static" if static else "main" + with self.rob.overrideDefaultSection(section): + if value is not None: + obj = self._op.importTextRange(value) + obj = obj.clone() + if static: + self._op._registerStaticOperand(obj) + return obj + else: + return self._newObject(RemoteTextRange, value, static=static) + + def getOperationStatus(self) -> RemoteInt: + instructionList = self.rob.getDefaultInstructionList() + result = RemoteInt(self.rob, self.rob.requestNewOperandId()) + instructionList.addInstruction( + instructions.GetOperationStatus( + result=result + ) + ) + return result + + def setOperationStatus(self, status: RemoteInt | int): + instructionList = self.rob.getDefaultInstructionList() + instructionList.addInstruction( + instructions.SetOperationStatus( + status=RemoteInt.ensureRemote(self.rob, status) + ) + ) + + _scopeInstructionJustExited: instructions.InstructionBase | None = None + + @contextlib.contextmanager + def ifBlock(self, condition: RemoteBool, silent: bool = False): + instructionList = self.rob.getDefaultInstructionList() + conditionInstruction = instructions.ForkIfFalse( + condition=condition, + branch=RelativeOffset(1), # offset updated after yield + ) + conditionInstructionIndex = instructionList.addInstruction(conditionInstruction) + if not silent: + instructionList.addComment("If block body") + yield + if not silent: + instructionList.addComment("End of if block body") + nextInstructionIndex = instructionList.getInstructionCount() + conditionInstruction.branch = RelativeOffset(nextInstructionIndex - conditionInstructionIndex) + self._scopeInstructionJustExited = conditionInstruction + + @contextlib.contextmanager + def elseBlock(self, silent: bool = False): + scopeInstructionJustExited = self._scopeInstructionJustExited + if not isinstance(scopeInstructionJustExited, instructions.ForkIfFalse): + raise RuntimeError("Else block not directly preceded by If block") + instructionList = self.rob.getDefaultInstructionList() + ifConditionInstruction = cast(instructions.ForkIfFalse, scopeInstructionJustExited) + # add a final jump instruction to the previous if block to skip over the else block. + if not silent: + instructionList.addComment("Jump over else block") + jumpElseInstruction = instructions.Fork(RelativeOffset(1)) # offset updated after yield + jumpElseInstructionIndex = instructionList.addInstruction(jumpElseInstruction) + # increment the false offset of the previous if block to take the new jump instruction into account. + ifConditionInstruction.branch.value += 1 + if not silent: + instructionList.addComment("Else block body") + yield + if not silent: + instructionList.addComment("End of else block body") + # update the jump instruction to jump to the real end of the else block. + nextInstructionIndex = instructionList.getInstructionCount() + jumpElseInstruction.jumpTo = RelativeOffset(nextInstructionIndex - jumpElseInstructionIndex) + self._scopeInstructionJustExited = None + + def continueLoop(self): + instructionList = self.rob.getDefaultInstructionList() + instructionList.addInstruction(instructions.ContinueLoop()) + + def breakLoop(self): + instructionList = self.rob.getDefaultInstructionList() + instructionList.addInstruction(instructions.BreakLoop()) + + @contextlib.contextmanager + def whileBlock(self, conditionBuilderFunc: Callable[[], RemoteBool], silent: bool = False): + instructionList = self.rob.getDefaultInstructionList() + # Add a new loop block instruction to start the while loop + loopBlockInstruction = instructions.NewLoopBlock( + breakBranch=RelativeOffset(1), # offset updated after yield + continueBranch=RelativeOffset(1) + ) + loopBlockInstructionIndex = instructionList.addInstruction(loopBlockInstruction) + # generate the loop condition. + # This must be evaluated lazily via a callable + # because any instructions that produce the condition bool + # must be added inside the loop block, + # so that the condition is fully re-evaluated on each iteration. + condition = conditionBuilderFunc() + with self.ifBlock(condition, silent=True): + # Add the loop body + if not silent: + instructionList.addComment("While block body") + yield + if not silent: + instructionList.addComment("End of while block body") + self.continueLoop() + instructionList.addInstruction(instructions.EndLoopBlock()) + # update the loop break offset to jump to the end of the loop body + nextInstructionIndex = instructionList.getInstructionCount() + loopBlockInstruction.breakBranch = RelativeOffset(nextInstructionIndex - loopBlockInstructionIndex) + self._scopeInstructionJustExited = loopBlockInstruction + + _range_intTypeVar = TypeVar('_range_intTypeVar', bound=RemoteIntBase) + + @remoteContextManager + def forEachNumInRange( + self, + start: _range_intTypeVar | int, + stop: _range_intTypeVar | int, + step: _range_intTypeVar | int = 1 + ) -> Generator[RemoteIntBase, None, None]: + RemoteType: Type[RemoteIntBase] = RemoteInt + for arg in (start, stop, step): + if isinstance(arg, RemoteUint): + RemoteType = RemoteUint + break + remoteStart = cast(RemoteIntBase, RemoteType).ensureRemote(self.rob, cast(RemoteIntBase, start)) + remoteStop = cast(RemoteIntBase, RemoteType).ensureRemote(self.rob, cast(RemoteIntBase, stop)) + remoteStep = cast(RemoteIntBase, RemoteType).ensureRemote(self.rob, cast(RemoteIntBase, step)) + counter = remoteStart.copy() + with self.whileBlock(lambda: counter < remoteStop): + yield cast(RemoteIntBase, counter) + counter += remoteStep + + @remoteContextManager + def forEachItemInArray( + self, + array: RemoteArray + ) -> Generator[RemoteVariant, None, None]: + with self.forEachNumInRange(0, array.size()) as index: + yield array[index] + + @contextlib.contextmanager + def tryBlock(self, silent: bool = False): + instructionList = self.rob.getDefaultInstructionList() + # Add a new try block instruction to start the try block + tryBlockInstruction = instructions.NewTryBlock( + catchBranch=RelativeOffset(1), # offset updated after yield + ) + tryBlockInstructionIndex = instructionList.addInstruction(tryBlockInstruction) + # Add the try block body + if not silent: + instructionList.addComment("Try block body") + yield + if not silent: + instructionList.addComment("End of try block body") + instructionList.addInstruction(instructions.EndTryBlock()) + # update the try block catch offset to jump to the end of the try block body + nextInstructionIndex = instructionList.getInstructionCount() + tryBlockInstruction.catchBranch = RelativeOffset(nextInstructionIndex - tryBlockInstructionIndex) + self._scopeInstructionJustExited = tryBlockInstruction + + @contextlib.contextmanager + def catchBlock(self, silent: bool = False): + scopeInstructionJustExited = self._scopeInstructionJustExited + if not isinstance(scopeInstructionJustExited, instructions.NewTryBlock): + raise RuntimeError("Catch block not directly preceded by Try block") + instructionList = self.rob.getDefaultInstructionList() + tryBlockInstruction = cast(instructions.NewTryBlock, scopeInstructionJustExited) + # add a final jump instruction to the previous try block to skip over the catch block. + if not silent: + instructionList.addComment("Jump over catch block") + jumpCatchInstruction = instructions.Fork( + jumpTo=RelativeOffset(1) # offset updated after yield + ) + jumpCatchInstructionIndex = instructionList.addInstruction(jumpCatchInstruction) + # increment the catch offset of the previous try block to take the new jump instruction into account. + tryBlockInstruction.catchBranch.value += 1 + # fetch the error status that caused the catch + status = self.getOperationStatus() + # reset the error status to 0 + self.setOperationStatus(0) + # Add the catch block body + if not silent: + instructionList.addComment("Catch block body") + yield status + if not silent: + instructionList.addComment("End of catch block body") + # update the jump instruction to jump to the real end of the catch block. + nextInstructionIndex = instructionList.getInstructionCount() + jumpCatchInstruction.jumpTo = RelativeOffset(nextInstructionIndex - jumpCatchInstructionIndex) + self._scopeInstructionJustExited = None + + def halt(self): + instructionList = self.rob.getDefaultInstructionList() + instructionList.addInstruction(instructions.Halt()) + + def logRuntimeMessage(self, *args: str | RemoteBaseObject) -> None: + if self._logObj is None: + return + instructionList = self.rob.getDefaultInstructionList() + logObj = self._logObj + instructionList.addComment("Begin logMessage code") + lastIndex = len(args) - 1 + requiresNewLine = True + for index, arg in enumerate(args): + if index == lastIndex and isinstance(arg, str): + arg += "\n" + requiresNewLine = False + if isinstance(arg, RemoteString): + string = arg + elif isinstance(arg, RemoteBaseObject): + string = arg.stringify() + else: # arg is str + string = self.newString(arg) + logObj += string + if requiresNewLine: + logObj += "\n" + instructionList.addComment("End logMessage code") + + def getLogObject(self) -> RemoteString | None: + return self._logObj + + def addCompiletimeComment(self, comment: str): + instructionList = self.rob.getDefaultInstructionList() + instructionList.addComment(comment) diff --git a/source/UIAHandler/_remoteOps/remoteAlgorithms.py b/source/UIAHandler/_remoteOps/remoteAlgorithms.py new file mode 100644 index 00000000000..2af9e16eb4b --- /dev/null +++ b/source/UIAHandler/_remoteOps/remoteAlgorithms.py @@ -0,0 +1,40 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + +from __future__ import annotations +from collections.abc import Generator +from .remoteFuncWrapper import ( + remoteContextManager +) +from .remoteAPI import RemoteAPI +from .remoteTypes import ( + RemoteIntEnum, + RemoteTextRange +) +from .lowLevel import ( + TextUnit +) + + +@remoteContextManager +def remote_forEachUnitInTextRange( + ra: RemoteAPI, + textRange: RemoteTextRange, + unit: RemoteIntEnum[TextUnit] | TextUnit, + reverse: bool = False +) -> Generator[RemoteTextRange, None, None]: + logicalTextRange = textRange.getLogicalAdapter(reverse) + logicalTempRange = logicalTextRange.clone() + logicalTempRange.end = logicalTempRange.start + with ra.whileBlock(lambda: logicalTempRange.end < logicalTextRange.end): + unitsMoved = logicalTempRange.end.moveByUnit(unit, 1) + endDelta = logicalTempRange.end.compareWith(logicalTextRange.end) + with ra.ifBlock((unitsMoved == 0) | (endDelta > 0)): + logicalTempRange.end = logicalTextRange.end + yield logicalTempRange.textRange + logicalTextRange.start = logicalTempRange.end + with ra.ifBlock((unitsMoved == 0) | (endDelta >= 0)): + ra.breakLoop() + logicalTempRange.start = logicalTempRange.end diff --git a/source/UIAHandler/_remoteOps/remoteFuncWrapper.py b/source/UIAHandler/_remoteOps/remoteFuncWrapper.py new file mode 100644 index 00000000000..54a3c09abe5 --- /dev/null +++ b/source/UIAHandler/_remoteOps/remoteFuncWrapper.py @@ -0,0 +1,114 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + + +from __future__ import annotations +from typing import ( + Generator, + ContextManager, + Callable, + Concatenate, + ParamSpec, + TypeVar +) +import functools +import contextlib +from . import builder + + +_remoteFunc_self = TypeVar('_remoteFunc_self', bound=builder._RemoteBase) +_remoteFunc_paramSpec = ParamSpec('_remoteFunc_paramSpec') +_remoteFunc_return = TypeVar('_remoteFunc_return') + + +class _BaseRemoteFuncWrapper: + + def generateArgsKwargsString(self, *args, **kwargs) -> str: + argsString = ", ".join(map(repr, args)) + kwargsString = ", ".join(f"{key}={repr(val)}" for key, val in kwargs.items()) + return f"({', '.join([argsString, kwargsString])})" + + def _execRawFunc( + self, + func: Callable[Concatenate[_remoteFunc_self, _remoteFunc_paramSpec], _remoteFunc_return], + funcSelf: _remoteFunc_self, + *args: _remoteFunc_paramSpec.args, + **kwargs: _remoteFunc_paramSpec.kwargs + ) -> _remoteFunc_return: + main = funcSelf.rob.getInstructionList('main') + main.addComment( + f"Entering {func.__qualname__}{self.generateArgsKwargsString(*args, **kwargs)}" + ) + res = func(funcSelf, *args, **kwargs) + main.addComment(f"Exiting {func.__qualname__}") + return res + + def __call__( + self, + func: Callable[Concatenate[_remoteFunc_self, _remoteFunc_paramSpec], _remoteFunc_return], + ) -> Callable[Concatenate[_remoteFunc_self, _remoteFunc_paramSpec], _remoteFunc_return]: + @functools.wraps(func) + def wrapper( + funcSelf: _remoteFunc_self, + *args: _remoteFunc_paramSpec.args, + **kwargs: _remoteFunc_paramSpec.kwargs + ) -> _remoteFunc_return: + return self._execRawFunc(func, funcSelf, *args, **kwargs) + return wrapper + + +class RemoteMethodWrapper(_BaseRemoteFuncWrapper): + + _mutable: bool + + def __init__(self, mutable: bool = False): + self._mutable = mutable + + def _execRawFunc( + self, + func: Callable[Concatenate[_remoteFunc_self, _remoteFunc_paramSpec], _remoteFunc_return], + funcSelf: _remoteFunc_self, + *args: _remoteFunc_paramSpec.args, + **kwargs: _remoteFunc_paramSpec.kwargs + ) -> _remoteFunc_return: + if self._mutable and not funcSelf._mutable: + raise RuntimeError(f"{funcSelf.__class__.__name__} is not mutable") + return super()._execRawFunc(func, funcSelf, *args, **kwargs) + + +class RemoteContextManager(_BaseRemoteFuncWrapper): + + def __call__( + self, + func: Callable[ + Concatenate[_remoteFunc_self, _remoteFunc_paramSpec], Generator[_remoteFunc_return, None, None] + ] + ) -> Callable[Concatenate[_remoteFunc_self, _remoteFunc_paramSpec], ContextManager[_remoteFunc_return]]: + contextFunc = contextlib.contextmanager(func) + return super().__call__(contextFunc) + + @contextlib.contextmanager + def _execRawFunc( + self, + func: Callable[Concatenate[_remoteFunc_self, _remoteFunc_paramSpec], ContextManager[_remoteFunc_return]], + funcSelf: _remoteFunc_self, + *args: _remoteFunc_paramSpec.args, + **kwargs: _remoteFunc_paramSpec.kwargs + ) -> Generator[_remoteFunc_return, None, None]: + main = funcSelf.rob.getInstructionList('main') + main.addComment( + f"Entering context manager {func.__qualname__}{self.generateArgsKwargsString(*args, **kwargs)}" + ) + with func(funcSelf, *args, **kwargs) as val: + main.addComment("Yielding to outer scope") + yield val + main.addComment(f"Reentering context manager {func.__qualname__}") + funcSelf.rob.getInstructionList('main').addComment(f"Exiting context manager {func.__qualname__}") + + +remoteFunc = _BaseRemoteFuncWrapper() +remoteMethod = RemoteMethodWrapper() +remoteMethod_mutable = RemoteMethodWrapper(mutable=True) +remoteContextManager = RemoteContextManager() diff --git a/source/UIAHandler/_remoteOps/remoteTypes/__init__.py b/source/UIAHandler/_remoteOps/remoteTypes/__init__.py new file mode 100644 index 00000000000..ed6f479fc95 --- /dev/null +++ b/source/UIAHandler/_remoteOps/remoteTypes/__init__.py @@ -0,0 +1,739 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + + +from __future__ import annotations +from typing import ( + Type, + Any, + Self, + ParamSpec, + Iterable, + Generic, + TypeVar, + cast +) +import ctypes +from ctypes import ( + _SimpleCData, + c_long, + c_ulong, + c_bool, +) +from comtypes import ( + GUID, + IUnknown, + COMError +) +import enum +from UIAHandler import UIA +from .. import lowLevel +from .. import instructions +from .. import builder +from ..remoteFuncWrapper import ( + remoteMethod, + remoteMethod_mutable +) +from .. import operation + + +_remoteFunc_self = TypeVar('_remoteFunc_self', bound=builder._RemoteBase) +_remoteFunc_paramSpec = ParamSpec('_remoteFunc_paramSpec') +_remoteFunc_return = TypeVar('_remoteFunc_return') + + +LocalTypeVar = TypeVar('LocalTypeVar') + + +class RemoteBaseObject(builder.Operand, Generic[LocalTypeVar]): + + _IsTypeInstruction: Type[builder.InstructionBase] + LocalType: Type[LocalTypeVar] | None = None + _initialValue: LocalTypeVar | None = None + _executionResult: operation.ExecutionResult | None = None + + def _setExecutionResult(self, executionResult: operation.ExecutionResult): + self._executionResult = executionResult + + def __bool__(self) -> bool: + raise TypeError(f"Cannot convert {self.__class__.__name__} to bool") + + def _generateDefaultInitialValue(self) -> LocalTypeVar: + if self.LocalType is None: + raise TypeError("LocalType not set") + return self.LocalType() + + def _generateInitInstructions(self) -> Iterable[instructions.InstructionBase]: + raise NotImplementedError() + + def _initOperand(self, initialValue: LocalTypeVar | None = None, const: bool =False): + if initialValue is not None: + if self.LocalType is None: + raise TypeError(f"{type(self).__name__} does not support an initial value") + if not isinstance(initialValue, self.LocalType): + raise TypeError( + f"initialValue must be of type {self.LocalType.__name__} " + f"not {type(initialValue).__name__}" + ) + self._initialValue = initialValue + self._mutable = not const + instructionList = self.rob.getDefaultInstructionList() + for instruction in self._generateInitInstructions(): + instructionList.addInstruction(instruction) + + @classmethod + def createNew( + cls, + rob: builder.RemoteOperationBuilder, + initialValue: LocalTypeVar | None = None, + operandId: lowLevel.OperandId | None = None, + const: bool = False + ) -> Self: + if operandId is None: + operandId = rob.requestNewOperandId() + obj = cls(rob, operandId) + obj._initOperand(initialValue=initialValue, const=const) + return obj + + @classmethod + def ensureRemote(cls, rob: builder.RemoteOperationBuilder, obj: Self | LocalTypeVar) -> Self: + if isinstance(obj, cls): + if obj.rob is not rob: + raise RuntimeError(f"Object {obj} is not bound to the given RemoteOperationBuilder") + return obj + if cls.LocalType is not None: + if not isinstance(obj, cls.LocalType): + raise TypeError(f"obj must be of type {cls.LocalType.__name__} not {type(obj).__name__}") + RemoteType = cls + else: + RemoteType = getRemoteTypeForLocalType(type(obj)) + if not issubclass(RemoteType, cls): + raise TypeError( + f"The RemoteType of {type(obj).__name__} is {RemoteType.__name__} " + f"which is not a subclass of {cls.__name__}" + ) + cacheKey = (RemoteType, obj) + cachedRemoteObj = rob._remotedArgCache.get(cacheKey) + if cachedRemoteObj is not None: + if not isinstance(cachedRemoteObj, RemoteType): + raise RuntimeError(f"Cache entry for {cacheKey} is not of type {RemoteType.__name__}") + rob.getDefaultInstructionList().addComment( + f"Using cached {cachedRemoteObj} for constant value {repr(obj)}" + ) + return cast(RemoteType, cachedRemoteObj) + with rob.overrideDefaultSection('const'): + remoteObj = RemoteType.createNew(rob, obj, const=True) + rob.getDefaultInstructionList().addComment( + f"Using cached {remoteObj} for constant value {repr(obj)}" + ) + rob._remotedArgCache[cacheKey] = remoteObj + return remoteObj + + @property + def initialValue(self) -> LocalTypeVar: + if self._initialValue is not None: + return self._initialValue + return self._generateDefaultInitialValue() + + @property + def isLocalValueAvailable(self) -> bool: + return self._executionResult is not None and self._executionResult.hasOperand(self.operandId) + + @property + def localValue(self) -> LocalTypeVar: + if self._executionResult is None: + raise RuntimeError("Operation not executed") + value = self._executionResult.getOperand(self.operandId) + return cast(LocalTypeVar, value) + + @remoteMethod_mutable + def set(self, other: Self | LocalTypeVar): + self.rob.getDefaultInstructionList().addInstruction( + instructions.Set( + target=self, + value=type(self).ensureRemote(self.rob, other) + ) + ) + + @remoteMethod + def copy(self) -> Self: + copy = type(self)(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.Set( + target=copy, + value=self + ) + ) + return copy + + def _doCompare(self, comparisonType: lowLevel.ComparisonType, other: Self | LocalTypeVar) -> RemoteBool: + result = RemoteBool(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.Compare( + result=result, + left=self, + right=type(self).ensureRemote(self.rob, other), + comparisonType=comparisonType + ) + ) + return result + + @remoteMethod + def __eq__(self, other: Self | LocalTypeVar) -> RemoteBool: + return self._doCompare(lowLevel.ComparisonType.Equal, other) + + @remoteMethod + def __ne__(self, other: Self | LocalTypeVar) -> RemoteBool: + return self._doCompare(lowLevel.ComparisonType.NotEqual, other) + + @remoteMethod + def stringify(self) -> RemoteString: + result = RemoteString(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.Stringify( + result=result, + target=self + ) + ) + return result + + +class RemoteVariant(RemoteBaseObject): + + def _generateInitInstructions(self) -> Iterable[instructions.InstructionBase]: + yield instructions.NewNull( + result=self + ) + + def _isType(self, RemoteClass: Type[RemoteBaseObject]) -> RemoteBool: + if not issubclass(RemoteClass, RemoteBaseObject): + raise TypeError("remoteClass must be a subclass of RemoteBaseObject") + result = RemoteBool(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + RemoteClass._IsTypeInstruction( + result=result, + target=self + ) + ) + return result + + @remoteMethod + def isNull(self) -> RemoteBool: + return self._isType(RemoteNull) + + @remoteMethod + def isBool(self) -> RemoteBool: + return self._isType(RemoteBool) + + @remoteMethod + def isInt(self) -> RemoteBool: + return self._isType(RemoteInt) + + @remoteMethod + def isUint(self) -> RemoteBool: + return self._isType(RemoteUint) + + @remoteMethod + def isFloat(self) -> RemoteBool: + return self._isType(RemoteFloat) + + @remoteMethod + def isString(self) -> RemoteBool: + return self._isType(RemoteString) + + @remoteMethod + def isGuid(self) -> RemoteBool: + return self._isType(RemoteGuid) + + @remoteMethod + def isArray(self) -> RemoteBool: + return self._isType(RemoteArray) + + @remoteMethod + def isElement(self) -> RemoteBool: + return self._isType(RemoteElement) + + _TV_asType = TypeVar('_TV_asType', bound=RemoteBaseObject) + + def asType(self, remoteClass: Type[_TV_asType]) -> _TV_asType: + return remoteClass(self.rob, self.operandId) + + +class RemoteNull(RemoteBaseObject): + _IsTypeInstruction = instructions.IsNull + + def _generateInitInstructions(self,) -> Iterable[instructions.InstructionBase]: + yield instructions.NewNull( + result=self + ) + + +class RemoteIntegral(RemoteBaseObject[LocalTypeVar], Generic[LocalTypeVar]): + + _NewInstruction: Type[builder.InstructionBase] + _ctype: Type[_SimpleCData] + + def _generateInitInstructions(self) -> Iterable[instructions.InstructionBase]: + yield self._NewInstruction( + result=self, + value=self._ctype(self.initialValue) + ) + + +class RemoteBool(RemoteIntegral[bool]): + _IsTypeInstruction = instructions.IsBool + _NewInstruction = instructions.NewBool + _ctype = c_bool + LocalType = bool + _defaultInitialValue = False + + @remoteMethod + def inverse(self) -> RemoteBool: + result = RemoteBool(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.BoolNot( + result=result, + target=self + ) + ) + return result + + @remoteMethod + def __and__(self, other: Self | bool) -> RemoteBool: + result = type(self)(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.BoolAnd( + result=result, + left=self, + right=RemoteBool.ensureRemote(self.rob, other) + ) + ) + return result + + @remoteMethod + def __rand__(self, other: Self | bool) -> RemoteBool: + result = type(self)(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.BoolAnd( + result=result, + left=self, + right=RemoteBool.ensureRemote(self.rob, other) + ) + ) + return result + + @remoteMethod + def __or__(self, other: Self | bool) -> RemoteBool: + result = type(self)(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.BoolOr( + result=result, + left=self, + right=RemoteBool.ensureRemote(self.rob, other) + ) + ) + return result + + @remoteMethod + def __ror__(self, other: Self | bool) -> RemoteBool: + result = type(self)(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.BoolOr( + result=result, + left=self, + right=RemoteBool.ensureRemote(self.rob, other) + ) + ) + return result + + +class RemoteNumber(RemoteIntegral[LocalTypeVar], Generic[LocalTypeVar]): + + @remoteMethod + def __gt__(self, other: Self | LocalTypeVar) -> RemoteBool: + return self._doCompare(lowLevel.ComparisonType.GreaterThan, other) + + @remoteMethod + def __lt__(self, other: Self | LocalTypeVar) -> RemoteBool: + return self._doCompare(lowLevel.ComparisonType.LessThan, other) + + @remoteMethod + def __ge__(self, other: Self | LocalTypeVar) -> RemoteBool: + return self._doCompare(lowLevel.ComparisonType.GreaterThanOrEqual, other) + + @remoteMethod + def __le__(self, other: Self | LocalTypeVar) -> RemoteBool: + return self._doCompare(lowLevel.ComparisonType.LessThanOrEqual, other) + + @remoteMethod + def __add__(self, other: Self | LocalTypeVar) -> Self: + result = type(self)(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.BinaryAdd( + result=result, + left=self, + right=type(self).ensureRemote(self.rob, other) + ) + ) + return result + + @remoteMethod + def __sub__(self, other: Self | LocalTypeVar) -> Self: + result = type(self)(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.BinarySubtract( + result=result, + left=self, + right=type(self).ensureRemote(self.rob, other) + ) + ) + return result + + @remoteMethod + def __mul__(self, other: Self | LocalTypeVar) -> Self: + result = type(self)(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.BinaryMultiply( + result=result, + left=self, + right=type(self).ensureRemote(self.rob, other) + ) + ) + return result + + @remoteMethod + def __truediv__(self, other: Self | LocalTypeVar) -> Self: + result = type(self)(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.BinaryDivide( + result=result, + left=self, + right=type(self).ensureRemote(self.rob, other) + ) + ) + return result + + @remoteMethod + def __mod__(self, other: Self | LocalTypeVar) -> Self: + return (self - (self / other) * other) + + @remoteMethod + def __radd__(self, other: Self | LocalTypeVar) -> Self: + result = type(self)(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.BinaryAdd( + result=result, + left=self, + right=type(self).ensureRemote(self.rob, other) + ) + ) + return result + + @remoteMethod + def __rsub__(self, other: Self | LocalTypeVar) -> Self: + result = type(self)(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.BinarySubtract( + result=result, + left=self, + right=type(self).ensureRemote(self.rob, other) + ) + ) + return result + + @remoteMethod + def __rmul__(self, other: Self | LocalTypeVar) -> Self: + result = type(self)(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.BinaryMultiply( + result=result, + left=self, + right=type(self).ensureRemote(self.rob, other) + ) + ) + return result + + @remoteMethod + def __rtruediv__(self, other: Self | LocalTypeVar) -> Self: + result = type(self)(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.BinaryDivide( + result=result, + left=self, + right=type(self).ensureRemote(self.rob, other) + ) + ) + return result + + @remoteMethod + def __rmod__(self, other: Self | LocalTypeVar) -> Self: + return (other - (other / self) * self) + + @remoteMethod_mutable + def __iadd__(self, other: Self | LocalTypeVar) -> Self: + self.rob.getDefaultInstructionList().addInstruction( + instructions.InplaceAdd( + target=self, + value=type(self).ensureRemote(self.rob, other) + ) + ) + return self + + @remoteMethod_mutable + def __isub__(self, other: Self | LocalTypeVar) -> Self: + self.rob.getDefaultInstructionList().addInstruction( + instructions.InplaceSubtract( + target=self, + value=type(self).ensureRemote(self.rob, other) + ) + ) + return self + + @remoteMethod_mutable + def __imul__(self, other: Self | LocalTypeVar) -> Self: + self.rob.getDefaultInstructionList().addInstruction( + instructions.InplaceMultiply( + target=self, + value=type(self).ensureRemote(self.rob, other) + ) + ) + return self + + @remoteMethod_mutable + def __itruediv__(self, other: Self | LocalTypeVar) -> Self: + self.rob.getDefaultInstructionList().addInstruction( + instructions.InplaceDivide( + target=self, + value=type(self).ensureRemote(self.rob, other) + ) + ) + return self + + @remoteMethod_mutable + def __imod__(self, other: Self | LocalTypeVar) -> Self: + self -= (self / other) * other + return self + + +class RemoteIntBase(RemoteNumber[int]): + __floordiv__ = RemoteNumber.__truediv__ + __rfloordiv__ = RemoteNumber.__rtruediv__ + __ifloordiv__ = RemoteNumber.__itruediv__ + + +class RemoteUint(RemoteIntBase): + _IsTypeInstruction = instructions.IsUint + _NewInstruction = instructions.NewUint + _ctype = c_ulong + LocalType = int + _defaultInitialValue = 0 + + +class RemoteInt(RemoteIntBase): + _IsTypeInstruction = instructions.IsInt + _NewInstruction = instructions.NewInt + _ctype = c_long + LocalType = int + _defaultInitialValue = 0 + + +class RemoteFloat(RemoteNumber[float]): + _IsTypeInstruction = instructions.IsFloat + _NewInstruction = instructions.NewFloat + _ctype = ctypes.c_double + LocalType = float + _defaultInitialValue = 0.0 + + +class RemoteString(RemoteBaseObject[str]): + _IsTypeInstruction = instructions.IsString + LocalType = str + _defaultInitialValue = "" + + def _generateInitInstructions(self) -> Iterable[instructions.InstructionBase]: + initialValue = self.initialValue + stringLen = len(initialValue) + 1 + stringVal = ctypes.create_unicode_buffer(initialValue) + yield instructions.NewString( + result=self, + length=c_ulong(stringLen), + value=stringVal + ) + + def _concat(self, other: Self | str) -> Self: + result = type(self)(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.StringConcat( + result=result, + left=self, + right=type(self).ensureRemote(self.rob, other) + ) + ) + return result + + @remoteMethod + def __add__(self, other: Self | str) -> RemoteString: + return self._concat(other) + + @remoteMethod + def __radd__(self, other: Self | str) -> RemoteString: + return self._concat(other) + + @remoteMethod_mutable + def __iadd__(self, other: Self | str) -> Self: + self.rob.getDefaultInstructionList().addInstruction( + instructions.StringConcat( + result=self, + left=self, + right=type(self).ensureRemote(self.rob, other) + ) + ) + return self + + @remoteMethod_mutable + def set(self, other: Self | str): + self.rob.getDefaultInstructionList().addInstruction( + instructions.NewString( + result=self, + length=c_ulong(1), + value=ctypes.create_unicode_buffer("") + ) + ) + self += other + + @remoteMethod + def copy(self) -> Self: + copy = type(self)(self.rob, self.rob.requestNewOperandId()) + copy += self + return copy + + +class RemoteArray(RemoteBaseObject): + + _LOCAL_COM_INTERFACES = [ + UIA.IUIAutomationElement, + UIA.IUIAutomationTextRange + ] + + def _correctCOMPointers(self, *items: object) -> list: + correctedItems = [] + for i, item in enumerate(items): + if isinstance(item, IUnknown): + for interface in self._LOCAL_COM_INTERFACES: + try: + item = item.QueryInterface(interface) + break + except COMError: + pass + elif isinstance(item, tuple): + item = self._correctCOMPointers(*item) + correctedItems.append(item) + return correctedItems + + @property + def localValue(self) -> list: + items = super().localValue + return self._correctCOMPointers(*items) + + def _generateInitInstructions(self) -> Iterable[instructions.InstructionBase]: + yield instructions.NewArray( + result=self + ) + + @remoteMethod + def __getitem__(self, index: RemoteIntBase | int) -> RemoteVariant: + result = RemoteVariant(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.ArrayGetAt( + result=result, + target=self, + index=RemoteIntBase.ensureRemote(self.rob, index) + ) + ) + return result + + @remoteMethod + def size(self) -> RemoteUint: + result = RemoteUint(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.ArraySize( + result=result, + target=self + ) + ) + return result + + @remoteMethod_mutable + def append(self, value: RemoteBaseObject | int | float | str) -> None: + self.rob.getDefaultInstructionList().addInstruction( + instructions.ArrayAppend( + target=self, + value=RemoteBaseObject.ensureRemote(self.rob, value) + ) + ) + + @remoteMethod_mutable + def __setitem__( + self, + index: RemoteIntBase | int, + value: RemoteBaseObject | int | float | str + ) -> None: + self.rob.getDefaultInstructionList().addInstruction( + instructions.ArraySetAt( + target=self, + index=RemoteIntBase.ensureRemote(self.rob, index), + value=RemoteBaseObject.ensureRemote(self.rob, value) + ) + ) + + @remoteMethod_mutable + def remove(self, index: RemoteIntBase | int) -> None: + self.rob.getDefaultInstructionList().addInstruction( + instructions.ArrayRemoveAt( + target=self, + index=RemoteIntBase.ensureRemote(self.rob, index) + ) + ) + + +class RemoteGuid(RemoteBaseObject[GUID]): + _IsTypeInstruction = instructions.IsGuid + LocalType = GUID + + @property + def _defaultInitialValue(self) -> GUID: + return GUID() + + def _generateInitInstructions(self) -> Iterable[instructions.InstructionBase]: + yield instructions.NewGuid( + result=self, + value=self.initialValue + ) + + +def getRemoteTypeForLocalType(LocalType: Type[object]) -> Type[RemoteBaseObject]: + if issubclass(LocalType, enum.IntEnum): + return RemoteIntEnum + elif issubclass(LocalType, bool): + return RemoteBool + elif issubclass(LocalType, int): + return RemoteInt + elif issubclass(LocalType, float): + return RemoteFloat + elif issubclass(LocalType, str): + return RemoteString + elif issubclass(LocalType, GUID): + return RemoteGuid + else: + raise TypeError(f"No mapping for type {LocalType.__name__}") + + +# Import some more complex types after defining the base classes to avoid circular imports +# flake8: noqa: F401 +# flake8: noqa: E402 +from .intEnum import RemoteIntEnum +from .extensionTarget import RemoteExtensionTarget +from .element import RemoteElement +from .textRange import RemoteTextRange diff --git a/source/UIAHandler/_remoteOps/remoteTypes/element.py b/source/UIAHandler/_remoteOps/remoteTypes/element.py new file mode 100644 index 00000000000..ccd60f75785 --- /dev/null +++ b/source/UIAHandler/_remoteOps/remoteTypes/element.py @@ -0,0 +1,95 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + + +from __future__ import annotations +from typing import ( + cast +) +from ctypes import ( + POINTER +) +from UIAHandler import UIA +from .. import lowLevel +from .. import instructions +from ..remoteFuncWrapper import ( + remoteMethod +) +from . import ( + RemoteExtensionTarget, + RemoteIntEnum, + RemoteBool, + RemoteVariant +) + + +class RemoteElement(RemoteExtensionTarget[POINTER(UIA.IUIAutomationElement)]): + """ + Represents a remote UI Automation element. + Allows for navigation and property retrieval. + """ + + _IsTypeInstruction = instructions.IsElement + LocalType = POINTER(UIA.IUIAutomationElement) + + def _initOperand(self, initialValue: None = None, const: bool = False): + if initialValue is not None: + raise TypeError("Cannot initialize RemoteElement with an initial value.") + return super()._initOperand() + + @property + def localValue(self) -> UIA.IUIAutomationElement: + value = super().localValue + if value is None: + return POINTER(UIA.IUIAutomationElement)() + return cast(UIA.IUIAutomationElement, value.QueryInterface(UIA.IUIAutomationElement)) + + @remoteMethod + def getPropertyValue( + self, + propertyId: RemoteIntEnum[lowLevel.PropertyId] | lowLevel.PropertyId, + ignoreDefault: RemoteBool | bool = False + ) -> RemoteVariant: + result = RemoteVariant(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.ElementGetPropertyValue( + result=result, + target=self, + propertyId=RemoteIntEnum.ensureRemote(self.rob, propertyId), + ignoreDefault=RemoteBool.ensureRemote(self.rob, ignoreDefault) + ) + ) + return result + + def _navigate(self, navigationDirection: lowLevel.NavigationDirection) -> RemoteElement: + result = RemoteElement(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.ElementNavigate( + result=result, + target=self, + direction=RemoteIntEnum.ensureRemote(self.rob, navigationDirection) + ) + ) + return result + + @remoteMethod + def getParentElement(self) -> RemoteElement: + return self._navigate(lowLevel.NavigationDirection.Parent) + + @remoteMethod + def getFirstChildElement(self) -> RemoteElement: + return self._navigate(lowLevel.NavigationDirection.FirstChild) + + @remoteMethod + def getLastChildElement(self) -> RemoteElement: + return self._navigate(lowLevel.NavigationDirection.LastChild) + + @remoteMethod + def getNextSiblingElement(self) -> RemoteElement: + return self._navigate(lowLevel.NavigationDirection.NextSibling) + + @remoteMethod + def getPreviousSiblingElement(self) -> RemoteElement: + return self._navigate(lowLevel.NavigationDirection.PreviousSibling) diff --git a/source/UIAHandler/_remoteOps/remoteTypes/extensionTarget.py b/source/UIAHandler/_remoteOps/remoteTypes/extensionTarget.py new file mode 100644 index 00000000000..21cec7d7d3b --- /dev/null +++ b/source/UIAHandler/_remoteOps/remoteTypes/extensionTarget.py @@ -0,0 +1,74 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + + +from __future__ import annotations +from typing import ( + Iterable, + Generic +) +from ctypes import ( + c_ulong, +) +from comtypes import ( + GUID +) +from .. import instructions +from ..remoteFuncWrapper import ( + remoteMethod, + remoteMethod_mutable +) +from . import ( + LocalTypeVar, + RemoteBaseObject, + RemoteVariant, + RemoteBool, + RemoteGuid, +) + + +class RemoteExtensionTarget(RemoteBaseObject[LocalTypeVar], Generic[LocalTypeVar]): + """ + Represents a remote object that supports UI Automation custom extensions. + Including checking for the existence of extensions and + calling extensions. + """ + + def _generateInitInstructions(self) -> Iterable[instructions.InstructionBase]: + yield instructions.NewNull( + result=self + ) + + @remoteMethod + def isNull(self): + variant = RemoteVariant(self.rob, self.operandId) + return variant.isNull() + + @remoteMethod + def isExtensionSupported(self, extensionId: RemoteGuid | GUID) -> RemoteBool: + result = RemoteBool(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.IsExtensionSupported( + result=result, + target=self, + extensionId=RemoteGuid.ensureRemote(self.rob, extensionId) + ) + ) + return result + + @remoteMethod_mutable + def callExtension( + self, + extensionId: RemoteGuid | GUID, + *params: RemoteBaseObject | int | float | str + ) -> None: + self.rob.getDefaultInstructionList().addInstruction( + instructions.CallExtension( + target=self, + extensionId=RemoteGuid.ensureRemote(self.rob, extensionId), + argCount=c_ulong(len(params)), + arguments=[RemoteBaseObject.ensureRemote(self.rob, param) for param in params] + ) + ) diff --git a/source/UIAHandler/_remoteOps/remoteTypes/intEnum.py b/source/UIAHandler/_remoteOps/remoteTypes/intEnum.py new file mode 100644 index 00000000000..091324166aa --- /dev/null +++ b/source/UIAHandler/_remoteOps/remoteTypes/intEnum.py @@ -0,0 +1,95 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + + +from __future__ import annotations +from typing import ( + Type, + Any, + Self, + Generic, + TypeVar, + cast, +) +from ctypes import ( + _SimpleCData, + c_long +) +import enum +from .. import builder +from . import ( + RemoteInt, + remoteMethod +) + + +class c_long_enum(c_long): + _enumType: Type[enum.IntEnum] + + def __repr__(self): + return f"{c_long.__name__} enum {repr(self._enumType(self.value))}" + + +_ctypeIntEnumCache: dict[Type[enum.IntEnum], Type[_SimpleCData]] = {} + + +def _makeCtypeIntEnum(enumType: Type[enum.IntEnum]) -> Type[_SimpleCData]: + cachedCls = _ctypeIntEnumCache.get(enumType) + if cachedCls is not None: + return cachedCls + + class cls(c_long_enum): + _enumType = enumType + + cls.__name__ = f"{cls.__name__}_{enumType.__name__}" + cast(Type[_SimpleCData], cls) + _ctypeIntEnumCache[enumType] = cls + return cls + + +_RemoteEnumCache: dict[Type[enum.IntEnum], Type[RemoteInt]] = {} + + +def _makeRemoteEnum(enumType: Type[enum.IntEnum]) -> Type[RemoteInt]: + cachedCls = _RemoteEnumCache.get(enumType) + if cachedCls is not None: + return cachedCls + + class cls(RemoteInt): + LocalType = enumType + _ctype = _makeCtypeIntEnum(enumType) + + cls.__name__ = f"RemoteEnum_{enumType.__name__}" + cast(Type[RemoteInt], cls) + _RemoteEnumCache[enumType] = cls + return cls + + +_RemoteIntEnum_LocalTypeVar = TypeVar('_RemoteIntEnum_LocalTypeVar', bound=enum.IntEnum) + + +class RemoteIntEnum(RemoteInt, Generic[_RemoteIntEnum_LocalTypeVar]): + localType = enum.IntEnum + _enumType: _RemoteIntEnum_LocalTypeVar + + def _initOperand(self, initialValue: _RemoteIntEnum_LocalTypeVar, const: bool = False): + if not isinstance(initialValue, enum.IntEnum): + raise TypeError(f"initialValue must be of type {enum.IntEnum.__name__} not {type(initialValue).__name__}") + self.LocalType = type(initialValue) + self._ctype = _makeCtypeIntEnum(type(initialValue)) + super()._initOperand(initialValue=initialValue, const=const) + + @classmethod + def ensureRemote( + cls, + rob: builder.RemoteOperationBuilder, + obj: RemoteIntEnum[_RemoteIntEnum_LocalTypeVar] | _RemoteIntEnum_LocalTypeVar + ) -> RemoteIntEnum[_RemoteIntEnum_LocalTypeVar]: + remoteObj = super().ensureRemote(rob, cast(Any, obj)) + return cast(RemoteIntEnum[_RemoteIntEnum_LocalTypeVar], remoteObj) + + @remoteMethod + def set(self, other: Self | _RemoteIntEnum_LocalTypeVar): + super().set(other) diff --git a/source/UIAHandler/_remoteOps/remoteTypes/textRange.py b/source/UIAHandler/_remoteOps/remoteTypes/textRange.py new file mode 100644 index 00000000000..2f18f1a89e2 --- /dev/null +++ b/source/UIAHandler/_remoteOps/remoteTypes/textRange.py @@ -0,0 +1,273 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2023-2024 NV Access Limited + + +from __future__ import annotations +from typing import ( + cast +) +from ctypes import ( + POINTER +) +from UIAHandler import UIA +from .. import lowLevel +from .. import instructions +from .. import builder +from ..remoteFuncWrapper import ( + remoteMethod, + remoteMethod_mutable +) +from . import ( + RemoteVariant, + RemoteBool, + RemoteInt, + RemoteIntEnum, + RemoteString, + RemoteExtensionTarget, + RemoteElement, +) + + +class RemoteTextRange(RemoteExtensionTarget[POINTER(UIA.IUIAutomationTextRange)]): + """ + Represents a remote UI Automation text range. + """ + + LocalType = POINTER(UIA.IUIAutomationTextRange) + + def _initOperand(self, initialValue: None = None, const: bool = False): + if initialValue is not None: + raise TypeError("Cannot initialize RemoteTextRange with an initial value.") + return super()._initOperand() + + @property + def localValue(self) -> UIA.IUIAutomationTextRange: + value = super().localValue + if value is None: + return POINTER(UIA.IUIAutomationTextRange)() + return cast(UIA.IUIAutomationTextRange, value.QueryInterface(UIA.IUIAutomationTextRange)) + + @remoteMethod + def clone(self) -> RemoteTextRange: + result = RemoteTextRange(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.TextRangeClone( + result=result, + target=self + ) + ) + return result + + @remoteMethod + def getEnclosingElement(self) -> RemoteElement: + result = RemoteElement(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.TextRangeGetEnclosingElement( + result=result, + target=self + ) + ) + return result + + @remoteMethod + def getText(self, maxLength: RemoteInt | int) -> RemoteString: + result = RemoteString(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.TextRangeGetText( + result=result, + target=self, + maxLength=RemoteInt.ensureRemote(self.rob, maxLength) + ) + ) + return result + + @remoteMethod_mutable + def expandToEnclosingUnit(self, unit: RemoteIntEnum[lowLevel.TextUnit] | lowLevel.TextUnit): + self.rob.getDefaultInstructionList().addInstruction( + instructions.TextRangeExpandToEnclosingUnit( + target=self, + unit=RemoteIntEnum.ensureRemote(self.rob, unit) + ) + ) + + @remoteMethod_mutable + def moveEndpointByUnit( + self, + endpoint: RemoteIntEnum[lowLevel.TextPatternRangeEndpoint] | lowLevel.TextPatternRangeEndpoint, + unit: RemoteIntEnum[lowLevel.TextUnit] | lowLevel.TextUnit, + count: RemoteInt | int + ) -> RemoteInt: + result = RemoteInt(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.TextRangeMoveEndpointByUnit( + result=result, + target=self, + endpoint=RemoteIntEnum.ensureRemote(self.rob, endpoint), + unit=RemoteIntEnum.ensureRemote(self.rob, unit), + count=RemoteInt.ensureRemote(self.rob, count) + ) + ) + return result + + @remoteMethod_mutable + def moveEndpointByRange( + self, + srcEndpoint: RemoteIntEnum[lowLevel.TextPatternRangeEndpoint] | lowLevel.TextPatternRangeEndpoint, + otherRange: RemoteTextRange, + otherEndpoint: RemoteIntEnum[lowLevel.TextPatternRangeEndpoint] | lowLevel.TextPatternRangeEndpoint + ): + self.rob.getDefaultInstructionList().addInstruction( + instructions.TextRangeMoveEndpointByRange( + target=self, + srcEndpoint=RemoteIntEnum.ensureRemote(self.rob, srcEndpoint), + otherRange=otherRange, + otherEndpoint=RemoteIntEnum.ensureRemote(self.rob, otherEndpoint) + ) + ) + + @remoteMethod + def getAttributeValue( + self, + attributeId: RemoteIntEnum[lowLevel.AttributeId] | lowLevel.AttributeId + ) -> RemoteVariant: + result = RemoteVariant(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.TextRangeGetAttributeValue( + result=result, + target=self, + attributeId=RemoteIntEnum.ensureRemote(self.rob, attributeId) + ) + ) + return result + + @remoteMethod + def compareEndpoints( + self, + thisEndpoint: RemoteIntEnum[lowLevel.TextPatternRangeEndpoint] | lowLevel.TextPatternRangeEndpoint, + otherRange: RemoteTextRange, + otherEndpoint: RemoteIntEnum[lowLevel.TextPatternRangeEndpoint] | lowLevel.TextPatternRangeEndpoint + ) -> RemoteInt: + result = RemoteInt(self.rob, self.rob.requestNewOperandId()) + self.rob.getDefaultInstructionList().addInstruction( + instructions.TextRangeCompareEndpoints( + result=result, + target=self, + thisEndpoint=RemoteIntEnum.ensureRemote(self.rob, thisEndpoint), + otherRange=otherRange, + otherEndpoint=RemoteIntEnum.ensureRemote(self.rob, otherEndpoint) + ) + ) + return result + + def getLogicalAdapter(self, reverse: bool = False) -> RemoteTextRangeLogicalAdapter: + obj = RemoteTextRangeLogicalAdapter(self.rob, self, reverse=reverse) + return obj + + +class _RemoteTextRangeEndpoint(builder._RemoteBase): + + def __init__( + self, + rob: builder.RemoteOperationBuilder, + textRangeLA: RemoteTextRangeLogicalAdapter, + isStart: bool + ): + super().__init__(rob) + self._la = textRangeLA + self._endpoint = ( + lowLevel.TextPatternRangeEndpoint.Start + if isStart ^ self.isReversed + else lowLevel.TextPatternRangeEndpoint.End + ) + + @property + def textRange(self: _RemoteTextRangeEndpoint) -> RemoteTextRange: + return self._la.textRange + + @property + def isReversed(self: _RemoteTextRangeEndpoint) -> bool: + return self._la.isReversed + + @property + def endpoint(self: _RemoteTextRangeEndpoint) -> lowLevel.TextPatternRangeEndpoint: + return self._endpoint + + def compareWith(self, other: _RemoteTextRangeEndpoint) -> RemoteInt: + res = self.textRange.compareEndpoints(self.endpoint, other.textRange, other.endpoint) + if self.isReversed: + res *= -1 + return res + + def moveTo(self, other: _RemoteTextRangeEndpoint): + self.textRange.moveEndpointByRange(self.endpoint, other.textRange, other.endpoint) + + def moveByUnit( + self, + unit: RemoteIntEnum[lowLevel.TextUnit] | lowLevel.TextUnit, + count: RemoteInt | int + ) -> RemoteInt: + realCount = (count * -1) if self.isReversed else count + res = self.textRange.moveEndpointByUnit(self.endpoint, unit, realCount) + return res + + def __lt__(self, other: _RemoteTextRangeEndpoint) -> RemoteBool: + return self.compareWith(other).__lt__(0) + + def __le__(self, other: _RemoteTextRangeEndpoint) -> RemoteBool: + return self.compareWith(other).__le__(0) + + def __gt__(self, other: _RemoteTextRangeEndpoint) -> RemoteBool: + return self.compareWith(other).__gt__(0) + + def __ge__(self, other: _RemoteTextRangeEndpoint) -> RemoteBool: + return self.compareWith(other).__ge__(0) + + def __eq__(self, other: _RemoteTextRangeEndpoint) -> RemoteBool: + return self.compareWith(other) == 0 + + def __ne__(self, other: _RemoteTextRangeEndpoint) -> RemoteBool: + return (self == other).inverse() + + +class RemoteTextRangeLogicalAdapter(builder._RemoteBase): + + def __init__( + self, + rob: builder.RemoteOperationBuilder, + textRange: RemoteTextRange, + reverse: bool = False + ): + super().__init__(rob) + self._textRange = textRange + self._isReversed = reverse + + @property + def textRange(self) -> RemoteTextRange: + return self._textRange + + @property + def isReversed(self) -> bool: + return self._isReversed + + @property + def start(self) -> _RemoteTextRangeEndpoint: + obj = _RemoteTextRangeEndpoint(self.rob, self, isStart=True) + return obj + + @start.setter + def start(self, value: _RemoteTextRangeEndpoint): + self.start.moveTo(value) + + @property + def end(self) -> _RemoteTextRangeEndpoint: + obj = _RemoteTextRangeEndpoint(self.rob, self, isStart=False) + return obj + + @end.setter + def end(self, value: _RemoteTextRangeEndpoint): + self.end.moveTo(value) + + def clone(self): + return self.textRange.clone().getLogicalAdapter(self.isReversed) diff --git a/source/UIAHandler/browseMode.py b/source/UIAHandler/browseMode.py index d89581b326a..1d932bfea3d 100644 --- a/source/UIAHandler/browseMode.py +++ b/source/UIAHandler/browseMode.py @@ -11,6 +11,7 @@ import array import winUser import UIAHandler +import UIAHandler.remote from .utils import ( createUIAMultiPropertyCondition, getDeepestLastChildUIAElementInWalker, @@ -23,7 +24,7 @@ import textInfos import browseMode from logHandler import log -from NVDAObjects.UIA import UIA +from NVDAObjects.UIA import UIA, UIATextInfo class UIADocumentWithTableNavigation(documentBase.DocumentWithTableNavigation): @@ -144,9 +145,21 @@ def UIATextAttributeQuicknavIterator(ItemClass,itemType,document,position,direct class HeadingUIATextInfoQuickNavItem(browseMode.TextInfoQuickNavItem): - def __init__(self,itemType,document,position,level=0): + def __init__( + self, + itemType: str, + document: UIA, + position: UIATextInfo, + label: str | None = None, + level: int = 0 + ): super(HeadingUIATextInfoQuickNavItem,self).__init__(itemType,document,position) self.level=level + self._label = label + + @property + def label(self): + return self._label or super().label def isChild(self,parent): if not isinstance(parent,HeadingUIATextInfoQuickNavItem): @@ -161,6 +174,8 @@ def UIAHeadingQuicknavIterator( direction: str = "next" ): reverse = bool(direction == "previous") + itemTypeBaseLen = len('heading') + wantedLevel = int(itemType[itemTypeBaseLen:]) if len(itemType) > itemTypeBaseLen else None entireDocument = document.makeTextInfo(textInfos.POSITION_ALL) if position is None: searchArea = entireDocument @@ -170,6 +185,19 @@ def UIAHeadingQuicknavIterator( searchArea.start = entireDocument.start else: searchArea.end = entireDocument.end + if UIAHandler.remote.isSupported(): + if position is None: + headings = UIAHandler.remote.collectAllHeadingsInTextRange(searchArea._rangeObj) + for level, label, rangeObj in headings: + pos = document.makeTextInfo(rangeObj) + yield HeadingUIATextInfoQuickNavItem(itemType, document, pos, label=label, level=level) + else: + heading = UIAHandler.remote.findFirstHeadingInTextRange(searchArea._rangeObj, wantedLevel, reverse) + if heading is not None: + level, label, rangeObj = heading + pos = document.makeTextInfo(rangeObj) + yield HeadingUIATextInfoQuickNavItem(itemType, document, pos, label=label, level=level) + return firstLoop=True for subrange in iterUIARangeByUnit(searchArea._rangeObj, UIAHandler.TextUnit_Paragraph, reverse=reverse): if firstLoop: @@ -183,7 +211,6 @@ def UIAHeadingQuicknavIterator( # In Python 3, comparing an int with a pointer raises a TypeError. if isinstance(styleIDValue, int) and UIAHandler.StyleId_Heading1 <= styleIDValue <= UIAHandler.StyleId_Heading9: foundLevel = (styleIDValue - UIAHandler.StyleId_Heading1) + 1 - wantedLevel = int(itemType[7:]) if len(itemType) > 7 else None if not wantedLevel or wantedLevel==foundLevel: tempInfo = document.makeTextInfo(subrange) yield HeadingUIATextInfoQuickNavItem(itemType, document, tempInfo, level=foundLevel) diff --git a/source/UIAHandler/remote.py b/source/UIAHandler/remote.py index 06096d9eb3b..01769f96db6 100644 --- a/source/UIAHandler/remote.py +++ b/source/UIAHandler/remote.py @@ -1,44 +1,170 @@ # A part of NonVisual Desktop Access (NVDA) # This file is covered by the GNU General Public License. # See the file COPYING for more details. -# Copyright (C) 2021-2022 NV Access Limited +# Copyright (C) 2021-2024 NV Access Limited -from typing import Optional, Union -import os -from ctypes import windll, byref, POINTER -from comtypes.automation import VARIANT -import NVDAHelper +from typing import ( + Optional, + Any, + Generator, + cast +) +from comtypes import GUID from comInterfaces import UIAutomationClient as UIA +import winVersion +from ._remoteOps import remoteAlgorithms +from ._remoteOps.remoteTypes import ( + RemoteExtensionTarget, + RemoteInt +) +from ._remoteOps import operation +from ._remoteOps import remoteAPI +from ._remoteOps.lowLevel import ( + TextUnit, + AttributeId, + StyleId +) -_dll = None +_isSupported: bool = False -def initialize(doRemote: bool, UIAClient: POINTER(UIA.IUIAutomation)): +def isSupported() -> bool: + """ + Returns whether UIA remote operations are supported on this version of Windows. + """ + return _isSupported + + +def initialize(doRemote: bool, UIAClient: UIA.IUIAutomation): """ Initializes UI Automation remote operations. + The following parameters are deprecated and ignored: @param doRemote: true if code should be executed remotely, or false for locally. @param UIAClient: the current instance of the UI Automation client library running in NVDA. """ - global _dll - _dll = windll[os.path.join(NVDAHelper.versionedLibPath, "UIARemote.dll")] - _dll.initialize(doRemote, UIAClient) + global _isSupported + _isSupported = winVersion.getWinVer() >= winVersion.WIN11 + return True def terminate(): """ Terminates UIA remote operations support.""" - _dll.cleanup() + global _isSupported + _isSupported = False def msWord_getCustomAttributeValue( - docElement: POINTER(UIA.IUIAutomationElement), - textRange: POINTER(UIA.IUIAutomationTextRange), + docElement: UIA.IUIAutomationElement, + textRange: UIA.IUIAutomationTextRange, customAttribID: int -) -> Optional[Union[int, str]]: - if _dll is None: - raise RuntimeError("UIARemote not initialized") - customAttribValue = VARIANT() - if _dll.msWord_getCustomAttributeValue(docElement, textRange, customAttribID, byref(customAttribValue)): - return customAttribValue.value - return None +) -> Optional[Any]: + guid_msWord_extendedTextRangePattern = GUID("{93514122-FF04-4B2C-A4AD-4AB04587C129}") + guid_msWord_getCustomAttributeValue = GUID("{081ACA91-32F2-46F0-9FB9-017038BC45F8}") + op = operation.Operation() + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + remoteDocElement = ra.newElement(docElement) + remoteTextRange = ra.newTextRange(textRange) + remoteCustomAttribValue = ra.newVariant() + with ra.ifBlock(remoteDocElement.isExtensionSupported(guid_msWord_extendedTextRangePattern)): + ra.logRuntimeMessage("docElement supports extendedTextRangePattern") + remoteResult = ra.newVariant() + ra.logRuntimeMessage("doing callExtension for extendedTextRangePattern") + remoteDocElement.callExtension( + guid_msWord_extendedTextRangePattern, + remoteResult + ) + with ra.ifBlock(remoteResult.isNull()): + ra.logRuntimeMessage("extendedTextRangePattern is null") + ra.halt() + with ra.elseBlock(): + ra.logRuntimeMessage("got extendedTextRangePattern") + remoteExtendedTextRangePattern = remoteResult.asType(RemoteExtensionTarget) + with ra.ifBlock( + remoteExtendedTextRangePattern.isExtensionSupported(guid_msWord_getCustomAttributeValue) + ): + ra.logRuntimeMessage("extendedTextRangePattern supports getCustomAttributeValue") + ra.logRuntimeMessage("doing callExtension for getCustomAttributeValue") + remoteExtendedTextRangePattern.callExtension( + guid_msWord_getCustomAttributeValue, + remoteTextRange, + customAttribID, + remoteCustomAttribValue + ) + ra.logRuntimeMessage("got customAttribValue of ", remoteCustomAttribValue) + ra.Return(remoteCustomAttribValue) + with ra.elseBlock(): + ra.logRuntimeMessage("extendedTextRangePattern does not support getCustomAttributeValue") + with ra.elseBlock(): + ra.logRuntimeMessage("docElement does not support extendedTextRangePattern") + ra.logRuntimeMessage("msWord_getCustomAttributeValue end") + + customAttribValue = op.execute() + return customAttribValue + + +def collectAllHeadingsInTextRange( + textRange: UIA.IUIAutomationTextRange +) -> Generator[tuple[int, str, UIA.IUIAutomationElement], None, None]: + op = operation.Operation() + + @op.buildIterableFunction + def code(ra: remoteAPI.RemoteAPI): + remoteTextRange = ra.newTextRange(textRange, static=True) + with remoteAlgorithms.remote_forEachUnitInTextRange( + ra, remoteTextRange, TextUnit.Paragraph + ) as paragraphRange: + val = paragraphRange.getAttributeValue(AttributeId.StyleId) + with ra.ifBlock(val.isInt()): + intVal = val.asType(RemoteInt) + with ra.ifBlock((intVal >= StyleId.Heading1) & (intVal <= StyleId.Heading9)): + level = (intVal - StyleId.Heading1) + 1 + label = paragraphRange.getText(-1) + ra.Yield(level, label, paragraphRange) + + for level, label, paragraphRange in op.iterExecute(maxTries=20): + yield level, label, paragraphRange + + +def findFirstHeadingInTextRange( + textRange: UIA.IUIAutomationTextRange, + wantedLevel: int | None = None, + reverse: bool = False +) -> tuple[int, str, UIA.IUIAutomationElement] | None: + op = operation.Operation() + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + remoteTextRange = ra.newTextRange(textRange, static=True) + remoteWantedLevel = ra.newInt(wantedLevel or 0) + executionCount = ra.newInt(0, static=True) + executionCount += 1 + ra.logRuntimeMessage("executionCount is ", executionCount) + with ra.ifBlock(executionCount == 1): + ra.logRuntimeMessage("Doing initial move") + remoteTextRange.getLogicalAdapter(reverse).start.moveByUnit(TextUnit.Paragraph, 1) + with remoteAlgorithms.remote_forEachUnitInTextRange( + ra, remoteTextRange, TextUnit.Paragraph, reverse=reverse + ) as paragraphRange: + val = paragraphRange.getAttributeValue(AttributeId.StyleId) + with ra.ifBlock(val.isInt()): + intVal = val.asType(RemoteInt) + with ra.ifBlock((intVal >= StyleId.Heading1) & (intVal <= StyleId.Heading9)): + level = (intVal - StyleId.Heading1) + 1 + with ra.ifBlock((remoteWantedLevel == 0) | (level == remoteWantedLevel)): + ra.logRuntimeMessage("found heading at level ", level) + label = paragraphRange.getText(-1) + ra.Return(level, label, paragraphRange) + + try: + level, label, paragraphRange = op.execute(maxTries=20) + except operation.NoReturnException: + return None + return ( + cast(int, level), + cast(str, label), + cast(UIA.IUIAutomationTextRange, paragraphRange) + ) diff --git a/tests/unit/test_UIARemoteOps/__init__.py b/tests/unit/test_UIARemoteOps/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/test_UIARemoteOps/test_highLevel/__init__.py b/tests/unit/test_UIARemoteOps/test_highLevel/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/test_UIARemoteOps/test_highLevel/test_bool.py b/tests/unit/test_UIARemoteOps/test_highLevel/test_bool.py new file mode 100644 index 00000000000..48493b9c682 --- /dev/null +++ b/tests/unit/test_UIARemoteOps/test_highLevel/test_bool.py @@ -0,0 +1,91 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2024 NV Access Limited + +""" +High-level UIA remote ops unit tests for setting and comparing booleans. +""" + +from unittest import TestCase +from UIAHandler._remoteOps import operation +from UIAHandler._remoteOps import remoteAPI + + +class Test_bool(TestCase): + + def test_false(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + b = ra.newBool(False) + ra.Return(b) + + b = op.execute() + self.assertFalse(b) + + def test_setTrue(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + b = ra.newBool(False) + b.set(True) + ra.Return(b) + + b = op.execute() + self.assertTrue(b) + + def test_inverse(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + t = ra.newBool(True) + f = ra.newBool(False) + t_inverse = t.inverse() + f_inverse = f.inverse() + ra.Return(t_inverse, f_inverse) + + t_inverse, f_inverse = op.execute() + self.assertFalse(t_inverse) + self.assertTrue(f_inverse) + + def test_and(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + t = ra.newBool(True) + f = ra.newBool(False) + t_and_t = t & t + t_and_f = t & f + f_and_t = f & t + f_and_f = f & f + ra.Return(t_and_t, t_and_f, f_and_t, f_and_f) + + t_and_t, t_and_f, f_and_t, f_and_f = op.execute() + self.assertTrue(t_and_t) + self.assertFalse(t_and_f) + self.assertFalse(f_and_t) + self.assertFalse(f_and_f) + + def test_or(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + t = ra.newBool(True) + f = ra.newBool(False) + t_or_t = t | t + t_or_f = t | f + f_or_t = f | t + f_or_f = f | f + ra.Return(t_or_t, t_or_f, f_or_t, f_or_f) + + t_or_t, t_or_f, f_or_t, f_or_f = op.execute() + self.assertTrue(t_or_t) + self.assertTrue(t_or_f) + self.assertTrue(f_or_t) + self.assertFalse(f_or_f) diff --git a/tests/unit/test_UIARemoteOps/test_highLevel/test_element.py b/tests/unit/test_UIARemoteOps/test_highLevel/test_element.py new file mode 100644 index 00000000000..a31937f4ead --- /dev/null +++ b/tests/unit/test_UIARemoteOps/test_highLevel/test_element.py @@ -0,0 +1,36 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2024 NV Access Limited + +""" +High-level UIA remote ops Unit tests for UIA element methods. +""" + +from unittest import TestCase +from unittest.mock import Mock +from ctypes import POINTER +from UIAHandler import UIA +from UIAHandler._remoteOps import operation +from UIAHandler._remoteOps import remoteAPI +from UIAHandler._remoteOps.lowLevel import ( + PropertyId, +) + + +class Test_element(TestCase): + + def test_getName(self): + uiaElement = Mock(spec=POINTER(UIA.IUIAutomationElement)) + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + element = ra.newElement(uiaElement) + name = element.getPropertyValue(PropertyId.Name) + ra.Return(name) + + uiaElement.GetCurrentPropertyValueEx.return_value = "foo" + name = op.execute() + uiaElement.GetCurrentPropertyValueEx.assert_called_once_with(PropertyId.Name, False) + self.assertEqual(name, "foo") diff --git a/tests/unit/test_UIARemoteOps/test_highLevel/test_errorHandling.py b/tests/unit/test_UIARemoteOps/test_highLevel/test_errorHandling.py new file mode 100644 index 00000000000..54c4bcb5b4e --- /dev/null +++ b/tests/unit/test_UIARemoteOps/test_highLevel/test_errorHandling.py @@ -0,0 +1,61 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2024 NV Access Limited + +""" +High-level UIA remote ops Unit tests for error handling including try, except, and uncaught errors. +""" + +from unittest import TestCase +from UIAHandler._remoteOps import operation +from UIAHandler._remoteOps import remoteAPI + + +class Test_errorHandling(TestCase): + + def test_error(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(3) + j = i / 0 + ra.Return(j) + + with self.assertRaises(operation.UnhandledException): + op.execute() + + def test_try_with_no_error(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(3) + was_in_catch = ra.newBool(False) + with ra.tryBlock(): + j = i + 1 + with ra.catchBlock(): + was_in_catch.set(True) + ra.Return(j, was_in_catch) + + j, was_in_catch = op.execute() + self.assertEqual(j, 4) + self.assertFalse(was_in_catch) + + def test_try_with_error(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(3) + was_in_catch = ra.newBool(False) + with ra.tryBlock(): + i / 0 + with ra.catchBlock(): + was_in_catch.set(True) + ra.Return(i, was_in_catch) + + i, was_in_catch = op.execute() + self.assertEqual(i, 3) + self.assertTrue(was_in_catch) diff --git a/tests/unit/test_UIARemoteOps/test_highLevel/test_float.py b/tests/unit/test_UIARemoteOps/test_highLevel/test_float.py new file mode 100644 index 00000000000..1ae5104afdc --- /dev/null +++ b/tests/unit/test_UIARemoteOps/test_highLevel/test_float.py @@ -0,0 +1,119 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2024 NV Access Limited + +""" +High-level UIA remote ops Unit tests for floating point arithmetic. +""" + +from unittest import TestCase +from UIAHandler._remoteOps import operation +from UIAHandler._remoteOps import remoteAPI + + +class Test_float(TestCase): + + def test_inplace_add(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newFloat(5.4) + j = ra.newFloat(3.4) + i += j + ra.Return(i) + + i = op.execute() + self.assertEqual(i, 8.8) + + def test_binary_add(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newFloat(3.3) + j = ra.newFloat(4.4) + k = i + j + ra.Return(k) + + k = op.execute() + self.assertEqual(k, 7.7) + + def test_inplace_subtract(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newFloat(5.7) + j = ra.newFloat(3.5) + i -= j + ra.Return(i) + + i = op.execute() + self.assertEqual(i, 2.2) + + def test_binary_subtract(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newFloat(7.3) + j = ra.newFloat(3.2) + k = i - j + ra.Return(k) + + k = op.execute() + self.assertEqual(k, 4.1) + + def test_inplace_multiply(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newFloat(5.0) + j = ra.newFloat(3.0) + i *= j + ra.Return(i) + + i = op.execute() + self.assertEqual(i, 15.0) + + def test_binary_multiply(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newFloat(3.2) + j = ra.newFloat(4.5) + k = i * j + ra.Return(k) + + k = op.execute() + self.assertEqual(k, 14.4) + + def test_inplace_divide(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newFloat(20.0) + j = ra.newFloat(2.5) + i /= j + ra.Return(i) + + i = op.execute() + self.assertEqual(i, 8.0) + + def test_binary_divide(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newFloat(21.0) + j = ra.newFloat(3.0) + k = i / j + ra.Return(k) + + k = op.execute() + self.assertEqual(k, 7.0) diff --git a/tests/unit/test_UIARemoteOps/test_highLevel/test_if.py b/tests/unit/test_UIARemoteOps/test_highLevel/test_if.py new file mode 100644 index 00000000000..a35f501b592 --- /dev/null +++ b/tests/unit/test_UIARemoteOps/test_highLevel/test_if.py @@ -0,0 +1,100 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2024 NV Access Limited + +""" +High-level UIA remote ops Unit tests for if conditions. +""" + +from unittest import TestCase +from UIAHandler._remoteOps import operation +from UIAHandler._remoteOps import remoteAPI + + +class Test_if(TestCase): + + def test_if_true(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + true_condition = ra.newBool(True) + was_in_if = ra.newBool(False) + was_in_else = ra.newBool(False) + with ra.ifBlock(true_condition): + was_in_if.set(True) + with ra.elseBlock(): + was_in_else.set(True) + ra.Return(was_in_if, was_in_else) + + was_in_if, was_in_else = op.execute() + self.assertTrue(was_in_if) + self.assertFalse(was_in_else) + + def test_if_false(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + false_condition = ra.newBool(False) + was_in_if = ra.newBool(False) + was_in_else = ra.newBool(False) + with ra.ifBlock(false_condition): + was_in_if.set(True) + with ra.elseBlock(): + was_in_else.set(True) + ra.Return(was_in_if, was_in_else) + + was_in_if, was_in_else = op.execute() + self.assertFalse(was_in_if) + self.assertTrue(was_in_else) + + def test_else_no_if(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + with self.assertRaises(RuntimeError): + with ra.elseBlock(): + pass + + def test_if_with_multiple_returns_first(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + condition = ra.newBool(True) + with ra.ifBlock(condition): + ra.Return(1) + with ra.elseBlock(): + ra.Return(2) + + res = op.execute() + self.assertEqual(res, 1) + + def test_if_with_multiple_returns_second(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + condition = ra.newBool(False) + with ra.ifBlock(condition): + ra.Return(1) + with ra.elseBlock(): + ra.Return(2) + + res = op.execute() + self.assertEqual(res, 2) + + def test_if_with_no_return(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + condition = ra.newBool(False) + with ra.ifBlock(condition): + ra.Return(1) + + with self.assertRaises(operation.NoReturnException): + op.execute() diff --git a/tests/unit/test_UIARemoteOps/test_highLevel/test_instructionLimit.py b/tests/unit/test_UIARemoteOps/test_highLevel/test_instructionLimit.py new file mode 100644 index 00000000000..fb3ca2d6470 --- /dev/null +++ b/tests/unit/test_UIARemoteOps/test_highLevel/test_instructionLimit.py @@ -0,0 +1,54 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2024 NV Access Limited + +""" +High-level UIA remote ops Unit tests for reaching and handling the remote ops instruction limit. +""" + +from unittest import TestCase +from UIAHandler._remoteOps import operation +from UIAHandler._remoteOps import remoteAPI + + +class Test_instructionLimit(TestCase): + + def test_instructionLimitExceeded(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(0) + with ra.whileBlock(lambda: i < 20000): + i += 1 + ra.Return(i) + + with self.assertRaises(operation.InstructionLimitExceededException): + op.execute() + + def test_instructionLimitExceeded_with_static(self): + """ + Tests that a long operation is automatically re-executed multiple times + until it is successfully run without exceeding the instruction limit. + Using a static variable which is not reset between executions, + and then incremented once each execution, + we can track how many times the operation was executed. + The operation itself involves incrementing a counter variable 'i' + within a while loop until it reaches 20000. + This is expected to take 9 executions to fully complete. + """ + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + executionCount = ra.newInt(0, static=True) + executionCount += 1 + i = ra.newInt(0, static=True) + with ra.whileBlock(lambda: i < 20000): + i += 1 + ra.Return(executionCount, i) + + executionCount, i = op.execute(maxTries=20) + self.assertEqual(i, 20000) + self.assertEqual(executionCount, 9) diff --git a/tests/unit/test_UIARemoteOps/test_highLevel/test_int.py b/tests/unit/test_UIARemoteOps/test_highLevel/test_int.py new file mode 100644 index 00000000000..6f0255d1887 --- /dev/null +++ b/tests/unit/test_UIARemoteOps/test_highLevel/test_int.py @@ -0,0 +1,145 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2024 NV Access Limited + +""" +High-level UIA remote ops Unit tests for integer arithmetic. +""" + +from unittest import TestCase +from UIAHandler._remoteOps import operation +from UIAHandler._remoteOps import remoteAPI + + +class Test_int(TestCase): + + def test_inplace_add(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(5) + j = ra.newInt(3) + i += j + ra.Return(i) + + i = op.execute() + self.assertEqual(i, 8) + + def test_binary_add(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(3) + j = ra.newInt(4) + k = i + j + ra.Return(k) + + k = op.execute() + self.assertEqual(k, 7) + + def test_inplace_subtract(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(5) + j = ra.newInt(3) + i -= j + ra.Return(i) + + i = op.execute() + self.assertEqual(i, 2) + + def test_binary_subtract(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(7) + j = ra.newInt(3) + k = i - j + ra.Return(k) + + k = op.execute() + self.assertEqual(k, 4) + + def test_inplace_multiply(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(5) + j = ra.newInt(3) + i *= j + ra.Return(i) + + i = op.execute() + self.assertEqual(i, 15) + + def test_binary_multiply(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(3) + j = ra.newInt(4) + k = i * j + ra.Return(k) + + k = op.execute() + self.assertEqual(k, 12) + + def test_inplace_divide(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(18) + j = ra.newInt(3) + i /= j + ra.Return(i) + + i = op.execute() + self.assertEqual(i, 6) + + def test_binary_divide(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(21) + j = ra.newInt(3) + k = i / j + ra.Return(k) + + k = op.execute() + self.assertEqual(k, 7) + + def test_inplace_mod(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(9) + j = ra.newInt(2) + i %= j + ra.Return(i) + + i = op.execute() + self.assertEqual(i, 1) + + def test_binary_mod(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(10) + j = ra.newInt(5) + k = i % j + ra.Return(k) + + k = op.execute() + self.assertEqual(k, 0) diff --git a/tests/unit/test_UIARemoteOps/test_highLevel/test_iterable.py b/tests/unit/test_UIARemoteOps/test_highLevel/test_iterable.py new file mode 100644 index 00000000000..4205290a9fb --- /dev/null +++ b/tests/unit/test_UIARemoteOps/test_highLevel/test_iterable.py @@ -0,0 +1,97 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2024 NV Access Limited + +""" +High-level UIA remote ops Unit tests for writing iterable functions that can yield values. +""" + +from unittest import TestCase +from UIAHandler._remoteOps import operation +from UIAHandler._remoteOps import remoteAPI + + +class Test_iterable(TestCase): + + def test_iterableFunction(self): + op = operation.Operation(localMode=True) + + @op.buildIterableFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(0) + with ra.whileBlock(lambda: i < 4): + ra.Yield(i) + i += 1 + + results = [] + for i in op.iterExecute(): + results.append(i) + self.assertEqual(results, [0, 1, 2, 3]) + + def test_long_iterableFunction(self): + """ + Tests that a long operation is automatically re-executed multiple times + until it is successfully run without exceeding the instruction limit. + Using a static variable which is not reset between executions, + and then incremented once each execution, + we can track how many times the operation was executed. + The operation itself involves incrementing a counter variable 'i' + within a while loop until it reaches 5000, + yielding i and the execution count on every loop. + This is expected to take 5 executions to fully complete. + """ + op = operation.Operation(localMode=True) + + @op.buildIterableFunction + def code(ra: remoteAPI.RemoteAPI): + executionCount = ra.newInt(0, static=True) + executionCount += 1 + i = ra.newInt(0, static=True) + with ra.whileBlock(lambda: i < 5000): + i += 1 + ra.Yield(i, executionCount) + + results = [] + for i, executionCount in op.iterExecute(maxTries=20): + results.append((i, executionCount)) + self.assertEqual(len(results), 5000) + last_i, last_executionCount = results[-1] + self.assertEqual(last_i, 5000) + self.assertEqual(last_executionCount, 5) + + def test_forEachNumInRange(self): + op = operation.Operation(localMode=True) + + @op.buildIterableFunction + def code(ra: remoteAPI.RemoteAPI): + with ra.forEachNumInRange(10, 15) as i: + ra.Yield(i) + + results = [] + for i in op.iterExecute(): + results.append(i) + self.assertEqual(results, [10, 11, 12, 13, 14]) + + def test_forEachItemInArray(self): + """ + Tests that a long operation is automatically re-executed multiple times + until it is successfully run without exceeding the instruction limit. + Using a static variable which is not reset between executions, + and then incremented once each execution, + we can track how many times the operation was executed. + """ + op = operation.Operation(localMode=True) + + @op.buildIterableFunction + def code(ra: remoteAPI.RemoteAPI): + array = ra.newArray() + with ra.forEachNumInRange(0, 10, 2) as i: + array.append(i) + with ra.forEachItemInArray(array) as item: + ra.Yield(item) + + results = [] + for i in op.iterExecute(): + results.append(i) + self.assertEqual(results, [0, 2, 4, 6, 8]) diff --git a/tests/unit/test_UIARemoteOps/test_highLevel/test_numericComparison.py b/tests/unit/test_UIARemoteOps/test_highLevel/test_numericComparison.py new file mode 100644 index 00000000000..47cc95b138f --- /dev/null +++ b/tests/unit/test_UIARemoteOps/test_highLevel/test_numericComparison.py @@ -0,0 +1,84 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2024 NV Access Limited + +""" +High-level UIA remote ops Unit tests for numeric comparisons. +""" + +from unittest import TestCase +from UIAHandler._remoteOps import operation +from UIAHandler._remoteOps import remoteAPI + + +class Test_numericComparison(TestCase): + + def test_compare_lowToHigh(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(3) + j = ra.newInt(4) + lt = i < j + le = i <= j + eq = i == j + ne = i != j + ge = i >= j + gt = i > j + ra.Return(lt, le, eq, ne, ge, gt) + + lt, le, eq, ne, ge, gt = op.execute() + self.assertTrue(lt) + self.assertTrue(le) + self.assertFalse(eq) + self.assertTrue(ne) + self.assertFalse(ge) + self.assertFalse(gt) + + def test_compare_highToLow(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(4) + j = ra.newInt(3) + lt = i < j + le = i <= j + eq = i == j + ne = i != j + ge = i >= j + gt = i > j + ra.Return(lt, le, eq, ne, ge, gt) + + lt, le, eq, ne, ge, gt = op.execute() + self.assertFalse(lt) + self.assertFalse(le) + self.assertFalse(eq) + self.assertTrue(ne) + self.assertTrue(ge) + self.assertTrue(gt) + + def test_compare_same(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(3) + j = ra.newInt(3) + lt = i < j + le = i <= j + eq = i == j + ne = i != j + ge = i >= j + gt = i > j + ra.Return(lt, le, eq, ne, ge, gt) + + lt, le, eq, ne, ge, gt = op.execute() + self.assertFalse(lt) + self.assertTrue(le) + self.assertTrue(eq) + self.assertFalse(ne) + self.assertTrue(ge) + self.assertFalse(gt) diff --git a/tests/unit/test_UIARemoteOps/test_highLevel/test_string.py b/tests/unit/test_UIARemoteOps/test_highLevel/test_string.py new file mode 100644 index 00000000000..7b99e44a658 --- /dev/null +++ b/tests/unit/test_UIARemoteOps/test_highLevel/test_string.py @@ -0,0 +1,28 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2024 NV Access Limited + +""" +High-level UIA remote ops Unit tests for strings. +""" + +from unittest import TestCase +from UIAHandler._remoteOps import operation +from UIAHandler._remoteOps import remoteAPI + + +class Test_string(TestCase): + + def test_concat(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + s = ra.newString("hello") + t = ra.newString(" world") + u = (s + t) + ra.Return(u) + + u = op.execute() + self.assertEqual(u, "hello world") diff --git a/tests/unit/test_UIARemoteOps/test_highLevel/test_while.py b/tests/unit/test_UIARemoteOps/test_highLevel/test_while.py new file mode 100644 index 00000000000..9acd7206a2a --- /dev/null +++ b/tests/unit/test_UIARemoteOps/test_highLevel/test_while.py @@ -0,0 +1,61 @@ +# A part of NonVisual Desktop Access (NVDA) +# This file is covered by the GNU General Public License. +# See the file COPYING for more details. +# Copyright (C) 2024 NV Access Limited + +""" +High-level UIA remote ops Unit tests for while loops, including break and continue. +""" + +from unittest import TestCase +from UIAHandler._remoteOps import operation +from UIAHandler._remoteOps import remoteAPI + + +class Test_while(TestCase): + + def test_while(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + counter = ra.newInt(0) + with ra.whileBlock(lambda: counter < 7): + counter += 2 + ra.Return(counter) + + counter = op.execute() + self.assertEqual(counter, 8) + + def test_breakLoop(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + counter = ra.newInt(0) + with ra.whileBlock(lambda: counter < 7): + counter += 2 + with ra.ifBlock(counter == 4): + ra.breakLoop() + ra.Return(counter) + + counter = op.execute() + self.assertEqual(counter, 4) + + def test_continueLoop(self): + op = operation.Operation(localMode=True) + + @op.buildFunction + def code(ra: remoteAPI.RemoteAPI): + i = ra.newInt(0) + j = ra.newInt(0) + with ra.whileBlock(lambda: i < 7): + i += 1 + with ra.ifBlock(i == 4): + ra.continueLoop() + j += 1 + ra.Return(i, j) + + i, j = op.execute() + self.assertEqual(i, 7) + self.assertEqual(j, 6)