Skip to content

Commit

Permalink
feat: Add an extension to press arbitrary keys (#166)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mykola-mokhnach
mykola-mokhnach authored Feb 19, 2023
1 parent 923b48b commit 8418f1e
Show file tree
Hide file tree
Showing 5 changed files with 293 additions and 47 deletions.
21 changes: 21 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,27 @@ endY | number | no | Same as in [windows: click](#windows-click) | 100
modifierKeys | string[] or string | no | Same as in [windows: click](#windows-click) | win
durationMs | number | no | The number of milliseconds to wait between pressing the left mouse button and moving the cursor to the ending drag point. 500ms by default. | 700

### windows: keys

This is a shortcut for a customized keyboard input.

> :warning: **If your Node.js version is 17 and newer**: As of January 2023 the [node-ffi-napi](https://github.com/node-ffi-napi), which we use to call native Windows APIs has a [bug](https://github.com/node-ffi-napi/node-ffi-napi/issues/244), which prevents it to work properly with Node.js version above 16. The only workaround until a fix is applied is to downgrade Node.js.
#### Arguments

Name | Type | Required | Description | Example
--- | --- | --- | --- | ---
actions | KeyAction[] or KeyAction | yes | One or more [KeyAction](#keyaction) dictionaries | ```json [{"virtualKeyCode": 0x10, "down": true}, {'text': "appium likes you"}, {"virtualKeyCode": 0x10, "down": false}]```

##### KeyAction

Name | Type | Required | Description | Example
--- | --- | --- | --- | ---
pause | number | no | Allows to set a delay in milliseconds between key input series. Either this property or `text` or `virtualKeyCode` must be provided. | 100
text | string | no | Non-empty string of Unicode text to type (surrogate characters like smileys are not supported). Either this property or `pause` or `virtualKeyCode` must be provided. | Привіт Світ!
virtualKeyCode | number | no | Valid virtual key code. The list of supported key codes is available at [Virtual-Key Codes](https://learn.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes) page. Either this property or `pause` or `text` must be provided. | 0x10
down | boolean | no | This property only makes sense in combination with `virtualKeyCode`. If set to `true` then the corresponding key will be depressed, `false` - released. By default the key is just pressed once. ! Do not forget to release depressed keys in your automated tests. | true


## Environment Variables

Expand Down
1 change: 1 addition & 0 deletions lib/commands/execute.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ const EXTENSION_COMMANDS_MAPPING = {
click: 'windowsClick',
scroll: 'windowsScroll',
clickAndDrag: 'windowsClickAndDrag',
keys: 'windowsKeys',
};

commands.execute = async function execute (script, args) {
Expand Down
197 changes: 196 additions & 1 deletion lib/commands/gestures.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@ import {
MOUSE_BUTTON_ACTION,
MOUSE_BUTTON,
KEY_ACTION,
toModifierInputs,
MODIFIER_KEY,
KEYEVENTF_KEYUP,
createKeyInput,
toUnicodeKeyInputs,
handleInputs,
toMouseButtonInput,
toMouseMoveInput,
Expand Down Expand Up @@ -92,6 +95,156 @@ async function toAbsoluteCoordinates(elementId, x, y, msgPrefix = '') {
return [absoluteX, absoluteY];
}

function isKeyDown(action) {
switch (_.toLower(action)) {
case KEY_ACTION.UP:
return false;
case KEY_ACTION.DOWN:
return true;
default:
throw new errors.InvalidArgumentError(
`Key action '${action}' is unknown. Only ${_.values(KEY_ACTION)} actions are supported`
);
}
}

/**
* Transforms the provided key modifiers array into the sequence
* of functional key inputs.
*
* @param {string[]|string} modifierKeys Array of key modifiers or a single key name
* @param {'down' | 'up'} action Either 'down' to depress the key or 'up' to release it
* @returns {INPUT[]} Array of inputs or an empty array if no inputs were parsed.
*/
function toModifierInputs(modifierKeys, action) {
const events = [];
const usedKeys = new Set();
for (const keyName of (_.isArray(modifierKeys) ? modifierKeys : [modifierKeys])) {
const lowerKeyName = _.toLower(keyName);
if (usedKeys.has(lowerKeyName)) {
continue;
}

const virtualKeyCode = MODIFIER_KEY[lowerKeyName];
if (_.isUndefined(virtualKeyCode)) {
throw new errors.InvalidArgumentError(
`Modifier key name '${keyName}' is unknown. Supported key names are: ${_.keys(MODIFIER_KEY)}`
);
}
events.push({virtualKeyCode, action});
usedKeys.add(lowerKeyName);
}
return events
.map(({virtualKeyCode, action}) => ({
wVk: virtualKeyCode,
dwFlags: isKeyDown(action) ? 0 : KEYEVENTF_KEYUP,
}))
.map(createKeyInput);
}

const KEY_ACTION_PROPERTIES = [
'pause',
'text',
'virtualKeyCode',
];

/**
* @param {KeyAction} action
* @param {number} index
* @returns {INPUT[] | number}
*/
function parseKeyAction(action, index) {
const hasPause = _.has(action, 'pause');
const hasText = _.has(action, 'text');
const hasVirtualKeyCode = _.has(action, 'virtualKeyCode');
const definedPropertiesCount = hasPause + hasText + hasVirtualKeyCode;
const actionPrefix = `Key Action #${index + 1} (${JSON.stringify(action)}): `;
if (definedPropertiesCount === 0) {
throw new errors.InvalidArgumentError(
`${actionPrefix}Some key action (${KEY_ACTION_PROPERTIES.join(' or ')}) must be defined`
);
} else if (definedPropertiesCount > 1) {
throw new errors.InvalidArgumentError(
`${actionPrefix}Only one key action (${KEY_ACTION_PROPERTIES.join(' or ')}) must be defined`
);
}

const {
pause,
text,
virtualKeyCode,
down,
} = action;

if (hasPause) {
const durationMs = pause;
if (!_.isInteger(durationMs) || durationMs < 0) {
throw new errors.InvalidArgumentError(
`${actionPrefix}Pause value must be a valid positive integer number of milliseconds`
);
}
return durationMs;
}
if (hasText) {
if (!_.isString(text) || _.isEmpty(text)) {
throw new errors.InvalidArgumentError(
`${actionPrefix}Text value must be a valid non-empty string`
);
}
return toUnicodeKeyInputs(text);
}

// has virtual code
if (_.has(action, 'down')) {
if (!_.isBoolean(down)) {
throw new errors.InvalidArgumentError(
`${actionPrefix}The down argument must be of type boolean if provided`
);
}

// only depress or release the key if `down` is provided
return [createKeyInput({
wVk: virtualKeyCode,
dwFlags: down ? 0 : KEYEVENTF_KEYUP,
})];
}
// otherwise just press the key
return [
createKeyInput({
wVk: virtualKeyCode,
dwFlags: 0,
}),
createKeyInput({
wVk: virtualKeyCode,
dwFlags: KEYEVENTF_KEYUP,
}),
];
}

/**
* @param {KeyAction[]} actions
* @returns {Array<number | Array<INPUT>>}
*/
function parseKeyActions(actions) {
if (_.isEmpty(actions)) {
throw new errors.InvalidArgumentError('Key actions must not be empty');
}

const combinedArray = [];
const allActions = actions.map(parseKeyAction);
for (let i = 0; i < allActions.length; ++i) {
const item = allActions[i];
if (_.isArray(item) && combinedArray.length > 0 && _.isArray(_.last(combinedArray))) {
_.last(combinedArray).push(...item);
} else {
combinedArray.push(item);
}
}
// The resulting array contains all keyboard inputs combined into a single array
// unless there are pauses that act as splitters
return combinedArray;
}


const commands = {};

Expand Down Expand Up @@ -321,5 +474,47 @@ commands.windowsClickAndDrag = async function windowsClickAndDrag (opts = {}) {
}
};

/**
* @typedef {Object} KeyAction
* @property {number} pause Allows to set a delay in milliseconds between key input series.
* Either this property or `text` or `virtualKeyCode` must be provided.
* @property {string} text Non-empty string of Unicode text to type.
* Either this property or `pause` or `virtualKeyCode` must be provided.
* @property {number} virtualKeyCode Valid virtual key code. The list of supported key codes
* is available at https://learn.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes
* Either this property or `pause` or `text` must be provided.
* @property {boolean?} down [undefined] This property only makes sense in combination with `virtualKeyCode`.
* If set to `true` then the corresponding key will be depressed, `false` - released. By default
* the key is just pressed once.
* ! Do not forget to release depressed keys in your automated tests.
*/

/**
* @typedef {Object} KeysOptions
* @property {KeyAction[] | KeyAction} actions One or more key actions.
*/

/**
* Performs customized keyboard input.
*
* @param {KeysOptions} opts
* @throws {Error} If given options are not acceptable or the gesture has failed.
*/
commands.windowsKeys = async function windowsKeys (opts = {}) {
const {
actions,
} = opts;

const parsedItems = parseKeyActions(_.isArray(actions) ? actions : [actions]);
this.log.debug(`Parsed ${util.pluralize('key action', parsedItems.length, true)}`);
for (const item of parsedItems) {
if (_.isArray(item)) {
await handleInputs(item);
} else {
await B.delay(item);
}
}
};

export { commands };
export default commands;
83 changes: 37 additions & 46 deletions lib/commands/winapi/user32.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ const MOUSEINPUT = StructType({
// ULONG_PTR dwExtraInfo;
// } KEYBDINPUT;
const KEYBDINPUT = StructType({
wVK: 'uint16',
wVk: 'uint16',
wScan: 'uint16',
dwFlags: 'uint32',
time: 'uint32',
Expand Down Expand Up @@ -88,12 +88,13 @@ const INPUT = StructType({
});

const INPUT_KEYBOARD = 1;
const KEYEVENTF_KEYUP = 0x0002;
export const KEYEVENTF_KEYUP = 0x0002;
const KEYEVENTF_UNICODE = 0x0004;
export const KEY_ACTION = Object.freeze({
UP: 'up',
DOWN: 'down',
});

const VK_RETURN = 0x0D;
const VK_SHIFT = 0x10;
const VK_CONTROL = 0x11;
const VK_LWIN = 0x5B;
Expand Down Expand Up @@ -152,28 +153,17 @@ async function sendInputs (inputs, inputsCount) {
}
}

function toKeyInput({virtualKeyCode, action}) {
let down = true;
switch (_.toLower(action)) {
case KEY_ACTION.UP:
down = false;
break;
case KEY_ACTION.DOWN:
break;
default:
throw createInvalidArgumentError(`Key action '${action}' is unknown. ` +
`Only ${_.values(KEY_ACTION)} actions are supported`);
}

export function createKeyInput(params = {}) {
return INPUT({
type: INPUT_KEYBOARD,
union: INPUT_UNION({
ki: KEYBDINPUT({
wVK: virtualKeyCode,
wVk: 0,
time: 0,
dwExtraInfo: ref.NULL_POINTER,
wScan: 0,
dwFlags: down ? 0 : KEYEVENTF_KEYUP,
dwFlags: 0,
...params,
})
})
});
Expand Down Expand Up @@ -203,34 +193,6 @@ export async function handleInputs(inputs) {
throw new Error('At least one input must be provided');
}

/**
* Transforms the provided key modifiers array into the sequence
* of functional key inputs.
*
* @param {string[]|string} modifierKeys Array of key modifiers or a single key name
* @param {'down' | 'up'} action Either 'down' to depress the key or 'up' to release it
* @returns {INPUT[]} Array of inputs or an empty array if no inputs were parsed.
*/
export function toModifierInputs(modifierKeys, action) {
const events = [];
const usedKeys = new Set();
for (const keyName of (_.isArray(modifierKeys) ? modifierKeys : [modifierKeys])) {
const lowerKeyName = _.toLower(keyName);
if (usedKeys.has(lowerKeyName)) {
continue;
}

const virtualKeyCode = MODIFIER_KEY[lowerKeyName];
if (_.isUndefined(virtualKeyCode)) {
throw createInvalidArgumentError(`Modifier key name '${keyName}' is unknown. ` +
`Supported key names are: ${_.keys(MODIFIER_KEY)}`);
}
events.push({virtualKeyCode, action});
usedKeys.add(lowerKeyName);
}
return events.map(toKeyInput);
}

async function getSystemMetrics(nIndex) {
return await new B((resolve, reject) =>
// eslint-disable-next-line promise/prefer-await-to-callbacks
Expand Down Expand Up @@ -416,3 +378,32 @@ export function toMouseWheelInput({dx, dy}) {
}
return null;
}

/**
* Transforms the given Unicode text into an array of inputs
* ready to be used as parameters for SendInput API
*
* @param {string} text An arbitrary Unicode string
* @returns {INPUT[]} Array of key inputs
*/
export function toUnicodeKeyInputs(text) {
const utf16Text = Buffer.from(text, 'ucs2');
const charCodes = new Uint16Array(utf16Text.buffer, utf16Text.byteOffset, utf16Text.length / 2);
const result = [];
for (const [, charCode] of charCodes.entries()) {
// The WM_CHAR event generated for carriage return is '\r', not '\n', and
// some applications may check for VK_RETURN explicitly, so handle
// newlines specially.
if (charCode === 0x0A) {
result.push(
createKeyInput({wVk: VK_RETURN, dwFlags: 0}),
createKeyInput({wVk: VK_RETURN, dwFlags: KEYEVENTF_KEYUP})
);
}
result.push(
createKeyInput({wScan: charCode, dwFlags: KEYEVENTF_UNICODE}),
createKeyInput({wScan: charCode, dwFlags: KEYEVENTF_UNICODE | KEYEVENTF_KEYUP})
);
}
return result;
}
Loading

0 comments on commit 8418f1e

Please sign in to comment.