Skip to content

Commit

Permalink
support shadow dom (#555)
Browse files Browse the repository at this point in the history
  • Loading branch information
@LawyZheng
LawyZheng authored Jul 4, 2024
1 parent be97d0b commit 690c1a1
Show file tree
Hide file tree
Showing 3 changed files with 85 additions and 28 deletions.
98 changes: 76 additions & 22 deletions skyvern/webeye/scraper/domUtils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,9 @@ function isInteractableInput(element) {
}

function isInteractable(element) {
if (element.shadowRoot) {
return false;
}
if (!isElementVisible(element)) {
return false;
}
Expand Down Expand Up @@ -620,8 +623,9 @@ function getListboxOptions(element) {
return selectOptions;
}

async function getSelect2OptionElements() {
async function getSelect2OptionElements(element) {
let optionList = [];
const document = element.getRootNode();

while (true) {
oldOptionCount = optionList.length;
Expand Down Expand Up @@ -653,8 +657,8 @@ async function getSelect2OptionElements() {
return optionList;
}

async function getSelect2Options() {
const optionList = await getSelect2OptionElements();
async function getSelect2Options(element) {
const optionList = await getSelect2OptionElements(element);

let selectOptions = [];
for (let i = 0; i < optionList.length; i++) {
Expand All @@ -672,6 +676,27 @@ async function getSelect2Options() {
return selectOptions;
}

function getDOMElementBySkyvenElement(elementObj) {
// if element has shadowHost set, we need to find the shadowHost element first then find the element
if (elementObj.shadowHost) {
let shadowHostEle = document.querySelector(
`[unique_id="${elementObj.shadowHost}"]`,
);
if (!shadowHostEle) {
console.log(
"Could not find shadowHost element with unique_id: ",
elementObj.shadowHost,
);
return null;
}
return shadowHostEle.shadowRoot.querySelector(
`[unique_id="${elementObj.id}"]`,
);
}

return document.querySelector(`[unique_id="${elementObj.id}"]`);
}

function uniqueId() {
const characters =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
Expand Down Expand Up @@ -744,6 +769,18 @@ async function buildTreeFromBody(frame = "main.frame", open_select = false) {
elementTagNameLower === "svg" || element.closest("svg") !== null,
};

let isInShadowRoot = element.getRootNode() instanceof ShadowRoot;
if (isInShadowRoot) {
let shadowHostEle = element.getRootNode().host;
let shadowHostId = shadowHostEle.getAttribute("unique_id");
// assign shadowHostId to the shadowHost element if it doesn't have unique_id
if (!shadowHostId) {
shadowHostId = uniqueId();
shadowHost.setAttribute("unique_id", shadowHostId);
}
elementObj.shadowHost = shadowHostId;
}

// get options for select element or for listbox element
let selectOptions = null;
if (elementTagNameLower === "select") {
Expand All @@ -754,9 +791,9 @@ async function buildTreeFromBody(frame = "main.frame", open_select = false) {
} else if (open_select && isComboboxDropdown(element)) {
// open combobox dropdown to get options
element.click();
const listBox = document.getElementById(
element.getAttribute("aria-controls"),
);
const listBox = element
.getRootNode()
.getElementById(element.getAttribute("aria-controls"));
if (listBox) {
selectOptions = getListboxOptions(listBox);
}
Expand All @@ -777,7 +814,7 @@ async function buildTreeFromBody(frame = "main.frame", open_select = false) {
}),
);

selectOptions = await getSelect2Options();
selectOptions = await getSelect2Options(element);

// HACK: click again to close the dropdown
element.dispatchEvent(
Expand All @@ -789,7 +826,7 @@ async function buildTreeFromBody(frame = "main.frame", open_select = false) {
} else if (open_select && isSelect2MultiChoice(element)) {
// click element to show options
element.click();
selectOptions = await getSelect2Options();
selectOptions = await getSelect2Options(element);

// HACK: press ESC to close the dropdown
element.dispatchEvent(
Expand Down Expand Up @@ -820,6 +857,11 @@ async function buildTreeFromBody(frame = "main.frame", open_select = false) {
return;
}

// skip proccessing option element as they are already added to the select.options
if (element.tagName.toLowerCase() === "option") {
return;
}

// if element is an "a" tag and has a target="_blank" attribute, remove the target attribute
// We're doing this so that skyvern can do all the navigation in a single page/tab and not open new tab
if (element.tagName.toLowerCase() === "a") {
Expand Down Expand Up @@ -847,10 +889,6 @@ async function buildTreeFromBody(frame = "main.frame", open_select = false) {
.find((element) => element.id === parentId)
.children.push(elementObj);
}
// options already added to the select.options, no need to add options anymore
if (elementObj.options && elementObj.options.length > 0) {
return elementObj;
}
// Recursively process the children of the element
const children = getChildElements(element);
for (let i = 0; i < children.length; i++) {
Expand All @@ -863,6 +901,17 @@ async function buildTreeFromBody(frame = "main.frame", open_select = false) {

elements.push(iframeElementObject);
resultArray.push(iframeElementObject);
} else if (element.shadowRoot) {
// shadow host element
let shadowHostElement = await buildElementObject(element, false);
elements.push(shadowHostElement);
resultArray.push(shadowHostElement);

const children = getChildElements(element.shadowRoot);
for (let i = 0; i < children.length; i++) {
const childElement = children[i];
await processElement(childElement, shadowHostElement.id);
}
} else {
// For a non-interactable element, if it has direct text, we also tagged
// it with unique_id, but with interatable=false in the element.
Expand Down Expand Up @@ -915,11 +964,6 @@ async function buildTreeFromBody(frame = "main.frame", open_select = false) {
const children = getChildElements(element);
for (let i = 0; i < children.length; i++) {
const childElement = children[i];

// Skip processing option-children of an non-interactable select element as they are already added to the select.options
if (childElement.tagName.toLowerCase() === "option") {
continue;
}
await processElement(childElement, parentId);
}
}
Expand All @@ -932,7 +976,10 @@ async function buildTreeFromBody(frame = "main.frame", open_select = false) {

// look up for 10 levels to find the most contextual parent element
let targetContextualParent = null;
let currentEle = document.querySelector(`[unique_id="${element.id}"]`);
let currentEle = getDOMElementBySkyvenElement(element);
if (!currentEle) {
return ctx;
}
let parentEle = currentEle;
for (var i = 0; i < 10; i++) {
parentEle = parentEle.parentElement;
Expand Down Expand Up @@ -970,7 +1017,12 @@ async function buildTreeFromBody(frame = "main.frame", open_select = false) {
};

const getContextByLinked = (element, ctx) => {
let currentEle = document.querySelector(`[unique_id="${element.id}"]`);
let currentEle = getDOMElementBySkyvenElement(element);
if (!currentEle) {
return ctx;
}

const document = currentEle.getRootNode();
// check labels pointed to this element
// 1. element id -> labels pointed to this id
// 2. by attr "aria-labelledby" -> only one label with this id
Expand Down Expand Up @@ -1023,9 +1075,11 @@ async function buildTreeFromBody(frame = "main.frame", open_select = false) {
// if the element is a child of a td, th, or tr, then pass the grandparent's context to the element
let parentTagsThatDelegateParentContext = new Set(["td", "th", "tr"]);
if (tagsWithDirectParentContext.has(element.tagName)) {
let parentElement = document.querySelector(
`[unique_id="${element.id}"]`,
).parentElement;
let curElement = getDOMElementBySkyvenElement(element);
if (!curElement) {
return ctx;
}
let parentElement = curElement.parentElement;
if (!parentElement) {
return ctx;
}
Expand Down
8 changes: 4 additions & 4 deletions skyvern/webeye/scraper/scraper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from typing import Any, Awaitable, Callable

import structlog
from playwright.async_api import Frame, Page
from playwright.async_api import ElementHandle, Frame, Page
from pydantic import BaseModel

from skyvern.constants import PAGE_CONTENT_TIMEOUT, SKYVERN_DIR, SKYVERN_ID_ATTR
Expand Down Expand Up @@ -318,10 +318,10 @@ async def get_page_content(page: Page, timeout: float = PAGE_CONTENT_TIMEOUT) ->
return await page.content()


async def get_select2_options(page: Page) -> list[dict[str, Any]]:
async def get_select2_options(page: Page, element: ElementHandle) -> list[dict[str, Any]]:
await page.evaluate(JS_FUNCTION_DEFS)
js_script = "async () => await getSelect2Options()"
return await page.evaluate(js_script)
js_script = "async (element) => await getSelect2Options(element)"
return await page.evaluate(js_script, element)


async def get_interactable_element_tree_in_frame(
Expand Down
7 changes: 5 additions & 2 deletions skyvern/webeye/utils/dom.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,8 +198,11 @@ async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACT
async def close(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
await self.page.locator("#select2-drop").press("Escape", timeout=timeout)

async def get_options(self) -> typing.List[SkyvernOptionType]:
options = await get_select2_options(self.page)
async def get_options(
self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
) -> typing.List[SkyvernOptionType]:
element_handler = await self.skyvern_element.locator.element_handle(timeout=timeout)
options = await get_select2_options(self.page, element_handler)
return typing.cast(typing.List[SkyvernOptionType], options)

async def select_by_index(
Expand Down

0 comments on commit 690c1a1

Please sign in to comment.