diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py index 873ccad3f..f3db518d7 100644 --- a/skyvern/exceptions.py +++ b/skyvern/exceptions.py @@ -47,7 +47,7 @@ def __init__(self, script_name: str | None = None): class MissingElement(SkyvernException): - def __init__(self, xpath: str | None = None, element_id: int | None = None): + def __init__(self, xpath: str | None = None, element_id: str | None = None): super().__init__( f"Found no elements. Might be due to previous actions which removed this element." f" xpath={xpath} element_id={element_id}", @@ -55,7 +55,7 @@ def __init__(self, xpath: str | None = None, element_id: int | None = None): class MultipleElementsFound(SkyvernException): - def __init__(self, num: int, xpath: str | None = None, element_id: int | None = None): + def __init__(self, num: int, xpath: str | None = None, element_id: str | None = None): super().__init__( f"Found {num} elements. Expected 1. num_elements={num} xpath={xpath} element_id={element_id}", ) diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py index 2065c8e57..01fa78661 100644 --- a/skyvern/forge/agent.py +++ b/skyvern/forge/agent.py @@ -493,7 +493,7 @@ async def agent_step( # build a linked action chain by the action_idx action_linked_list: list[ActionLinkedNode] = [] - element_id_to_action_index: dict[int, int] = dict() + element_id_to_action_index: dict[str, int] = dict() for action_idx, action in enumerate(actions): node = ActionLinkedNode(action=action) action_linked_list.append(node) @@ -508,7 +508,7 @@ async def agent_step( element_id_to_action_index[action.element_id] = action_idx - element_id_to_last_action: dict[int, int] = dict() + element_id_to_last_action: dict[str, int] = dict() for action_idx, action_node in enumerate(action_linked_list): action = action_node.action if isinstance(action, WebAction): diff --git a/skyvern/forge/prompts/skyvern/extract-action-claude3-sonnet.j2 b/skyvern/forge/prompts/skyvern/extract-action-claude3-sonnet.j2 index 4b349df5a..c74c5a213 100644 --- a/skyvern/forge/prompts/skyvern/extract-action-claude3-sonnet.j2 +++ b/skyvern/forge/prompts/skyvern/extract-action-claude3-sonnet.j2 @@ -13,12 +13,12 @@ Reply in JSON format with the following keys: "reasoning": str, // The reasoning behind the action. Be specific, referencing any user information and their fields and element ids in your reasoning. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point. "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence "action_type": str, // It's a string enum: "CLICK", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION", "WAIT", "SOLVE_CAPTCHA", "COMPLETE", "TERMINATE". "CLICK" is an element you'd like to click. "INPUT_TEXT" is an element you'd like to input text into. "UPLOAD_FILE" is an element you'd like to upload a file into. "SELECT_OPTION" is an element you'd like to select an option from. "WAIT" action should be used if there are no actions to take and there is some indication on screen that waiting could yield more actions. "WAIT" should not be used if there are actions to take. "SOLVE_CAPTCHA" should be used if there's a captcha to solve on the screen. "COMPLETE" is used when the user goal has been achieved AND if there's any data extraction goal, you should be able to get data from the page. Never return a COMPLETE action unless you confirm user goal is achieved through the elements or the screenshots. "TERMINATE" is used to terminate the whole task with a failure when it doesn't seem like the user goal can be achieved. Do not use "TERMINATE" if waiting could lead the user towards the goal. Only return "TERMINATE" if you are on a page where the user goal cannot be achieved. If you are returning "COMPLETE" or "TERMINATE", never return any other action in the same response. The "COMPLETE" and "TERMINATE" actions can only be returned once in the whole task. When they are returned, they have to be the only action in the response. - "id": int, // The id of the element to take action on. The id has to be one from the elements list + "id": str, // The id of the element to take action on. The id has to be one from the elements list "text": str, // Text for INPUT_TEXT action only "file_url": str, // The url of the file to upload if applicable. This field must be present for UPLOAD_FILE but can also be present for CLICK only if the click is to upload the file. It should be null otherwise. "option": { // The option to select for SELECT_OPTION action only. null if not SELECT_OPTION action "label": str, // the label of the option if any. MAKE SURE YOU USE THIS LABEL TO SELECT THE OPTION. DO NOT PUT ANYTHING OTHER THAN A VALID OPTION LABEL HERE - "index": int, // the id corresponding to the optionIndex under the the select element. + "index": int, // the index corresponding to the option index under the the select element. "value": str // the value of the option. MAKE SURE YOU USE THIS VALUE TO SELECT THE OPTION. DO NOT PUT ANYTHING OTHER THAN A VALID OPTION VALUE HERE }, {% if error_code_mapping_str %} diff --git a/skyvern/forge/prompts/skyvern/extract-action.j2 b/skyvern/forge/prompts/skyvern/extract-action.j2 index 2b11a194e..eb088d0bb 100644 --- a/skyvern/forge/prompts/skyvern/extract-action.j2 +++ b/skyvern/forge/prompts/skyvern/extract-action.j2 @@ -13,13 +13,13 @@ Reply in JSON format with the following keys: "reasoning": str, // The reasoning behind the action. Be specific, referencing any user information and their fields and element ids in your reasoning. Mention why you chose the action type, and why you chose the element id. Keep the reasoning short and to the point. "confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence "action_type": str, // It's a string enum: "CLICK", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION", "WAIT", "SOLVE_CAPTCHA", "COMPLETE", "TERMINATE". "CLICK" is an element you'd like to click. "INPUT_TEXT" is an element you'd like to input text into. "UPLOAD_FILE" is an element you'd like to upload a file into. "SELECT_OPTION" is an element you'd like to select an option from. "WAIT" action should be used if there are no actions to take and there is some indication on screen that waiting could yield more actions. "WAIT" should not be used if there are actions to take. "SOLVE_CAPTCHA" should be used if there's a captcha to solve on the screen. "COMPLETE" is used when the user goal has been achieved AND if there's any data extraction goal, you should be able to get data from the page. Never return a COMPLETE action unless the user goal is achieved. "TERMINATE" is used to terminate the whole task with a failure when it doesn't seem like the user goal can be achieved. Do not use "TERMINATE" if waiting could lead the user towards the goal. Only return "TERMINATE" if you are on a page where the user goal cannot be achieved. All other actions are ignored when "TERMINATE" is returned. - "id": int, // The id of the element to take action on. The id has to be one from the elements list + "id": str, // The id of the element to take action on. The id has to be one from the elements list "text": str, // Text for INPUT_TEXT action only "file_url": str, // The url of the file to upload if applicable. This field must be present for UPLOAD_FILE but can also be present for CLICK only if the click is to upload the file. It should be null otherwise. "download": bool, // Can only be true for CLICK actions. If true, the browser will trigger a download by clicking the element. If false, the browser will click the element without triggering a download. "option": { // The option to select for SELECT_OPTION action only. null if not SELECT_OPTION action "label": str, // the label of the option if any. MAKE SURE YOU USE THIS LABEL TO SELECT THE OPTION. DO NOT PUT ANYTHING OTHER THAN A VALID OPTION LABEL HERE - "index": int, // the id corresponding to the optionIndex under the the select element. + "index": int, // the index corresponding to the option index under the select element. "value": str // the value of the option. MAKE SURE YOU USE THIS VALUE TO SELECT THE OPTION. DO NOT PUT ANYTHING OTHER THAN A VALID OPTION VALUE HERE }, {% if error_code_mapping_str %} diff --git a/skyvern/webeye/actions/actions.py b/skyvern/webeye/actions/actions.py index e8b208ff9..9b5340905 100644 --- a/skyvern/webeye/actions/actions.py +++ b/skyvern/webeye/actions/actions.py @@ -37,7 +37,7 @@ class Action(BaseModel): class WebAction(Action, abc.ABC): - element_id: int + element_id: str class UserDefinedError(BaseModel): diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py index af87f3b06..a4689ea70 100644 --- a/skyvern/webeye/actions/handler.py +++ b/skyvern/webeye/actions/handler.py @@ -1,7 +1,6 @@ import asyncio import json import os -import re import uuid from typing import Any, Awaitable, Callable, List @@ -516,40 +515,20 @@ async def handle_select_option_action( return [ActionFailure(e)] try: - option_xpath = scraped_page.id_to_xpath_dict[action.option.index] - match = re.search(r"option\[(\d+)]$", option_xpath) - if match: - # This means we were trying to select an option xpath, click the option - option_index = int(match.group(1)) - await page.click( - f"xpath={xpath}", - timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, - ) - await page.select_option( - xpath, - index=option_index, - timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, - ) - await page.click( - f"xpath={xpath}", - timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, - ) - return [ActionSuccess()] - else: - # This means the supplied index was for the select element, not a reference to the xpath dict - await page.click( - f"xpath={xpath}", - timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, - ) - await page.select_option( - xpath, - index=action.option.index, - timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, - ) - await page.click( - f"xpath={xpath}", - timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, - ) + # This means the supplied index was for the select element, not a reference to the xpath dict + await page.click( + f"xpath={xpath}", + timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, + ) + await page.select_option( + xpath, + index=action.option.index, + timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, + ) + await page.click( + f"xpath={xpath}", + timeout=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS, + ) return [ActionSuccess()] except Exception as e: LOG.warning("Failed to click on the option by index", action=action, exc_info=True) @@ -782,12 +761,11 @@ async def chain_click( page.remove_listener("filechooser", fc_func) -def get_anchor_to_click(scraped_page: ScrapedPage, element_id: int) -> str | None: +def get_anchor_to_click(scraped_page: ScrapedPage, element_id: str) -> str | None: """ Get the anchor tag under the label to click """ LOG.info("Getting anchor tag to click", element_id=element_id) - element_id = int(element_id) for ele in scraped_page.elements: if "id" in ele and ele["id"] == element_id: for child in ele["children"]: @@ -796,7 +774,7 @@ def get_anchor_to_click(scraped_page: ScrapedPage, element_id: int) -> str | Non return None -def get_select_id_in_label_children(scraped_page: ScrapedPage, element_id: int) -> int | None: +def get_select_id_in_label_children(scraped_page: ScrapedPage, element_id: str) -> str | None: """ search