From 6d02f8a4e9430770336e7090191674ee709879ef Mon Sep 17 00:00:00 2001 From: LawyZheng Date: Tue, 21 May 2024 17:02:16 +0800 Subject: [PATCH 1/2] better handle duplicate elements --- skyvern/forge/agent.py | 71 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 6 deletions(-) diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py index e6cf3d9fc..5ac0174cb 100644 --- a/skyvern/forge/agent.py +++ b/skyvern/forge/agent.py @@ -48,6 +48,10 @@ LOG = structlog.get_logger() +class ActionLinkedNode: + def __init__(self, action: ActionTypeUnion) -> None: + self.action = action + self.next: ActionLinkedNode | None = None class ForgeAgent: def __init__(self) -> None: @@ -457,16 +461,57 @@ async def agent_step( # of an exception, we can still see all the actions detailed_agent_step_output.actions_and_results = [(action, []) for action in actions] - web_action_element_ids = set() + # build a linked action chain by the action_idx + action_linked_list: list[ActionLinkedNode] = [] + element_id_to_action_index: dict[int, int] = dict() for action_idx, action in enumerate(actions): + node = ActionLinkedNode(action=action) + action_linked_list.append(node) + + previous_action_idx = element_id_to_action_index.get(action.element_id) + if previous_action_idx is not None: + previous_node = action_linked_list[previous_action_idx] + previous_node.next = node + + element_id_to_action_index[action.element_id] = action_idx + + element_id_to_last_action: dict[int, int] = dict() + for action_idx, action_node in enumerate(action_linked_list): + action = action_node.action if isinstance(action, WebAction): - if action.element_id in web_action_element_ids: - LOG.error( - "Duplicate action element id. Action handling stops", + previous_action_idx = element_id_to_last_action.get(action.element_id) + if previous_action_idx is not None: + LOG.warning( + "Duplicate action element id.", + task_id=task.task_id, + step_id=step.step_id, + step_order=step.order, action=action, ) - break - web_action_element_ids.add(action.element_id) + + # if the last action succeeded, then skip handling + previous_action, previous_result = detailed_agent_step_output.actions_and_results[previous_action_idx] + if len(previous_result) > 0 and previous_result[-1].success: + LOG.info( + "Previous action succeeded, so skip this one.", + task_id=task.task_id, + step_id=step.step_id, + step_order=step.order, + previouse_action=previous_action, + previouse_result=previous_result, + ) + continue + + LOG.warning( + "Previous action failed, so handle this action.", + task_id=task.task_id, + step_id=step.step_id, + step_order=step.order, + previouse_action=previous_action, + previouse_result=previous_result, + ) + + element_id_to_last_action[action.element_id] = action_idx self.async_operation_pool.run_operation(task.task_id, AgentPhase.action) results = await ActionHandler.handle_action(scraped_page, task, step, browser_state, action) @@ -505,6 +550,20 @@ async def agent_step( # stop executing the rest actions break else: + if action_node.next is not None: + LOG.warning( + "Action failed, but have duplicated element id in the action list. Continue excuting.", + task_id=task.task_id, + step_id=step.step_id, + step_order=step.order, + step_retry=step.retry_index, + action_idx=action_idx, + action=action, + next_action=action_node.next.action, + action_result=results, + ) + continue + LOG.warning( "Action failed, marking step as failed", task_id=task.task_id, From 37ad5365f095ba24fe26a3ec6e16ed9472e8b219 Mon Sep 17 00:00:00 2001 From: LawyZheng Date: Tue, 21 May 2024 17:03:10 +0800 Subject: [PATCH 2/2] better handle duplicate elements --- skyvern/forge/agent.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py index 5ac0174cb..9b9f48349 100644 --- a/skyvern/forge/agent.py +++ b/skyvern/forge/agent.py @@ -48,11 +48,13 @@ LOG = structlog.get_logger() + class ActionLinkedNode: def __init__(self, action: ActionTypeUnion) -> None: self.action = action self.next: ActionLinkedNode | None = None + class ForgeAgent: def __init__(self) -> None: if SettingsManager.get_settings().ADDITIONAL_MODULES: @@ -490,7 +492,9 @@ async def agent_step( ) # if the last action succeeded, then skip handling - previous_action, previous_result = detailed_agent_step_output.actions_and_results[previous_action_idx] + previous_action, previous_result = detailed_agent_step_output.actions_and_results[ + previous_action_idx + ] if len(previous_result) > 0 and previous_result[-1].success: LOG.info( "Previous action succeeded, so skip this one.",