Skip to content

Commit

Permalink
🔄 synced local 'skyvern/' with remote 'skyvern/'
Browse files Browse the repository at this point in the history
here's an example of a current action history:
```
[
   {
      "action":{
         "action_type":"click",
         "description":null,
         "reasoning":"The user needs to submit a support request to resolve the issue with the job application submission flagged as bot traffic. The 'Submit a new support request' link with id 11 is the relevant element to click to access the support request form, as indicated by the previous successful action results and the current screenshot showing the Greenhouse support bot prompt.",
         "element_id":11,
         "file_url":null
      },
      "results":[
         {
            "success":false,
            "exception_type":"MissingElement",
            "exception_message":"Found no elements. Might be due to previous actions which removed this element. xpath=//*[@unique_id='11'] element_id=11",
            "data":null,
            "step_retry_number":0,
            "step_order":8,
            "javascript_triggered":false,
            "interacted_with_sibling":false,
            "interacted_with_parent":false
         }
      ]
   },
   {
      "action":{
         "action_type":"click",
         "description":null,
         "reasoning":"The user needs to submit a support request to resolve the issue with the job application submission flagged as bot traffic. The 'Get help' button with id 8 is the relevant element to click to access the support request form, as indicated by the current screenshot showing the Greenhouse support bot prompt.",
         "element_id":8,
         "file_url":null
      },
      "results":[
         {
            "success":true,
            "exception_type":null,
            "exception_message":null,
            "data":null,
            "step_retry_number":2,
            "step_order":8,
            "javascript_triggered":false,
            "interacted_with_sibling":false,
            "interacted_with_parent":false
         }
      ]
   },
   {
      "action":{
         "action_type":"input_text",
         "description":null,
         "reasoning":"The user needs to interact with the Greenhouse support bot to resolve the issue with the job application submission flagged as bot traffic. The 'Write a message' input field is the relevant element to input text into, as indicated by the current screenshot showing the Greenhouse support bot prompt. The user will explain the situation and request assistance.",
         "element_id":44,
         "file_url":null
      },
      "results":[
         {
            "success":false,
            "exception_type":"Error",
            "exception_message":"Error: Node is not an <input>, <textarea> or <select> element",
            "data":null,
            "step_retry_number":0,
            "step_order":9,
            "javascript_triggered":false,
            "interacted_with_sibling":false,
            "interacted_with_parent":false
         }
      ]
   }
]
```

1. let's remove null
2.             "javascript_triggered":false,
            "interacted_with_sibling":false,
            "interacted_with_parent":false
these are definitely useless so let's remove them

3. TBD: I'm not sure how important successful actions are - shall we remove?
4.             "step_retry_number":0,
            "step_order":9,
TBD: are these important at all?
  • Loading branch information
ykeremy authored and wintonzheng committed May 5, 2024
1 parent be39516 commit b3bdccb
Showing 1 changed file with 34 additions and 13 deletions.
47 changes: 34 additions & 13 deletions skyvern/forge/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -616,20 +616,9 @@ async def _build_and_record_step_prompt(
num_elements=len(scraped_page.elements),
url=task.url,
)
# Get action results from the last app.SETTINGS.PROMPT_ACTION_HISTORY_WINDOW steps
steps = await app.DATABASE.get_task_steps(task_id=task.task_id, organization_id=task.organization_id)
window_steps = steps[-1 * SettingsManager.get_settings().PROMPT_ACTION_HISTORY_WINDOW :]
actions_and_results: list[tuple[ActionTypeUnion, list[ActionResult]]] = []
for window_step in window_steps:
if window_step.output and window_step.output.actions_and_results:
actions_and_results.extend(window_step.output.actions_and_results)

actions_and_results_str = json.dumps(
[
{"action": action.model_dump(), "results": [result.model_dump() for result in results]}
for action, results in actions_and_results
]
)
actions_and_results_str = await self._get_action_results(task)

# Generate the extract action prompt
navigation_goal = task.navigation_goal
starting_url = task.url
Expand Down Expand Up @@ -667,6 +656,38 @@ async def _build_and_record_step_prompt(

return scraped_page, extract_action_prompt

async def _get_action_results(self, task: Task) -> str:
# Get action results from the last app.SETTINGS.PROMPT_ACTION_HISTORY_WINDOW steps
steps = await app.DATABASE.get_task_steps(task_id=task.task_id, organization_id=task.organization_id)
window_steps = steps[-1 * SettingsManager.get_settings().PROMPT_ACTION_HISTORY_WINDOW :]
actions_and_results: list[tuple[ActionTypeUnion, list[ActionResult]]] = []
for window_step in window_steps:
if window_step.output and window_step.output.actions_and_results:
actions_and_results.extend(window_step.output.actions_and_results)

# shall we exclude successful actions?
return json.dumps(
[
{
"action": action.model_dump(exclude_none=True),
"results": [
result.model_dump(
exclude_none=True,
exclude={
"javascript_triggered",
"interacted_with_sibling",
"interacted_with_parent",
"step_retry_number",
"step_order",
},
)
for result in results
],
}
for action, results in actions_and_results
]
)

async def get_extracted_information_for_task(self, task: Task) -> dict[str, Any] | list | str | None:
"""
Find the last successful ScrapeAction for the task and return the extracted information.
Expand Down

0 comments on commit b3bdccb

Please sign in to comment.