update custom selection prompt (#689)

skyvern/forge/prompts/skyvern/custom-select.j2

@@ -1,4 +1,9 @@
 You are doing a select action on HTML page. Help to click the best match element for the target value among HTML elements based on the context.
+You can find the match element based on the following attempts:
+  1. Find the semantically most similar element
+  2. Reconsider if target value is reasonable based on context and the options in the HTML elements. If it doesn't make sense, you can tweak the target value into a reasonable one.
+  3. Find the element, which semantically is the superset of target value. Like "Others", "None of them matched"
+
 MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
 Each interactable element is tagged with an ID.
 

skyvern/webeye/actions/handler.py

@@ -886,8 +886,18 @@ async def select_from_dropdown(
             task=task,
         )
 
-    await incremental_scraped.get_incremental_element_tree(app.AGENT_FUNCTION.cleanup_element_tree)
-    html = incremental_scraped.build_html_tree()
+    trimmed_element_tree = await incremental_scraped.get_incremental_element_tree(
+        app.AGENT_FUNCTION.cleanup_element_tree
+    )
+    if dropdown_menu_element:
+        # if there's a dropdown menu detected, only elements in the dropdown should be sent to LLM
+        dropdown_id = dropdown_menu_element.get_id()
+        for head_node in trimmed_element_tree:
+            if head_node.get("id") == dropdown_id:
+                trimmed_element_tree = [head_node]
+                break
+
+    html = incremental_scraped.build_html_tree(element_tree=trimmed_element_tree)
 
     target_value = action.option.label or action.option.value
     if target_value is None:

skyvern/webeye/scraper/scraper.py

@@ -442,8 +442,8 @@ async def get_incremental_elements_num(self) -> int:
         js_script = "() => window.globalOneTimeIncrementElements.length"
         return await self.skyvern_frame.get_frame().evaluate(js_script)
 
-    def build_html_tree(self) -> str:
-        return "".join([json_to_html(element) for element in self.element_tree_trimmed])
+    def build_html_tree(self, element_tree: list[dict] = []) -> str:
+        return "".join([json_to_html(element) for element in (element_tree or self.element_tree_trimmed)])
 
 
 def trim_element_tree(elements: list[dict]) -> list[dict]: