refactor select2 (#485)

skyvern/exceptions.py

@@ -297,3 +297,10 @@ class MissingElementDict(SkyvernException):
 class MissingElementInIframe(SkyvernException):
     def __init__(self, element_id: str) -> None:
         super().__init__(f"Found no iframe includes the element. element_id={element_id}")
+
+
+class InputActionOnSelect2Dropdown(SkyvernException):
+    def __init__(self, element_id: str):
+        super().__init__(
+            f"Input action on a select element, please try to use select action on this element. element_id={element_id}"
+        )

skyvern/webeye/actions/handler.py

@@ -11,6 +11,7 @@ from playwright.async_api import Locator, Page, TimeoutError
 from skyvern.constants import INPUT_TEXT_TIMEOUT, REPO_ROOT_DIR
 from skyvern.exceptions import (
     ImaginaryFileUrl,
+    InputActionOnSelect2Dropdown,
     InvalidElementForTextInput,
     MissingElement,
     MissingFileUrl,
@@ -41,7 +42,7 @@ from skyvern.webeye.actions.actions import (
 from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccess
 from skyvern.webeye.browser_factory import BrowserState
 from skyvern.webeye.scraper.scraper import ScrapedPage
-from skyvern.webeye.utils.dom import resolve_locator
+from skyvern.webeye.utils.dom import DomUtil, InteractiveElement, Select2Dropdown, resolve_locator
 
 LOG = structlog.get_logger()
 TEXT_INPUT_DELAY = 10  # 10ms between each character input
@@ -241,6 +242,11 @@ async def handle_input_text_action(
     task: Task,
     step: Step,
 ) -> list[ActionResult]:
+    dom = DomUtil(scraped_page, page)
+    skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)
+    if await skyvern_element.is_select2_dropdown():
+        return [ActionFailure(InputActionOnSelect2Dropdown(element_id=action.element_id))]
+
     xpath, frame = await validate_actions_in_dom(action, page, scraped_page)
 
     locator = resolve_locator(scraped_page, page, frame, xpath)
@@ -392,6 +398,9 @@ async def handle_select_option_action(
     task: Task,
     step: Step,
 ) -> list[ActionResult]:
+    dom = DomUtil(scraped_page, page)
+    skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)
+
     xpath, frame = await validate_actions_in_dom(action, page, scraped_page)
 
     locator = resolve_locator(scraped_page, page, frame, xpath)
@@ -428,17 +437,23 @@ async def handle_select_option_action(
 
     # check if the element is an a tag first. If yes, click it instead of selecting the option
     if tag_name == "label":
-        # TODO: this is a hack to handle the case where the label is the only thing that's clickable
+        # label pointed to select2 <a> element
-        # it's a label, look for the anchor tag
+        select2_element_id: str | None = None
-        child_anchor_xpath = get_anchor_to_click(scraped_page, action.element_id)
+        # search <a> anchor first and then search <input> anchor
-        if child_anchor_xpath:
+        select2_element_id = skyvern_element.find_element_id_in_label_children(InteractiveElement.A)
-            LOG.info(
+        if select2_element_id is None:
-                "SelectOptionAction is a label tag. Clicking the anchor tag instead of selecting the option",
+            select2_element_id = skyvern_element.find_element_id_in_label_children(InteractiveElement.INPUT)
-                action=action,
+
-                child_anchor_xpath=child_anchor_xpath,
+        if select2_element_id is not None:
-            )
+            select2_skyvern_element = await dom.get_skyvern_element_by_id(element_id=select2_element_id)
-            click_action = ClickAction(element_id=action.element_id)
+            if await select2_skyvern_element.is_select2_dropdown():
-            return await chain_click(task, scraped_page, page, click_action, child_anchor_xpath, frame)
+                LOG.info(
+                    "SelectOptionAction is on <label>. take the action on the real select2 element",
+                    action=action,
+                    select2_element_id=select2_element_id,
+                )
+                select_action = SelectOptionAction(element_id=select2_element_id, option=action.option)
+                return await handle_select_option_action(select_action, page, scraped_page, task, step)
 
         # handler the select action on <label>
         select_element_id = get_select_id_in_label_children(scraped_page, action.element_id)
@@ -462,16 +477,77 @@ async def handle_select_option_action(
             check_action = CheckboxAction(element_id=checkbox_element_id, is_checked=True)
             return await handle_checkbox_action(check_action, page, scraped_page, task, step)
 
-        return [ActionFailure(Exception("No anchor tag or select children found for the label for SelectOptionAction"))]
+        return [ActionFailure(Exception("No element pointed by the label found"))]
-    elif tag_name == "a":
+    elif await skyvern_element.is_select2_dropdown():
-        # turn the SelectOptionAction into a ClickAction
         LOG.info(
-            "SelectOptionAction is an anchor tag. Clicking it instead of selecting the option",
+            "This is a select2 dropdown",
             action=action,
         )
-        click_action = ClickAction(element_id=action.element_id)
+        timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
-        action_result = await chain_click(task, scraped_page, page, click_action, xpath, frame)
+
-        return action_result
+        select2_element = Select2Dropdown(page=page, skyvern_element=skyvern_element)
+
+        await select2_element.open()
+        options = await select2_element.get_options()
+
+        result: List[ActionResult] = []
+        # select by label first, then by index
+        if action.option.label is not None or action.option.value is not None:
+            try:
+                for option in options:
+                    option_content = option.get("text")
+                    option_index = option.get("optionIndex", None)
+                    if option_index is None:
+                        LOG.warning(
+                            "Select2 option index is None",
+                            option=option,
+                        )
+                        continue
+                    if action.option.label == option_content or action.option.value == option_content:
+                        await select2_element.select_by_index(index=option_index, timeout=timeout)
+                        result.append(ActionSuccess())
+                        return result
+                LOG.info(
+                    "no target select2 option matched by label, try to select by index",
+                    action=action,
+                )
+            except Exception as e:
+                result.append(ActionFailure(e))
+                LOG.info(
+                    "failed to select by label in select2, try to select by index",
+                    exc_info=True,
+                    action=action,
+                )
+
+        if action.option.index is not None:
+            if action.option.index >= len(options):
+                result.append(ActionFailure(Exception("Select index out of bound")))
+                return result
+
+            try:
+                option_content = options[action.option.index].get("text")
+                if option_content != action.option.label:
+                    LOG.warning(
+                        "Select option label is not consistant to the action value. Might select wrong option.",
+                        option_content=option_content,
+                        action=action,
+                    )
+
+                await select2_element.select_by_index(index=action.option.index, timeout=timeout)
+                result.append(ActionSuccess())
+                return result
+            except Exception as e:
+                result.append(ActionFailure(e))
+                LOG.info(
+                    "failed to select by index in select2, try to select by label",
+                    exc_info=True,
+                    action=action,
+                )
+
+        if len(result) == 0:
+            result.append(ActionFailure(Exception("nothing is selected, try to select again.")))
+
+        return result
     elif tag_name == "ul" or tag_name == "div" or tag_name == "li":
         # if the role is listbox, find the option with the "label" or "value" and click that option element
         # references:

skyvern/webeye/scraper/domUtils.js

@@ -216,6 +216,10 @@ function isElementVisible(element) {
   if (element.tagName.toLowerCase() === "option")
     return element.parentElement && isElementVisible(element.parentElement);
 
+  if (element.className.toString().includes("select2-offscreen")) {
+    return false;
+  }
+
   const style = getElementComputedStyle(element);
   if (!style) return true;
   if (style.display === "contents") {
@@ -414,6 +418,20 @@ const isComboboxDropdown = (element) => {
   return role && haspopup && controls && readonly;
 };
 
+const isSelect2Dropdown = (element) => {
+  return (
+    element.tagName.toLowerCase() === "span" &&
+    element.className.toString().includes("select2-chosen")
+  );
+};
+
+const isSelect2MultiChoice = (element) => {
+  return (
+    element.tagName.toLowerCase() === "input" &&
+    element.className.toString().includes("select2-input")
+  );
+};
+
 const checkParentClass = (className) => {
   const targetParentClasses = ["field", "entry"];
   for (let i = 0; i < targetParentClasses.length; i++) {
@@ -594,6 +612,58 @@ function getListboxOptions(element) {
   return selectOptions;
 }
 
+async function getSelect2OptionElements() {
+  let optionList = [];
+
+  while (true) {
+    oldOptionCount = optionList.length;
+    let newOptionList = document.querySelectorAll(
+      "#select2-drop li[role='option']",
+    );
+    if (newOptionList.length === oldOptionCount) {
+      console.log("no more options loaded, wait 5s to query again");
+      // sometimes need more time to load the options, so sleep 10s and try again
+      await sleep(5000); // wait 5s
+      newOptionList = document.querySelectorAll(
+        "#select2-drop li[role='option']",
+      );
+      console.log(newOptionList.length, " options found, after 5s");
+    }
+
+    optionList = newOptionList;
+    if (optionList.length === 0 || optionList.length === oldOptionCount) {
+      break;
+    }
+
+    lastOption = optionList[optionList.length - 1];
+    if (!lastOption.className.toString().includes("select2-more-results")) {
+      break;
+    }
+    lastOption.scrollIntoView();
+  }
+
+  return optionList;
+}
+
+async function getSelect2Options() {
+  const optionList = await getSelect2OptionElements();
+
+  let selectOptions = [];
+  for (let i = 0; i < optionList.length; i++) {
+    let ele = optionList[i];
+    if (ele.className.toString().includes("select2-more-results")) {
+      continue;
+    }
+
+    selectOptions.push({
+      optionIndex: i,
+      text: removeMultipleSpaces(ele.textContent),
+    });
+  }
+
+  return selectOptions;
+}
+
 function uniqueId() {
   const characters =
     "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
@@ -605,63 +675,11 @@ function uniqueId() {
   return result;
 }
 
-function buildTreeFromBody(frame = "main.frame") {
+async function buildTreeFromBody(frame = "main.frame", open_select = false) {
   var elements = [];
   var resultArray = [];
 
-  const checkSelect2 = () => {
+  async function buildElementObject(element, interactable) {
-    const showInvisible = (element) => {
-      if (element.style.display === "none") {
-        element.style.removeProperty("display");
-        return true;
-      }
-
-      const removedClass = [];
-      for (let i = 0; i < element.classList.length; i++) {
-        const className = element.classList[i];
-        if (className.includes("hidden")) {
-          removedClass.push(className);
-        }
-      }
-      if (removedClass.length !== 0) {
-        removedClass.forEach((className) => {
-          element.classList.remove(className);
-        });
-        return true;
-      }
-      return false;
-    };
-    // according to select2(https://select2.org/getting-started/basic-usage)
-    // select2-container seems to be the most common class in select2,
-    // and the invisible select seems to be the sibling to the "select2-container" element.
-    const selectContainers = document.querySelectorAll(".select2-container");
-
-    selectContainers.forEach((element) => {
-      // search select in previous
-      let _pre = element.previousElementSibling;
-      while (_pre) {
-        if (_pre.tagName.toLowerCase() === "select" && showInvisible(_pre)) {
-          // only hide the select2 container when an alternative select found
-          element.style.display = "none";
-          return;
-        }
-        _pre = _pre.previousElementSibling;
-      }
-
-      // search select in next
-      let _next = element.nextElementSibling;
-      while (_next) {
-        if (_next.tagName.toLowerCase() === "select" && showInvisible(_next)) {
-          // only hide the select2 container when an alternative select found
-          element.style.display = "none";
-          return;
-        }
-        _next = _next.nextElementSibling;
-      }
-    });
-  };
-
-  function buildElementObject(element, interactable) {
     var element_id = element.getAttribute("unique_id") ?? uniqueId();
     var elementTagNameLower = element.tagName.toLowerCase();
     element.setAttribute("unique_id", element_id);
@@ -718,7 +736,7 @@ function buildTreeFromBody(frame = "main.frame") {
     } else if (attrs["role"] && attrs["role"].toLowerCase() === "listbox") {
       // if "role" key is inside attrs, then get all the elements with role "option" and get their text
       selectOptions = getListboxOptions(element);
-    } else if (isComboboxDropdown(element)) {
+    } else if (open_select && isComboboxDropdown(element)) {
       // open combobox dropdown to get options
       element.click();
       const listBox = document.getElementById(
@@ -735,6 +753,37 @@ function buildTreeFromBody(frame = "main.frame") {
           key: "Tab",
         }),
       );
+    } else if (open_select && isSelect2Dropdown(element)) {
+      // click element to show options
+      element.dispatchEvent(
+        new MouseEvent("mousedown", {
+          bubbles: true,
+          view: window,
+        }),
+      );
+
+      selectOptions = await getSelect2Options();
+
+      // HACK: click again to close the dropdown
+      element.dispatchEvent(
+        new MouseEvent("mousedown", {
+          bubbles: true,
+          view: window,
+        }),
+      );
+    } else if (open_select && isSelect2MultiChoice(element)) {
+      // click element to show options
+      element.click();
+      selectOptions = await getSelect2Options();
+
+      // HACK: press ESC to close the dropdown
+      element.dispatchEvent(
+        new KeyboardEvent("keydown", {
+          keyCode: 27,
+          bubbles: true,
+          key: "Escape",
+        }),
+      );
     }
     if (selectOptions) {
       elementObj.options = selectOptions;
@@ -750,7 +799,7 @@ function buildTreeFromBody(frame = "main.frame") {
       return [];
     }
   }
-  function processElement(element, parentId) {
+  async function processElement(element, parentId) {
     if (element === null) {
       console.log("get a null element");
       return;
@@ -766,7 +815,7 @@ function buildTreeFromBody(frame = "main.frame") {
 
     // Check if the element is interactable
     if (isInteractable(element)) {
-      var elementObj = buildElementObject(element, true);
+      var elementObj = await buildElementObject(element, true);
       elements.push(elementObj);
       // If the element is interactable but has no interactable parent,
       // then it starts a new tree, so add it to the result array
@@ -788,12 +837,14 @@ function buildTreeFromBody(frame = "main.frame") {
         return elementObj;
       }
       // Recursively process the children of the element
-      getChildElements(element).forEach((child) => {
+      const children = getChildElements(element);
-        processElement(child, elementObj.id);
+      for (let i = 0; i < children.length; i++) {
-      });
+        const childElement = children[i];
+        await processElement(childElement, elementObj.id);
+      }
       return elementObj;
     } else if (element.tagName.toLowerCase() === "iframe") {
-      let iframeElementObject = buildElementObject(element, false);
+      let iframeElementObject = await buildElementObject(element, false);
 
       elements.push(iframeElementObject);
       resultArray.push(iframeElementObject);
@@ -820,7 +871,7 @@ function buildTreeFromBody(frame = "main.frame") {
         // we don't use element context in HTML format,
         // so we need to make sure we parse all text node to avoid missing text in HTML.
         if (textContent && textContent.length <= 5000) {
-          var elementObj = buildElementObject(element, false);
+          var elementObj = await buildElementObject(element, false);
           elements.push(elementObj);
           if (parentId === null) {
             resultArray.push(elementObj);
@@ -833,9 +884,12 @@ function buildTreeFromBody(frame = "main.frame") {
           parentId = elementObj.id;
         }
       }
-      getChildElements(element).forEach((child) => {
+
-        processElement(child, parentId);
+      const children = getChildElements(element);
-      });
+      for (let i = 0; i < children.length; i++) {
+        const childElement = children[i];
+        await processElement(childElement, parentId);
+      }
     }
   }
 
@@ -1030,8 +1084,7 @@ function buildTreeFromBody(frame = "main.frame") {
 
   // TODO: Handle iframes
   // setup before parsing the dom
-  checkSelect2();
+  await processElement(document.body, null);
-  processElement(document.body, null);
 
   for (var element of elements) {
     if (
@@ -1247,17 +1300,17 @@ function removeBoundingBoxes() {
   }
 }
 
-function scrollToTop(draw_boxes) {
+async function scrollToTop(draw_boxes) {
   removeBoundingBoxes();
   window.scroll({ left: 0, top: 0, behavior: "instant" });
   if (draw_boxes) {
-    var elementsAndResultArray = buildTreeFromBody();
+    var elementsAndResultArray = await buildTreeFromBody();
     drawBoundingBoxes(elementsAndResultArray[0]);
   }
   return window.scrollY;
 }
 
-function scrollToNextPage(draw_boxes) {
+async function scrollToNextPage(draw_boxes) {
   // remove bounding boxes, scroll to next page with 200px overlap, then draw bounding boxes again
   // return true if there is a next page, false otherwise
   removeBoundingBoxes();
@@ -1267,8 +1320,12 @@ function scrollToNextPage(draw_boxes) {
     behavior: "instant",
   });
   if (draw_boxes) {
-    var elementsAndResultArray = buildTreeFromBody();
+    var elementsAndResultArray = await buildTreeFromBody();
     drawBoundingBoxes(elementsAndResultArray[0]);
   }
   return window.scrollY;
 }
+
+async function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

skyvern/webeye/scraper/scraper.py

@@ -297,6 +297,12 @@ async def scrape_web_unsafe(
     )
 
 
+async def get_select2_options(page: Page) -> list[dict[str, Any]]:
+    await page.evaluate(JS_FUNCTION_DEFS)
+    js_script = "async () => await getSelect2Options()"
+    return await page.evaluate(js_script)
+
+
 async def get_interactable_element_tree_in_frame(
     frames: list[Frame], elements: list[dict], element_tree: list[dict]
 ) -> tuple[list[dict], list[dict]]:
@@ -315,7 +321,7 @@ async def get_interactable_element_tree_in_frame(
 
         unique_id = await frame_element.get_attribute("unique_id")
 
-        frame_js_script = f"() => buildTreeFromBody('{unique_id}')"
+        frame_js_script = f"async () => await buildTreeFromBody('{unique_id}', true)"
 
         await frame.evaluate(JS_FUNCTION_DEFS)
         frame_elements, frame_element_tree = await frame.evaluate(frame_js_script)
@@ -345,7 +351,7 @@ async def get_interactable_element_tree(page: Page) -> tuple[list[dict], list[di
     :return: Tuple containing the element tree and a map of element IDs to elements.
     """
     await page.evaluate(JS_FUNCTION_DEFS)
-    main_frame_js_script = "() => buildTreeFromBody('main.frame')"
+    main_frame_js_script = "async () => await buildTreeFromBody('main.frame', true)"
     elements, element_tree = await page.evaluate(main_frame_js_script)
 
     # FIXME: some unexpected exception in iframe. turn off temporarily
@@ -365,7 +371,7 @@ async def scroll_to_top(page: Page, drow_boxes: bool) -> float:
     :return: Screenshot of the page.
     """
     await page.evaluate(JS_FUNCTION_DEFS)
-    js_script = f"() => scrollToTop({str(drow_boxes).lower()})"
+    js_script = f"async () => await scrollToTop({str(drow_boxes).lower()})"
     scroll_y_px = await page.evaluate(js_script)
     return scroll_y_px
 
@@ -378,7 +384,7 @@ async def scroll_to_next_page(page: Page, drow_boxes: bool) -> bool:
     :return: Screenshot of the page.
     """
     await page.evaluate(JS_FUNCTION_DEFS)
-    js_script = f"() => scrollToNextPage({str(drow_boxes).lower()})"
+    js_script = f"async () => await scrollToNextPage({str(drow_boxes).lower()})"
     scroll_y_px = await page.evaluate(js_script)
     return scroll_y_px
 

skyvern/webeye/utils/dom.py

@@ -1,3 +1,4 @@
+import asyncio
 import typing
 from enum import StrEnum
 
@@ -14,7 +15,7 @@ from skyvern.exceptions import (
     SkyvernException,
 )
 from skyvern.forge.sdk.settings_manager import SettingsManager
-from skyvern.webeye.scraper.scraper import ScrapedPage
+from skyvern.webeye.scraper.scraper import ScrapedPage, get_select2_options
 
 LOG = structlog.get_logger()
 
@@ -45,11 +46,17 @@ def resolve_locator(scrape_page: ScrapedPage, page: Page, frame: str, xpath: str
 
 
 class InteractiveElement(StrEnum):
+    A = "a"
     INPUT = "input"
     SELECT = "select"
     BUTTON = "button"
 
 
+class SkyvernOptionType(typing.TypedDict):
+    optionIndex: int
+    text: str
+
+
 class SkyvernElement:
     """
     SkyvernElement is a python interface to interact with js elements built during the scarping.
@@ -60,6 +67,18 @@ class SkyvernElement:
         self.__static_element = static_element
         self.locator = locator
 
+    async def is_select2_dropdown(self) -> bool:
+        tag_name = self.get_tag_name()
+        element_class = await self.get_attr("class")
+        if element_class is None:
+            return False
+        return (
+            (tag_name == "a" and "select2-choice" in element_class)
+            or (tag_name == "span" and "select2-chosen" in element_class)
+            or (tag_name == "span" and "select2-arrow" in element_class)
+            or (tag_name == "input" and "select2-input" in element_class)
+        )
+
     def get_tag_name(self) -> str:
         return self.__static_element.get("tagName", "")
 
@@ -136,3 +155,24 @@ class DomUtil:
             raise MultipleElementsFound(num=num_elements, xpath=xpath, element_id=element_id)
 
         return SkyvernElement(locator, element)
+
+
+class Select2Dropdown:
+    def __init__(self, page: Page, skyvern_element: SkyvernElement) -> None:
+        self.skyvern_element = skyvern_element
+        self.page = page
+
+    async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
+        await self.skyvern_element.locator.click(timeout=timeout)
+        # wait for the options to load
+        await asyncio.sleep(3)
+
+    async def get_options(self) -> typing.List[SkyvernOptionType]:
+        options = await get_select2_options(self.page)
+        return typing.cast(typing.List[SkyvernOptionType], options)
+
+    async def select_by_index(
+        self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
+    ) -> None:
+        anchor = self.page.locator("#select2-drop li[role='option']")
+        await anchor.nth(index).click(timeout=timeout)