vikarti.anatra/Skyvern
1
0
Fork 0
mirror of https://github.com/Skyvern-AI/skyvern.git synced 2025-09-12 16:29:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

refactor select2 (#485)

Browse source
This commit is contained in:
LawyZheng 2024-06-18 11:34:52 +08:00 • committed by GitHub
parent b300f9dcf0
commit be86a33c3b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 281 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

7
skyvern/exceptions.py
View file

@ -297,3 +297,10 @@ class MissingElementDict(SkyvernException):
class MissingElementInIframe(SkyvernException):
def __init__(self, element_id: str) -> None:
super().__init__(f"Found no iframe includes the element. element_id={element_id}")
class InputActionOnSelect2Dropdown(SkyvernException):
def __init__(self, element_id: str):
super().__init__(
f"Input action on a select element, please try to use select action on this element. element_id={element_id}"
)

108
skyvern/webeye/actions/handler.py
View file

@ -11,6 +11,7 @@ from playwright.async_api import Locator, Page, TimeoutError
from skyvern.constants import INPUT_TEXT_TIMEOUT, REPO_ROOT_DIR
from skyvern.exceptions import (
ImaginaryFileUrl,
InputActionOnSelect2Dropdown,
InvalidElementForTextInput,
MissingElement,
MissingFileUrl,
@ -41,7 +42,7 @@ from skyvern.webeye.actions.actions import (
from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccess
from skyvern.webeye.browser_factory import BrowserState
from skyvern.webeye.scraper.scraper import ScrapedPage
from skyvern.webeye.utils.dom import resolve_locator
from skyvern.webeye.utils.dom import DomUtil, InteractiveElement, Select2Dropdown, resolve_locator
LOG = structlog.get_logger()
TEXT_INPUT_DELAY = 10 # 10ms between each character input
@ -241,6 +242,11 @@ async def handle_input_text_action(
task: Task,
step: Step,
) -> list[ActionResult]:
dom = DomUtil(scraped_page, page)
skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)
if await skyvern_element.is_select2_dropdown():
return [ActionFailure(InputActionOnSelect2Dropdown(element_id=action.element_id))]
xpath, frame = await validate_actions_in_dom(action, page, scraped_page)
locator = resolve_locator(scraped_page, page, frame, xpath)
@ -392,6 +398,9 @@ async def handle_select_option_action(
task: Task,
step: Step,
) -> list[ActionResult]:
dom = DomUtil(scraped_page, page)
skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)
xpath, frame = await validate_actions_in_dom(action, page, scraped_page)
locator = resolve_locator(scraped_page, page, frame, xpath)
@ -428,17 +437,23 @@ async def handle_select_option_action(
# check if the element is an a tag first. If yes, click it instead of selecting the option
if tag_name == "label":
# TODO: this is a hack to handle the case where the label is the only thing that's clickable
# it's a label, look for the anchor tag
child_anchor_xpath = get_anchor_to_click(scraped_page, action.element_id)
if child_anchor_xpath:
# label pointed to select2 <a> element
select2_element_id: str | None = None
# search <a> anchor first and then search <input> anchor
select2_element_id = skyvern_element.find_element_id_in_label_children(InteractiveElement.A)
if select2_element_id is None:
select2_element_id = skyvern_element.find_element_id_in_label_children(InteractiveElement.INPUT)
if select2_element_id is not None:
select2_skyvern_element = await dom.get_skyvern_element_by_id(element_id=select2_element_id)
if await select2_skyvern_element.is_select2_dropdown():
LOG.info(
"SelectOptionAction is a label tag. Clicking the anchor tag instead of selecting the option",
"SelectOptionAction is on <label>. take the action on the real select2 element",
action=action,
child_anchor_xpath=child_anchor_xpath,
select2_element_id=select2_element_id,
)
click_action = ClickAction(element_id=action.element_id)
return await chain_click(task, scraped_page, page, click_action, child_anchor_xpath, frame)
select_action = SelectOptionAction(element_id=select2_element_id, option=action.option)
return await handle_select_option_action(select_action, page, scraped_page, task, step)
# handler the select action on <label>
select_element_id = get_select_id_in_label_children(scraped_page, action.element_id)
@ -462,16 +477,77 @@ async def handle_select_option_action(
check_action = CheckboxAction(element_id=checkbox_element_id, is_checked=True)
return await handle_checkbox_action(check_action, page, scraped_page, task, step)
return [ActionFailure(Exception("No anchor tag or select children found for the label for SelectOptionAction"))]
elif tag_name == "a":
# turn the SelectOptionAction into a ClickAction
return [ActionFailure(Exception("No element pointed by the label found"))]
elif await skyvern_element.is_select2_dropdown():
LOG.info(
"SelectOptionAction is an anchor tag. Clicking it instead of selecting the option",
"This is a select2 dropdown",
action=action,
)
click_action = ClickAction(element_id=action.element_id)
action_result = await chain_click(task, scraped_page, page, click_action, xpath, frame)
return action_result
timeout = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
select2_element = Select2Dropdown(page=page, skyvern_element=skyvern_element)
await select2_element.open()
options = await select2_element.get_options()
result: List[ActionResult] = []
# select by label first, then by index
if action.option.label is not None or action.option.value is not None:
try:
for option in options:
option_content = option.get("text")
option_index = option.get("optionIndex", None)
if option_index is None:
LOG.warning(
"Select2 option index is None",
option=option,
)
continue
if action.option.label == option_content or action.option.value == option_content:
await select2_element.select_by_index(index=option_index, timeout=timeout)
result.append(ActionSuccess())
return result
LOG.info(
"no target select2 option matched by label, try to select by index",
action=action,
)
except Exception as e:
result.append(ActionFailure(e))
LOG.info(
"failed to select by label in select2, try to select by index",
exc_info=True,
action=action,
)
if action.option.index is not None:
if action.option.index >= len(options):
result.append(ActionFailure(Exception("Select index out of bound")))
return result
try:
option_content = options[action.option.index].get("text")
if option_content != action.option.label:
LOG.warning(
"Select option label is not consistant to the action value. Might select wrong option.",
option_content=option_content,
action=action,
)
await select2_element.select_by_index(index=action.option.index, timeout=timeout)
result.append(ActionSuccess())
return result
except Exception as e:
result.append(ActionFailure(e))
LOG.info(
"failed to select by index in select2, try to select by label",
exc_info=True,
action=action,
)
if len(result) == 0:
result.append(ActionFailure(Exception("nothing is selected, try to select again.")))
return result
elif tag_name == "ul" or tag_name == "div" or tag_name == "li":
# if the role is listbox, find the option with the "label" or "value" and click that option element
# references:

199
skyvern/webeye/scraper/domUtils.js
View file

@ -216,6 +216,10 @@ function isElementVisible(element) {
if (element.tagName.toLowerCase() === "option")
return element.parentElement && isElementVisible(element.parentElement);
if (element.className.toString().includes("select2-offscreen")) {
return false;
}
const style = getElementComputedStyle(element);
if (!style) return true;
if (style.display === "contents") {
@ -414,6 +418,20 @@ const isComboboxDropdown = (element) => {
return role && haspopup && controls && readonly;
};
const isSelect2Dropdown = (element) => {
return (
element.tagName.toLowerCase() === "span" &&
element.className.toString().includes("select2-chosen")
);
};
const isSelect2MultiChoice = (element) => {
return (
element.tagName.toLowerCase() === "input" &&
element.className.toString().includes("select2-input")
);
};
const checkParentClass = (className) => {
const targetParentClasses = ["field", "entry"];
for (let i = 0; i < targetParentClasses.length; i++) {
@ -594,6 +612,58 @@ function getListboxOptions(element) {
return selectOptions;
}
async function getSelect2OptionElements() {
let optionList = [];
while (true) {
oldOptionCount = optionList.length;
let newOptionList = document.querySelectorAll(
"#select2-drop li[role='option']",
);
if (newOptionList.length === oldOptionCount) {
console.log("no more options loaded, wait 5s to query again");
// sometimes need more time to load the options, so sleep 10s and try again
await sleep(5000); // wait 5s
newOptionList = document.querySelectorAll(
"#select2-drop li[role='option']",
);
console.log(newOptionList.length, " options found, after 5s");
}
optionList = newOptionList;
if (optionList.length === 0 || optionList.length === oldOptionCount) {
break;
}
lastOption = optionList[optionList.length - 1];
if (!lastOption.className.toString().includes("select2-more-results")) {
break;
}
lastOption.scrollIntoView();
}
return optionList;
}
async function getSelect2Options() {
const optionList = await getSelect2OptionElements();
let selectOptions = [];
for (let i = 0; i < optionList.length; i++) {
let ele = optionList[i];
if (ele.className.toString().includes("select2-more-results")) {
continue;
}
selectOptions.push({
optionIndex: i,
text: removeMultipleSpaces(ele.textContent),
});
}
return selectOptions;
}
function uniqueId() {
const characters =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
@ -605,63 +675,11 @@ function uniqueId() {
return result;
}
function buildTreeFromBody(frame = "main.frame") {
async function buildTreeFromBody(frame = "main.frame", open_select = false) {
var elements = [];
var resultArray = [];
const checkSelect2 = () => {
const showInvisible = (element) => {
if (element.style.display === "none") {
element.style.removeProperty("display");
return true;
}
const removedClass = [];
for (let i = 0; i < element.classList.length; i++) {
const className = element.classList[i];
if (className.includes("hidden")) {
removedClass.push(className);
}
}
if (removedClass.length !== 0) {
removedClass.forEach((className) => {
element.classList.remove(className);
});
return true;
}
return false;
};
// according to select2(https://select2.org/getting-started/basic-usage)
// select2-container seems to be the most common class in select2,
// and the invisible select seems to be the sibling to the "select2-container" element.
const selectContainers = document.querySelectorAll(".select2-container");
selectContainers.forEach((element) => {
// search select in previous
let _pre = element.previousElementSibling;
while (_pre) {
if (_pre.tagName.toLowerCase() === "select" && showInvisible(_pre)) {
// only hide the select2 container when an alternative select found
element.style.display = "none";
return;
}
_pre = _pre.previousElementSibling;
}
// search select in next
let _next = element.nextElementSibling;
while (_next) {
if (_next.tagName.toLowerCase() === "select" && showInvisible(_next)) {
// only hide the select2 container when an alternative select found
element.style.display = "none";
return;
}
_next = _next.nextElementSibling;
}
});
};
function buildElementObject(element, interactable) {
async function buildElementObject(element, interactable) {
var element_id = element.getAttribute("unique_id") ?? uniqueId();
var elementTagNameLower = element.tagName.toLowerCase();
element.setAttribute("unique_id", element_id);
@ -718,7 +736,7 @@ function buildTreeFromBody(frame = "main.frame") {
} else if (attrs["role"] && attrs["role"].toLowerCase() === "listbox") {
// if "role" key is inside attrs, then get all the elements with role "option" and get their text
selectOptions = getListboxOptions(element);
} else if (isComboboxDropdown(element)) {
} else if (open_select && isComboboxDropdown(element)) {
// open combobox dropdown to get options
element.click();
const listBox = document.getElementById(
@ -735,6 +753,37 @@ function buildTreeFromBody(frame = "main.frame") {
key: "Tab",
}),
);
} else if (open_select && isSelect2Dropdown(element)) {
// click element to show options
element.dispatchEvent(
new MouseEvent("mousedown", {
bubbles: true,
view: window,
}),
);
selectOptions = await getSelect2Options();
// HACK: click again to close the dropdown
element.dispatchEvent(
new MouseEvent("mousedown", {
bubbles: true,
view: window,
}),
);
} else if (open_select && isSelect2MultiChoice(element)) {
// click element to show options
element.click();
selectOptions = await getSelect2Options();
// HACK: press ESC to close the dropdown
element.dispatchEvent(
new KeyboardEvent("keydown", {
keyCode: 27,
bubbles: true,
key: "Escape",
}),
);
}
if (selectOptions) {
elementObj.options = selectOptions;
@ -750,7 +799,7 @@ function buildTreeFromBody(frame = "main.frame") {
return [];
}
}
function processElement(element, parentId) {
async function processElement(element, parentId) {
if (element === null) {
console.log("get a null element");
return;
@ -766,7 +815,7 @@ function buildTreeFromBody(frame = "main.frame") {
// Check if the element is interactable
if (isInteractable(element)) {
var elementObj = buildElementObject(element, true);
var elementObj = await buildElementObject(element, true);
elements.push(elementObj);
// If the element is interactable but has no interactable parent,
// then it starts a new tree, so add it to the result array
@ -788,12 +837,14 @@ function buildTreeFromBody(frame = "main.frame") {
return elementObj;
}
// Recursively process the children of the element
getChildElements(element).forEach((child) => {
processElement(child, elementObj.id);
});
const children = getChildElements(element);
for (let i = 0; i < children.length; i++) {
const childElement = children[i];
await processElement(childElement, elementObj.id);
}
return elementObj;
} else if (element.tagName.toLowerCase() === "iframe") {
let iframeElementObject = buildElementObject(element, false);
let iframeElementObject = await buildElementObject(element, false);
elements.push(iframeElementObject);
resultArray.push(iframeElementObject);
@ -820,7 +871,7 @@ function buildTreeFromBody(frame = "main.frame") {
// we don't use element context in HTML format,
// so we need to make sure we parse all text node to avoid missing text in HTML.
if (textContent && textContent.length <= 5000) {
var elementObj = buildElementObject(element, false);
var elementObj = await buildElementObject(element, false);
elements.push(elementObj);
if (parentId === null) {
resultArray.push(elementObj);
@ -833,9 +884,12 @@ function buildTreeFromBody(frame = "main.frame") {
parentId = elementObj.id;
}
}
getChildElements(element).forEach((child) => {
processElement(child, parentId);
});
const children = getChildElements(element);
for (let i = 0; i < children.length; i++) {
const childElement = children[i];
await processElement(childElement, parentId);
}
}
}
@ -1030,8 +1084,7 @@ function buildTreeFromBody(frame = "main.frame") {
// TODO: Handle iframes
// setup before parsing the dom
checkSelect2();
processElement(document.body, null);
await processElement(document.body, null);
for (var element of elements) {
if (
@ -1247,17 +1300,17 @@ function removeBoundingBoxes() {
}
}
function scrollToTop(draw_boxes) {
async function scrollToTop(draw_boxes) {
removeBoundingBoxes();
window.scroll({ left: 0, top: 0, behavior: "instant" });
if (draw_boxes) {
var elementsAndResultArray = buildTreeFromBody();
var elementsAndResultArray = await buildTreeFromBody();
drawBoundingBoxes(elementsAndResultArray[0]);
}
return window.scrollY;
}
function scrollToNextPage(draw_boxes) {
async function scrollToNextPage(draw_boxes) {
// remove bounding boxes, scroll to next page with 200px overlap, then draw bounding boxes again
// return true if there is a next page, false otherwise
removeBoundingBoxes();
@ -1267,8 +1320,12 @@ function scrollToNextPage(draw_boxes) {
behavior: "instant",
});
if (draw_boxes) {
var elementsAndResultArray = buildTreeFromBody();
var elementsAndResultArray = await buildTreeFromBody();
drawBoundingBoxes(elementsAndResultArray[0]);
}
return window.scrollY;
}
async function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}

14
skyvern/webeye/scraper/scraper.py
View file

@ -297,6 +297,12 @@ async def scrape_web_unsafe(
)
async def get_select2_options(page: Page) -> list[dict[str, Any]]:
await page.evaluate(JS_FUNCTION_DEFS)
js_script = "async () => await getSelect2Options()"
return await page.evaluate(js_script)
async def get_interactable_element_tree_in_frame(
frames: list[Frame], elements: list[dict], element_tree: list[dict]
) -> tuple[list[dict], list[dict]]:
@ -315,7 +321,7 @@ async def get_interactable_element_tree_in_frame(
unique_id = await frame_element.get_attribute("unique_id")
frame_js_script = f"() => buildTreeFromBody('{unique_id}')"
frame_js_script = f"async () => await buildTreeFromBody('{unique_id}', true)"
await frame.evaluate(JS_FUNCTION_DEFS)
frame_elements, frame_element_tree = await frame.evaluate(frame_js_script)
@ -345,7 +351,7 @@ async def get_interactable_element_tree(page: Page) -> tuple[list[dict], list[di
:return: Tuple containing the element tree and a map of element IDs to elements.
"""
await page.evaluate(JS_FUNCTION_DEFS)
main_frame_js_script = "() => buildTreeFromBody('main.frame')"
main_frame_js_script = "async () => await buildTreeFromBody('main.frame', true)"
elements, element_tree = await page.evaluate(main_frame_js_script)
# FIXME: some unexpected exception in iframe. turn off temporarily
@ -365,7 +371,7 @@ async def scroll_to_top(page: Page, drow_boxes: bool) -> float:
:return: Screenshot of the page.
"""
await page.evaluate(JS_FUNCTION_DEFS)
js_script = f"() => scrollToTop({str(drow_boxes).lower()})"
js_script = f"async () => await scrollToTop({str(drow_boxes).lower()})"
scroll_y_px = await page.evaluate(js_script)
return scroll_y_px
@ -378,7 +384,7 @@ async def scroll_to_next_page(page: Page, drow_boxes: bool) -> bool:
:return: Screenshot of the page.
"""
await page.evaluate(JS_FUNCTION_DEFS)
js_script = f"() => scrollToNextPage({str(drow_boxes).lower()})"
js_script = f"async () => await scrollToNextPage({str(drow_boxes).lower()})"
scroll_y_px = await page.evaluate(js_script)
return scroll_y_px

42
skyvern/webeye/utils/dom.py
View file

@ -1,3 +1,4 @@
import asyncio
import typing
from enum import StrEnum
@ -14,7 +15,7 @@ from skyvern.exceptions import (
SkyvernException,
)
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.webeye.scraper.scraper import ScrapedPage
from skyvern.webeye.scraper.scraper import ScrapedPage, get_select2_options
LOG = structlog.get_logger()
@ -45,11 +46,17 @@ def resolve_locator(scrape_page: ScrapedPage, page: Page, frame: str, xpath: str
class InteractiveElement(StrEnum):
A = "a"
INPUT = "input"
SELECT = "select"
BUTTON = "button"
class SkyvernOptionType(typing.TypedDict):
optionIndex: int
text: str
class SkyvernElement:
"""
SkyvernElement is a python interface to interact with js elements built during the scarping.
@ -60,6 +67,18 @@ class SkyvernElement:
self.__static_element = static_element
self.locator = locator
async def is_select2_dropdown(self) -> bool:
tag_name = self.get_tag_name()
element_class = await self.get_attr("class")
if element_class is None:
return False
return (
(tag_name == "a" and "select2-choice" in element_class)
or (tag_name == "span" and "select2-chosen" in element_class)
or (tag_name == "span" and "select2-arrow" in element_class)
or (tag_name == "input" and "select2-input" in element_class)
)
def get_tag_name(self) -> str:
return self.__static_element.get("tagName", "")
@ -136,3 +155,24 @@ class DomUtil:
raise MultipleElementsFound(num=num_elements, xpath=xpath, element_id=element_id)
return SkyvernElement(locator, element)
class Select2Dropdown:
def __init__(self, page: Page, skyvern_element: SkyvernElement) -> None:
self.skyvern_element = skyvern_element
self.page = page
async def open(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
await self.skyvern_element.locator.click(timeout=timeout)
# wait for the options to load
await asyncio.sleep(3)
async def get_options(self) -> typing.List[SkyvernOptionType]:
options = await get_select2_options(self.page)
return typing.cast(typing.List[SkyvernOptionType], options)
async def select_by_index(
self, index: int, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
) -> None:
anchor = self.page.locator("#select2-drop li[role='option']")
await anchor.nth(index).click(timeout=timeout)