fix(SKY-9402): preserve class and readable pseudo-text for icon-only navigation elements (#6092)

pyproject.toml

@@ -254,6 +254,7 @@ override-dependencies = [
 constraint-dependencies = [
     "authlib>=1.7.1",
     "flask>=3.1.3",
+    "idna>=3.15",
     "joserfc>=1.6.3",
     "mako>=1.3.12",
     "pyasn1>=0.6.3",

skyvern/webeye/scraper/scraped_page.py

@@ -1,5 +1,6 @@
 import copy
 import json
+import re
 import typing
 from abc import ABC, abstractmethod
 from enum import StrEnum
@@ -25,6 +26,14 @@ ELEMENT_NODE_ATTRIBUTES = {
     "id",
 }
 
+_PUA_PATTERN = re.compile(r"[\uE000-\uF8FF\U000F0000-\U000FFFFD\U00100000-\U0010FFFD]+")
+
+
+def _replace_pua_with_marker(text: str | None) -> str:
+    if not text:
+        return ""
+    return _PUA_PATTERN.sub("[icon]", text)
+
 
 def build_attribute(key: str, value: Any) -> str:
     if isinstance(value, bool) or isinstance(value, int):
@@ -89,8 +98,8 @@ def json_to_html(element: dict, need_skyvern_attrs: bool = True) -> str:
     if element.get("purgeable", False):
         return children_html + option_html
 
-    before_pseudo_text = element.get("beforePseudoText") or ""
-    after_pseudo_text = element.get("afterPseudoText") or ""
+    before_pseudo_text = _replace_pua_with_marker(element.get("beforePseudoText"))
+    after_pseudo_text = _replace_pua_with_marker(element.get("afterPseudoText"))
 
     # Check if the element is self-closing
     if (

skyvern/webeye/scraper/scraper.py

@@ -810,7 +810,11 @@ def trim_element(element: dict) -> dict:
                 del queue_ele["attributes"]
 
         if "attributes" in queue_ele and not queue_ele.get("keepAllAttr", False):
-            new_attributes = _trimmed_attributes(queue_ele["attributes"])
+            has_pseudo = bool(queue_ele.get("beforePseudoText") or queue_ele.get("afterPseudoText"))
+            is_icon_only = (
+                queue_ele.get("interactable", False) and not str(queue_ele.get("text", "")).strip() and has_pseudo
+            )
+            new_attributes = _trimmed_attributes(queue_ele["attributes"], keep_class=is_icon_only)
             if new_attributes:
                 queue_ele["attributes"] = new_attributes
             else:
@@ -861,7 +865,7 @@ def _trimmed_base64_data(attributes: dict) -> dict:
     return new_attributes
 
 
-def _trimmed_attributes(attributes: dict) -> dict:
+def _trimmed_attributes(attributes: dict, *, keep_class: bool = False) -> dict:
     new_attributes: dict = {}
 
     for key in attributes:
@@ -870,6 +874,13 @@ def _trimmed_attributes(attributes: dict) -> dict:
         if key in RESERVED_ATTRIBUTES:
             new_attributes[key] = attributes[key]
 
+    if keep_class and "class" in attributes:
+        cls = str(attributes["class"])
+        if len(cls) > 100:
+            last_space = cls.rfind(" ", 0, 100)
+            cls = cls[: last_space if last_space > 0 else 100]
+        new_attributes["class"] = cls
+
     return new_attributes
 
 

tests/unit/test_icon_only_element_serialization.py

@@ -0,0 +1,177 @@
+"""
+Tests for icon-only interactable element serialization.
+
+Icon-only navigation controls (e.g. CSS-rendered arrow buttons with no visible text)
+need their class attribute and pseudo-text preserved through the trim/serialization
+pipeline so LLMs can identify them.
+"""
+
+import copy
+
+import pytest
+
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.webeye.scraper.scraped_page import _replace_pua_with_marker, json_to_html
+from skyvern.webeye.scraper.scraper import _trimmed_attributes, trim_element
+
+
+@pytest.fixture(autouse=True)
+def _skyvern_ctx():
+    with skyvern_context.scoped(SkyvernContext(organization_id="o_test")):
+        yield
+
+
+ICON_ONLY_NAV_ELEMENT = {
+    "id": "AACO",
+    "frame": "main.frame",
+    "frame_index": 0,
+    "interactable": True,
+    "hoverOnly": False,
+    "tagName": "div",
+    "attributes": {
+        "class": "icon-nav forward",
+        "id": "widget_10",
+        "lang": "en",
+        "unique_id": "AACO",
+    },
+    "beforePseudoText": None,
+    "text": "",
+    "afterPseudoText": "\uf061",
+    "children": [],
+    "purgeable": False,
+    "keepAllAttr": False,
+    "isSelectable": False,
+}
+
+NORMAL_TEXT_BUTTON = {
+    "id": "NEXT1",
+    "frame": "main.frame",
+    "frame_index": 0,
+    "interactable": True,
+    "tagName": "button",
+    "attributes": {
+        "class": "btn-primary large-button extra-styles",
+        "type": "submit",
+        "unique_id": "NEXT1",
+    },
+    "text": "Submit",
+    "beforePseudoText": None,
+    "afterPseudoText": None,
+    "children": [],
+    "purgeable": False,
+    "keepAllAttr": False,
+    "isSelectable": False,
+}
+
+NON_INTERACTABLE_DIV = {
+    "id": "DIV1",
+    "frame": "main.frame",
+    "frame_index": 0,
+    "interactable": False,
+    "tagName": "div",
+    "attributes": {
+        "class": "layout-wrapper",
+        "unique_id": "DIV1",
+    },
+    "text": "",
+    "beforePseudoText": None,
+    "afterPseudoText": "\uf061",
+    "children": [],
+    "purgeable": False,
+    "keepAllAttr": False,
+    "isSelectable": False,
+}
+
+
+class TestTrimmedAttributesClassPreservation:
+    def test_icon_only_interactable_preserves_class(self):
+        attrs = {"class": "icon-nav forward", "data-dojo-type": "dojosite/Noop", "lang": "en"}
+        result = _trimmed_attributes(attrs, keep_class=True)
+        assert result["class"] == "icon-nav forward"
+
+    def test_normal_element_strips_class(self):
+        attrs = {"class": "btn-primary", "type": "submit"}
+        result = _trimmed_attributes(attrs, keep_class=False)
+        assert "class" not in result
+        assert result["type"] == "submit"
+
+    def test_long_class_truncated(self):
+        long_class = "tw-" + "x" * 200
+        attrs = {"class": long_class}
+        result = _trimmed_attributes(attrs, keep_class=True)
+        assert len(result["class"]) == 100
+
+    def test_reserved_attrs_always_kept(self):
+        attrs = {"aria-label": "Next", "class": "appGo", "lang": "en"}
+        result = _trimmed_attributes(attrs, keep_class=False)
+        assert result["aria-label"] == "Next"
+        assert "class" not in result
+        assert "lang" not in result
+
+
+class TestTrimElementIconOnly:
+    def test_icon_nav_element_retains_class_after_trim(self):
+        el = copy.deepcopy(ICON_ONLY_NAV_ELEMENT)
+        trimmed = trim_element(el)
+        assert "attributes" in trimmed
+        assert trimmed["attributes"].get("class") == "icon-nav forward"
+
+    def test_text_button_does_not_retain_class(self):
+        el = copy.deepcopy(NORMAL_TEXT_BUTTON)
+        trimmed = trim_element(el)
+        attrs = trimmed.get("attributes", {})
+        assert "class" not in attrs
+        assert attrs.get("type") == "submit"
+
+    def test_non_interactable_icon_does_not_retain_class(self):
+        el = copy.deepcopy(NON_INTERACTABLE_DIV)
+        trimmed = trim_element(el)
+        attrs = trimmed.get("attributes", {})
+        assert "class" not in attrs
+
+
+class TestPUAReplacement:
+    def test_fontawesome_arrow_right(self):
+        assert _replace_pua_with_marker("\uf061") == "[icon]"
+
+    def test_multiple_pua_chars(self):
+        assert _replace_pua_with_marker("\uf061\uf062") == "[icon]"
+
+    def test_mixed_pua_and_text(self):
+        assert _replace_pua_with_marker("Next \uf061") == "Next [icon]"
+
+    def test_no_pua(self):
+        assert _replace_pua_with_marker("Next") == "Next"
+
+    def test_empty_string(self):
+        assert _replace_pua_with_marker("") == ""
+
+    def test_none_returns_empty(self):
+        assert _replace_pua_with_marker(None) == ""
+
+
+class TestJsonToHtmlWithIconElements:
+    def test_icon_nav_after_trim_produces_readable_output(self):
+        el = copy.deepcopy(ICON_ONLY_NAV_ELEMENT)
+        trimmed = trim_element(el)
+        html = json_to_html(trimmed)
+        assert 'class="icon-nav forward"' in html
+        assert "[icon]" in html
+        assert "AACO" in html
+
+    def test_text_button_no_icon_marker(self):
+        el = copy.deepcopy(NORMAL_TEXT_BUTTON)
+        trimmed = trim_element(el)
+        html = json_to_html(trimmed)
+        assert "Submit" in html
+        assert "[icon]" not in html
+        assert "class=" not in html
+
+    def test_before_pseudo_pua_also_replaced(self):
+        el = copy.deepcopy(ICON_ONLY_NAV_ELEMENT)
+        el["beforePseudoText"] = "\uf060"
+        el["afterPseudoText"] = None
+        trimmed = trim_element(el)
+        html = json_to_html(trimmed)
+        assert "[icon]" in html

uv.lock

@@ -17,6 +17,7 @@ resolution-markers = [
 constraints = [
     { name = "authlib", specifier = ">=1.7.1" },
     { name = "flask", specifier = ">=3.1.3" },
+    { name = "idna", specifier = ">=3.15" },
     { name = "joserfc", specifier = ">=1.6.3" },
     { name = "mako", specifier = ">=1.3.12" },
     { name = "pyasn1", specifier = ">=0.6.3" },
@@ -1976,11 +1977,11 @@ wheels = [
 
 [[package]]
 name = "idna"
-version = "3.13"
+version = "3.15"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/ce/cc/762dfb036166873f0059f3b7de4565e1b5bc3d6f28a414c13da27e442f99/idna-3.13.tar.gz", hash = "sha256:585ea8fe5d69b9181ec1afba340451fba6ba764af97026f92a91d4eef164a242", size = 194210, upload-time = "2026-04-22T16:42:42.314Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/5d/13/ad7d7ca3808a898b4612b6fe93cde56b53f3034dcde235acb1f0e1df24c6/idna-3.13-py3-none-any.whl", hash = "sha256:892ea0cde124a99ce773decba204c5552b69c3c67ffd5f232eb7696135bc8bb3", size = 68629, upload-time = "2026-04-22T16:42:40.909Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" },
 ]
 
 [[package]]