vikarti.anatra/Skyvern
1
0
Fork 0
mirror of https://github.com/Skyvern-AI/skyvern.git synced 2026-07-09 16:09:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Replace non-vision blob with enrich_tree experiment (#6266)
Some checks are pending
Auto Create GitHub Release on Version Change / check-version-change (push) Waiting to run
Details
Auto Create GitHub Release on Version Change / create-release (push) Blocked by required conditions
Details
Run tests and pre-commit / Run tests and pre-commit hooks (push) Waiting to run
Details
Run tests and pre-commit / Frontend Lint and Build (push) Waiting to run
Details
Run tests and pre-commit / pip Package Smoke Tests (3.11) (push) Waiting to run
Details
Run tests and pre-commit / pip Package Smoke Tests (3.13) (push) Waiting to run
Details
Publish Fern Docs / run (push) Waiting to run
Details
Build Skyvern SDK and publish to PyPI / check-version-change (push) Waiting to run
Details
Build Skyvern SDK and publish to PyPI / run-ci (push) Blocked by required conditions
Details
Build Skyvern SDK and publish to PyPI / build-sdk (push) Blocked by required conditions
Details
Build Skyvern SDK and publish to PyPI / publish-sdk (push) Blocked by required conditions
Details

Browse source
This commit is contained in:
Cindy Li 2026-05-29 22:49:41 -04:00 • committed by GitHub
parent 9ffa39d5a2
commit d89ad158db
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
25 changed files with 516 additions and 854 deletions
Download patch file Download diff file Expand all files Collapse all files

22
skyvern/core/script_generations/real_skyvern_page_ai.py
View file

@ -46,7 +46,6 @@ from skyvern.webeye.actions.handler import (
handle_upload_file_action,
)
from skyvern.webeye.actions.parse_actions import parse_actions
from skyvern.webeye.scraper.non_vision_context import build_non_vision_page_context_if_needed
from skyvern.webeye.scraper.scraped_page import ScrapedPage
jinja_sandbox_env = SandboxedEnvironment()
@ -230,10 +229,6 @@ class RealSkyvernPageAi(SkyvernPageAi):
elements=element_tree,
local_datetime=datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat(),
user_context=context.prompt,
non_vision_page_context=await build_non_vision_page_context_if_needed(
scraped_page=self.scraped_page,
page=getattr(self, "page", None),
),
)
json_response = await app.SINGLE_CLICK_AGENT_LLM_API_HANDLER(
prompt=single_click_prompt,
@ -1344,10 +1339,6 @@ class RealSkyvernPageAi(SkyvernPageAi):
extracted_text=extracted_text_for_prompt,
error_code_mapping_str=error_code_mapping_str,
local_datetime=local_datetime_str,
non_vision_page_context=await build_non_vision_page_context_if_needed(
scraped_page=self.scraped_page,
page=getattr(self, "page", None),
),
)
# Cache extract-information within this script-generation path. The
@ -1390,7 +1381,6 @@ class RealSkyvernPageAi(SkyvernPageAi):
error_code_mapping=error_code_mapping_str,
llm_key=None,
workflow_system_prompt=workflow_system_prompt,
non_vision_page_context=post_ceiling_kwargs.get("non_vision_page_context"),
)
lookup_result = extraction_cache.lookup(workflow_run_id, cache_key)
except Exception:
@ -1514,10 +1504,6 @@ class RealSkyvernPageAi(SkyvernPageAi):
data_extraction_goal=prompt_rendered,
current_url=scraped_page_refreshed.url,
local_datetime=datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat(),
non_vision_page_context=await build_non_vision_page_context_if_needed(
scraped_page=scraped_page_refreshed,
page=getattr(self, "page", None),
),
)
step = None
@ -1675,10 +1661,6 @@ class RealSkyvernPageAi(SkyvernPageAi):
current_url=self.page.url,
elements=element_tree,
local_datetime=local_datetime,
non_vision_page_context=await build_non_vision_page_context_if_needed(
scraped_page=self.scraped_page,
page=getattr(self, "page", None),
),
)
try:
@ -1705,10 +1687,6 @@ class RealSkyvernPageAi(SkyvernPageAi):
current_url=self.page.url,
elements=element_tree,
local_datetime=local_datetime,
non_vision_page_context=await build_non_vision_page_context_if_needed(
scraped_page=self.scraped_page,
page=getattr(self, "page", None),
),
)
action_response = await llm_handler(
prompt=single_action_prompt,

43
skyvern/forge/agent.py
View file

@ -95,8 +95,8 @@ from skyvern.forge.sdk.core.security import generate_skyvern_webhook_signature
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db.enums import TaskType
from skyvern.forge.sdk.event.factory import EventStrategyFactory
from skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_for_context
from skyvern.forge.sdk.experimentation.llm_prompt_config import resolve_check_user_goal_handler
from skyvern.forge.sdk.experimentation.llm_vision_mode import resolve_llm_vision_mode_for_context
from skyvern.forge.sdk.log_artifacts import save_step_logs, save_task_logs
from skyvern.forge.sdk.models import SpeculativeLLMMetadata, Step, StepStatus
from skyvern.forge.sdk.schemas.files import FileInfo
@ -159,7 +159,6 @@ from skyvern.webeye.actions.parse_actions import (
)
from skyvern.webeye.actions.responses import ActionResult, ActionSuccess
from skyvern.webeye.browser_state import BrowserState
from skyvern.webeye.scraper.non_vision_context import build_non_vision_page_context_if_needed
from skyvern.webeye.scraper.scraped_page import ElementTreeFormat, ScrapedPage
from skyvern.webeye.utils.page import SkyvernFrame
@ -1344,7 +1343,7 @@ class ForgeAgent:
context.step_id = step.step_id
context.step_retry_index = step.retry_index
if not task.workflow_run_id and step.order == 0 and step.retry_index == 0:
await resolve_llm_vision_mode_for_context(
await resolve_enrich_tree_for_context(
context,
task.task_id,
task.organization_id,
@ -2781,10 +2780,7 @@ class ForgeAgent:
# no new_elements_ids threading — so we also drop Skyvern IDs.
_ctx = skyvern_context.current()
lean_enabled = bool(_ctx and _ctx.enable_lean_element_tree)
non_vision_page_context = await build_non_vision_page_context_if_needed(
scraped_page=scraped_page_refreshed,
page=page,
)
llm_screenshots_enabled = bool(_ctx and _ctx.llm_screenshots_enabled_for_prompt(retry_index=step.retry_index))
verification_prompt = load_prompt_with_elements(
element_tree_builder=scraped_page_refreshed,
prompt_engine=prompt_engine,
@ -2795,7 +2791,7 @@ class ForgeAgent:
terminate_criterion=task.terminate_criterion,
action_history=actions_and_results_str,
local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),
non_vision_page_context=non_vision_page_context,
without_screenshots=not llm_screenshots_enabled,
html_need_skyvern_attrs=False,
lean_compress_long_href=lean_enabled,
lean_compress_image_src=lean_enabled,
@ -3488,7 +3484,8 @@ class ForgeAgent:
verification_code_check: bool,
show_close_page_action: bool,
complete_criterion: str | None,
non_vision_enabled: bool = False,
enriched_tree_enabled: bool = False,
llm_screenshots_enabled: bool = True,
) -> str:
"""
Build a short-but-unique cache variant identifier so extract-action prompts that
@ -3500,8 +3497,10 @@ class ForgeAgent:
variant_parts.append("vc")
if show_close_page_action:
variant_parts.append("cp")
if non_vision_enabled:
variant_parts.append("nv")
if enriched_tree_enabled:
variant_parts.append("et")
if enriched_tree_enabled and not llm_screenshots_enabled:
variant_parts.append("ni")
if complete_criterion:
normalized = " ".join(complete_criterion.split())
digest = hashlib.sha256(normalized.encode("utf-8"), usedforsecurity=False).hexdigest()[:6]
@ -3773,10 +3772,8 @@ class ForgeAgent:
open_tabs_context = await _build_open_tabs_context(browser_state, page)
show_close_page_action = open_tabs_context is not None
non_vision_page_context = await build_non_vision_page_context_if_needed(
scraped_page=scraped_page,
page=page,
)
llm_screenshots_enabled = context.llm_screenshots_enabled_for_prompt(retry_index=step.retry_index)
enriched_tree_enabled = context.enriched_tree_enabled()
# Format-then-clear so a render failure can't drop the signal permanently;
# gate on extract-action template since other task types don't render it.
@ -3804,13 +3801,15 @@ class ForgeAgent:
"show_close_page_action": show_close_page_action,
"open_tabs_context": open_tabs_context,
"recent_dialog_messages_str": recent_dialog_messages_str,
"non_vision_page_context": non_vision_page_context,
"llm_screenshots_enabled": llm_screenshots_enabled,
"enriched_tree_enabled": enriched_tree_enabled,
}
cache_variant = self._build_extract_action_cache_variant(
verification_code_check=verification_code_check,
show_close_page_action=show_close_page_action,
complete_criterion=task.complete_criterion.strip() if task.complete_criterion else None,
non_vision_enabled=bool(non_vision_page_context),
enriched_tree_enabled=enriched_tree_enabled,
llm_screenshots_enabled=llm_screenshots_enabled,
)
static_prompt = prompt_engine.load_prompt(f"{template}-static", **prompt_kwargs)
dynamic_prompt = prompt_engine.load_prompt(
@ -3898,12 +3897,8 @@ class ForgeAgent:
show_close_page_action=show_close_page_action,
open_tabs_context=open_tabs_context,
recent_dialog_messages_str=recent_dialog_messages_str,
non_vision_page_context=non_vision_page_context,
# SKY-9718 Layer 1: planner non-cached fallback. Keep Skyvern IDs
# (default html_need_skyvern_attrs=True) — the planner emits
# `click(id=...)` references. Gate lean on the PostHog flag.
# `lean_compress_long_href` intentionally stays OFF — the planner
# reads the href to decide whether to click a link.
llm_screenshots_enabled=llm_screenshots_enabled,
enriched_tree_enabled=enriched_tree_enabled,
lean_compress_long_href=False,
lean_compress_image_src=context.enable_lean_element_tree,
lean_strip_url_query_strings=context.enable_lean_element_tree,
@ -5297,7 +5292,6 @@ class ForgeAgent:
steps=steps_results,
error_code_mapping_str=(json.dumps(task.error_code_mapping) if task.error_code_mapping else None),
local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),
non_vision_page_context=await build_non_vision_page_context_if_needed(page=page),
)
json_response = await app.LLM_API_HANDLER(
prompt=prompt,
@ -5443,7 +5437,6 @@ class ForgeAgent:
max_retries=max_retries,
error_code_mapping_str=(json.dumps(task.error_code_mapping) if task.error_code_mapping else None),
local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),
non_vision_page_context=await build_non_vision_page_context_if_needed(page=page),
)
json_response = await app.SECONDARY_LLM_API_HANDLER(
prompt=prompt,

6
skyvern/forge/prompts/skyvern/extract-action-dynamic.j2
View file

@ -40,12 +40,6 @@ Open browser tabs (CLOSE_PAGE always closes the tab marked [current] — other t
{{ open_tabs_context }}
```
{% endif %}
{% if non_vision_page_context %}
Non-vision page context and accessibility tree:
```json
{{ non_vision_page_context }}
```
{% endif %}
Clickable elements from `{{ current_url }}`:
```
{{ elements }}

10
skyvern/forge/prompts/skyvern/extract-action-static.j2
View file

@ -1,12 +1,12 @@
Identify actions to help user progress towards the user goal using the DOM elements given in the list{% if non_vision_page_context %}, the non-vision page context, and the accessibility tree of the website{% else %} and the screenshot of the website{% endif %}.
Identify actions to help user progress towards the user goal using the DOM elements given in the list{% if llm_screenshots_enabled %} and the screenshot of the website{% endif %}.
Include only the elements that are relevant to the user goal, without altering or imagining new elements.
Accurately interpret and understand the functional significance of SVG elements based on their shapes and context within the webpage.
Use the user details to fill in necessary values. Always satisfy required fields if the field isn't already filled in. Don't return any action for the same field, if this field is already filled in and the value is the same as the one you would have filled in.
MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
Each interactable element is tagged with an ID. Avoid taking action on a disabled element when there is an alternative action available.
{% if non_vision_page_context %}If the non-vision page context shows invalid fields, validation messages, error-colored text, dialogs, overlays, or expanded controls, prioritize actions that address those states.
{% else %}If you see any information in red in the page screenshot, this means a condition wasn't satisfied. prioritize actions with the red information.
{% if llm_screenshots_enabled %}If you see any information in red in the page screenshot, this means a condition wasn't satisfied. prioritize actions with the red information.
If you see a popup in the page screenshot, prioritize actions on the popup.
{% elif enriched_tree_enabled %}If element attributes show invalid fields, validation messages, or error text, prioritize actions that address those states.
{% endif %}
Reply in JSON format with the following keys:
@ -56,5 +56,5 @@ Reply in JSON format with the following keys:
"should_verify_by_magic_link": bool // Whether the page instructs the user to check their email for a magic link to verify the login.{% endif %}
}
Consider the action history from the last step and the {% if non_vision_page_context %}non-vision page context{% else %}screenshot{% endif %} together, if actions from the last step don't yield positive impact, try other actions or other action combinations.
Action history from previous steps: (note: even if the action history suggests goal is achieved, check the {% if non_vision_page_context %}non-vision page context{% else %}screenshot{% endif %} and the DOM elements to make sure the goal is achieved)
Consider the action history from the last step{% if llm_screenshots_enabled %} and the screenshot{% endif %} together, if actions from the last step don't yield positive impact, try other actions or other action combinations.
Action history from previous steps: (note: even if the action history suggests goal is achieved, check{% if llm_screenshots_enabled %} the screenshot and{% endif %} the DOM elements to make sure the goal is achieved)

16
skyvern/forge/prompts/skyvern/extract-action.j2
View file

@ -1,12 +1,12 @@
Identify actions to help user progress towards the user goal using the DOM elements given in the list{% if non_vision_page_context %}, the non-vision page context, and the accessibility tree of the website{% else %} and the screenshot of the website{% endif %}.
Identify actions to help user progress towards the user goal using the DOM elements given in the list{% if llm_screenshots_enabled %} and the screenshot of the website{% endif %}.
Include only the elements that are relevant to the user goal, without altering or imagining new elements.
Accurately interpret and understand the functional significance of SVG elements based on their shapes and context within the webpage.
Use the user details to fill in necessary values. Always satisfy required fields if the field isn't already filled in. Don't return any action for the same field, if this field is already filled in and the value is the same as the one you would have filled in.
MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.
Each interactable element is tagged with an ID. Avoid taking action on a disabled element when there is an alternative action available.
{% if non_vision_page_context %}If the non-vision page context shows invalid fields, validation messages, error-colored text, dialogs, overlays, or expanded controls, prioritize actions that address those states.
{% else %}If you see any information in red in the page screenshot, this means a condition wasn't satisfied. prioritize actions with the red information.
{% if llm_screenshots_enabled %}If you see any information in red in the page screenshot, this means a condition wasn't satisfied. prioritize actions with the red information.
If you see a popup in the page screenshot, prioritize actions on the popup.
{% elif enriched_tree_enabled %}If element attributes show invalid fields, validation messages, or error text, prioritize actions that address those states.
{% endif %}
Reply in JSON format with the following keys:
@ -56,8 +56,8 @@ Reply in JSON format with the following keys:
"should_verify_by_magic_link": bool // Whether the page instructs the user to check their email for a magic link to verify the login.{% endif %}
}
Consider the action history from the last step and the {% if non_vision_page_context %}non-vision page context{% else %}screenshot{% endif %} together, if actions from the last step don't yield positive impact, try other actions or other action combinations.
Action history from previous steps: (note: even if the action history suggests goal is achieved, check the {% if non_vision_page_context %}non-vision page context{% else %}screenshot{% endif %} and the DOM elements to make sure the goal is achieved)
Consider the action history from the last step{% if llm_screenshots_enabled %} and the screenshot{% endif %} together, if actions from the last step don't yield positive impact, try other actions or other action combinations.
Action history from previous steps: (note: even if the action history suggests goal is achieved, check{% if llm_screenshots_enabled %} the screenshot and{% endif %} the DOM elements to make sure the goal is achieved)
```
{{ action_history }}
```
@ -100,12 +100,6 @@ Open browser tabs (CLOSE_PAGE always closes the tab marked [current] — other t
{{ open_tabs_context }}
```
{% endif %}
{% if non_vision_page_context %}
Non-vision page context and accessibility tree:
```json
{{ non_vision_page_context }}
```
{% endif %}
Clickable elements from `{{ current_url }}`:
```
{{ elements }}

25
skyvern/forge/sdk/api/llm/api_handler_factory.py
View file

@ -43,7 +43,7 @@ from skyvern.forge.sdk.api.llm.utils import (
from skyvern.forge.sdk.artifact.manager import BulkArtifactCreationRequest
from skyvern.forge.sdk.artifact.models import ArtifactType
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import LLMVisionMode, SkyvernContext
from skyvern.forge.sdk.core.skyvern_context import EnrichTreeMode, SkyvernContext
from skyvern.forge.sdk.db.enums import WorkflowRunTriggerType
from skyvern.forge.sdk.models import SpeculativeLLMMetadata, Step
from skyvern.forge.sdk.schemas.ai_suggestions import AISuggestion
@ -202,20 +202,19 @@ def _llm_screenshots_enabled_metric(
return True
def _llm_vision_log_fields(context: SkyvernContext | None, step: Step | None = None) -> dict[str, Any]:
def _enrich_tree_log_fields(context: SkyvernContext | None, step: Step | None = None) -> dict[str, Any]:
if context is None:
return {
"llm_vision_mode": LLMVisionMode.CONTROL.value,
"llm_vision_fallback_active": False,
"llm_accessibility_context_enabled": False,
"enrich_tree_mode": EnrichTreeMode.CONTROL.value,
"enrich_tree_fallback_active": False,
"enriched_tree_enabled": False,
}
retry_index = step.retry_index if step else None
return {
"llm_vision_mode": context.effective_llm_vision_mode().value,
"llm_vision_fallback_active": context.llm_vision_fallback_active(
retry_index=step.retry_index if step else None
),
"llm_accessibility_context_enabled": context.llm_accessibility_context_enabled(),
"enrich_tree_mode": context.enrich_tree_mode.value,
"enrich_tree_fallback_active": context.enrich_tree_fallback_active(retry_index=retry_index),
"enriched_tree_enabled": context.enriched_tree_enabled(),
}
@ -1453,7 +1452,7 @@ class LLMAPIHandlerFactory:
llm_cost=llm_cost if llm_cost > 0 else None,
service_tier=getattr(response, "service_tier", None),
llm_screenshots_enabled=llm_screenshots_enabled,
**_llm_vision_log_fields(context, step),
**_enrich_tree_log_fields(context, step),
**_consume_prompt_breakdown(context),
)
@ -1995,7 +1994,7 @@ class LLMAPIHandlerFactory:
llm_cost=llm_cost if llm_cost > 0 else None,
service_tier=getattr(response, "service_tier", None),
llm_screenshots_enabled=llm_screenshots_enabled,
**_llm_vision_log_fields(context, step),
**_enrich_tree_log_fields(context, step),
**_consume_prompt_breakdown(context),
)
@ -2471,7 +2470,7 @@ class LLMCaller:
cached_tokens=call_stats.cached_tokens if call_stats and call_stats.cached_tokens is not None else None,
llm_cost=call_stats.llm_cost if call_stats and call_stats.llm_cost is not None else None,
llm_screenshots_enabled=llm_screenshots_enabled,
**_llm_vision_log_fields(context, step),
**_enrich_tree_log_fields(context, step),
**_consume_prompt_breakdown(context),
)

4
skyvern/forge/sdk/cache/extraction_cache.py vendored
View file

@ -44,8 +44,6 @@ Key derivation (shared with the cross-run tier):
The prompt is sent to the LLM as the `system` message; changing it
changes the output even if all user-prompt inputs are identical, so two
calls that differ only in workflow_system_prompt must not collide.
- non_vision_page_context — rendered only when screenshots are disabled,
so screenshot-backed and non-vision-backed prompts do not collide.
- Date is intentionally NOT in the key. Two calls on byte-identical page
content are semantically the same extraction regardless of wall-clock
date; relying on the content hash keeps hit rate up for scheduled
@ -508,7 +506,6 @@ def compute_cache_key(
previous_extracted_information: Any = None,
llm_key: str | None = None,
workflow_system_prompt: str | None = None,
non_vision_page_context: str | None = None,
) -> str:
"""Return a stable sha256 hex digest for the inputs that affect extraction output.
@ -548,7 +545,6 @@ def compute_cache_key(
_normalize(previous_extracted_information),
_s(llm_key),
_s(workflow_system_prompt),
_s(non_vision_page_context),
]
joined = "\x1f".join(parts).encode("utf-8", errors="replace")
return hashlib.sha256(joined).hexdigest()

56
skyvern/forge/sdk/core/skyvern_context.py
View file

@ -36,22 +36,22 @@ class DialogEntry(TypedDict):
count: int
class LLMVisionMode(StrEnum):
class EnrichTreeMode(StrEnum):
CONTROL = "control"
NO_IMAGES_WITH_A11Y = "no_images_with_a11y"
FALLBACK_WITH_A11Y = "fallback_with_a11y"
FALLBACK_WITHOUT_A11Y = "fallback_without_a11y"
ENRICHED_TREE = "enriched_tree"
ENRICHED_TREE_NO_IMAGES = "enriched_tree_no_images"
ENRICHED_TREE_NO_IMAGES_FALLBACK = "enriched_tree_no_images_fallback"
def parse_llm_vision_mode(value: Any) -> LLMVisionMode:
if isinstance(value, LLMVisionMode):
def parse_enrich_tree_mode(value: Any) -> EnrichTreeMode:
if isinstance(value, EnrichTreeMode):
return value
if isinstance(value, str):
try:
return LLMVisionMode(value)
return EnrichTreeMode(value)
except ValueError:
LOG.warning("Unknown LLM vision mode value, defaulting to control", llm_vision_mode=value)
return LLMVisionMode.CONTROL
LOG.warning("Unknown enrich_tree mode value, defaulting to control", enrich_tree_mode=value)
return EnrichTreeMode.CONTROL
@dataclass
@ -115,8 +115,7 @@ class SkyvernContext:
# PostHog flag ENABLE_LEAN_ELEMENT_TREE, evaluated once per run at scrape time
# and read sync from prompt-build sites.
enable_lean_element_tree: bool = False
disable_llm_screenshots: bool = False
llm_vision_mode: LLMVisionMode = LLMVisionMode.CONTROL
enrich_tree_mode: EnrichTreeMode = EnrichTreeMode.CONTROL
step_retry_index: int = 0
# Trigger type of the enclosing workflow run (manual/api/scheduled/webhook).
@ -198,25 +197,15 @@ class SkyvernContext:
# can be completed when a subsequent click triggers the actual file chooser.
pending_file_chooser: PendingFileChooserListener | None = None
def set_llm_vision_mode(self, mode: Any) -> None:
self.llm_vision_mode = parse_llm_vision_mode(mode)
self.disable_llm_screenshots = self.llm_vision_mode == LLMVisionMode.NO_IMAGES_WITH_A11Y
def set_enrich_tree_mode(self, mode: Any) -> None:
self.enrich_tree_mode = parse_enrich_tree_mode(mode)
def effective_llm_vision_mode(self) -> LLMVisionMode:
if self.disable_llm_screenshots:
return LLMVisionMode.NO_IMAGES_WITH_A11Y
return parse_llm_vision_mode(self.llm_vision_mode)
def enriched_tree_enabled(self) -> bool:
return self.enrich_tree_mode != EnrichTreeMode.CONTROL
def llm_vision_fallback_active(self, *, retry_index: int | None = None) -> bool:
def enrich_tree_fallback_active(self, *, retry_index: int | None = None) -> bool:
effective_retry_index = self.step_retry_index if retry_index is None else retry_index
return (
self.effective_llm_vision_mode()
in {
LLMVisionMode.FALLBACK_WITH_A11Y,
LLMVisionMode.FALLBACK_WITHOUT_A11Y,
}
and effective_retry_index > 0
)
return self.enrich_tree_mode == EnrichTreeMode.ENRICHED_TREE_NO_IMAGES_FALLBACK and effective_retry_index > 0
def llm_screenshots_enabled_for_prompt(
self,
@ -227,22 +216,15 @@ class SkyvernContext:
if is_vision_fallback_prompt:
return True
mode = self.effective_llm_vision_mode()
if mode == LLMVisionMode.CONTROL:
mode = self.enrich_tree_mode
if mode in {EnrichTreeMode.CONTROL, EnrichTreeMode.ENRICHED_TREE}:
return True
if mode == LLMVisionMode.NO_IMAGES_WITH_A11Y:
if mode == EnrichTreeMode.ENRICHED_TREE_NO_IMAGES:
return False
effective_retry_index = self.step_retry_index if retry_index is None else retry_index
# Fallback variants attach screenshots only on retries of the step being sent to the LLM.
return effective_retry_index > 0
def llm_accessibility_context_enabled(self) -> bool:
return self.effective_llm_vision_mode() in {
LLMVisionMode.NO_IMAGES_WITH_A11Y,
LLMVisionMode.FALLBACK_WITH_A11Y,
}
def cleanup_pending_file_chooser(self) -> None:
if self.pending_file_chooser is not None:
if not self.pending_file_chooser.triggered:

20
skyvern/forge/sdk/experimentation/llm_vision_mode.py → skyvern/forge/sdk/experimentation/enrich_tree.py
View file

@ -4,14 +4,14 @@ from typing import Any
import structlog
from skyvern.forge import app
from skyvern.forge.sdk.core.skyvern_context import LLMVisionMode, SkyvernContext
from skyvern.forge.sdk.core.skyvern_context import EnrichTreeMode, SkyvernContext
LOG = structlog.get_logger()
LLM_VISION_MODE_FLAG = "llm_vision_mode"
ENRICH_TREE_FLAG = "enrich_tree"
async def resolve_llm_vision_mode_for_context(
async def resolve_enrich_tree_for_context(
context: SkyvernContext,
distinct_id: str,
organization_id: str | None,
@ -21,7 +21,11 @@ async def resolve_llm_vision_mode_for_context(
log_context: dict[str, Any] | None = None,
) -> None:
if os.getenv("FORCE_DISABLE_LLM_SCREENSHOTS", "").lower() in ("true", "1", "yes"):
context.set_llm_vision_mode(LLMVisionMode.NO_IMAGES_WITH_A11Y)
LOG.info(
"FORCE_DISABLE_LLM_SCREENSHOTS is set; using enriched_tree_no_images mode",
distinct_id=distinct_id,
)
context.set_enrich_tree_mode(EnrichTreeMode.ENRICHED_TREE_NO_IMAGES)
return
properties: dict[str, str] = {}
@ -34,16 +38,16 @@ async def resolve_llm_vision_mode_for_context(
try:
flag_value = await app.EXPERIMENTATION_PROVIDER.get_value_cached(
LLM_VISION_MODE_FLAG,
ENRICH_TREE_FLAG,
distinct_id,
properties=properties,
)
context.set_llm_vision_mode(flag_value)
context.set_enrich_tree_mode(flag_value)
except Exception:
LOG.warning(
"Failed to check llm_vision_mode feature flag",
"Failed to check enrich_tree feature flag",
exc_info=True,
distinct_id=distinct_id,
**(log_context or {}),
)
context.set_llm_vision_mode(LLMVisionMode.CONTROL)
context.set_enrich_tree_mode(EnrichTreeMode.CONTROL)

13
skyvern/forge/sdk/prompting.py
View file

@ -31,15 +31,6 @@ from skyvern.constants import SKYVERN_DIR
LOG = structlog.get_logger()
NON_VISION_CONTEXT_HEADER = "Non-vision page context and accessibility tree:"
def _with_non_vision_context(prompt: str, template: str, kwargs: dict[str, Any]) -> str:
non_vision_page_context = kwargs.get("non_vision_page_context")
if not non_vision_page_context or template.endswith("-static") or NON_VISION_CONTEXT_HEADER in prompt:
return prompt
return f"{prompt.rstrip()}\n\n{NON_VISION_CONTEXT_HEADER}\n```json\n{non_vision_page_context}\n```"
class PromptEngine:
"""
@ -109,7 +100,7 @@ class PromptEngine:
try:
template = "/".join([self.model, template])
jinja_template = self.env.get_template(f"{template}.j2")
return _with_non_vision_context(jinja_template.render(**kwargs), template, kwargs)
return jinja_template.render(**kwargs)
except Exception:
LOG.error(
"Failed to load prompt.",
@ -132,7 +123,7 @@ class PromptEngine:
"""
try:
jinja_template = self.env.from_string(template)
return _with_non_vision_context(jinja_template.render(**kwargs), template, kwargs)
return jinja_template.render(**kwargs)
except Exception:
LOG.error(
"Failed to load prompt from string.",

4
skyvern/forge/sdk/workflow/service.py
View file

@ -61,7 +61,7 @@ from skyvern.forge.sdk.core.security import generate_skyvern_webhook_signature
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.db._sentinels import _UNSET
from skyvern.forge.sdk.db.enums import OrganizationAuthTokenType, WorkflowRunTriggerType
from skyvern.forge.sdk.experimentation.llm_vision_mode import resolve_llm_vision_mode_for_context
from skyvern.forge.sdk.experimentation.enrich_tree import resolve_enrich_tree_for_context
from skyvern.forge.sdk.models import Step
from skyvern.forge.sdk.schemas.files import FileInfo
from skyvern.forge.sdk.schemas.organizations import Organization
@ -1085,7 +1085,7 @@ class WorkflowService:
workflow_run_id=workflow_run.workflow_run_id,
)
await resolve_llm_vision_mode_for_context(
await resolve_enrich_tree_for_context(
new_context,
workflow_run.workflow_run_id,
organization.organization_id,

2
skyvern/services/script_service.py
View file

@ -92,7 +92,6 @@ from skyvern.schemas.workflows import BlockResult, BlockStatus, BlockType, FileS
from skyvern.utils.css_selector import build_action_summaries_with_timing
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import Action, DecisiveAction
from skyvern.webeye.scraper.non_vision_context import build_non_vision_page_context_if_needed
from skyvern.webeye.scraper.scraped_page import ElementTreeFormat
LOG = structlog.get_logger()
@ -1098,7 +1097,6 @@ async def _detect_user_defined_errors(
action_history=[],
local_datetime=datetime.now(tz_info).isoformat(),
reasoning=None,
non_vision_page_context=await build_non_vision_page_context_if_needed(scraped_page=scraped_page),
)
# Call LLM to detect errors

20
skyvern/services/task_v2_service.py
View file

@ -69,7 +69,6 @@ from skyvern.services.webhook_delivery import deliver_webhook_with_retries
from skyvern.utils.prompt_engine import load_prompt_with_elements
from skyvern.utils.strings import generate_random_string
from skyvern.webeye.browser_state import BrowserState
from skyvern.webeye.scraper.non_vision_context import build_non_vision_page_context_if_needed
from skyvern.webeye.scraper.scraped_page import ScrapedPage
from skyvern.webeye.utils.page import SkyvernFrame
@ -140,7 +139,6 @@ async def _summarize_max_steps_failure_reason(
navigation_goal=task_v2.prompt,
history=history,
local_datetime=datetime.now(context.tz_info).isoformat(),
non_vision_page_context=await build_non_vision_page_context_if_needed(page=page),
)
json_response = await app.LLM_API_HANDLER(
@ -823,10 +821,6 @@ async def run_task_v2_helper(
user_goal=user_prompt,
task_history=task_history,
local_datetime=datetime.now(context.tz_info).isoformat(),
non_vision_page_context=await build_non_vision_page_context_if_needed(
scraped_page=scraped_page,
page=page,
),
)
thought = await app.DATABASE.observer.create_thought(
task_v2_id=task_v2_id,
@ -889,10 +883,6 @@ async def run_task_v2_helper(
task_history=task_history,
context=context,
screenshots=scraped_page.screenshots,
non_vision_page_context=await build_non_vision_page_context_if_needed(
scraped_page=scraped_page,
page=page,
),
)
if task_v2.run_with == "code":
await app.WORKFLOW_SERVICE.generate_script_if_needed(
@ -1086,9 +1076,6 @@ async def run_task_v2_helper(
user_goal=user_prompt,
task_history=task_history,
local_datetime=datetime.now(context.tz_info).isoformat(),
non_vision_page_context=await build_non_vision_page_context_if_needed(
scraped_page=completion_scraped_page
),
)
thought = await app.DATABASE.observer.create_thought(
task_v2_id=task_v2_id,
@ -1140,9 +1127,6 @@ async def run_task_v2_helper(
task_history=task_history,
context=context,
screenshots=completion_screenshots,
non_vision_page_context=await build_non_vision_page_context_if_needed(
scraped_page=completion_scraped_page
),
)
if task_v2.run_with == "code":
await app.WORKFLOW_SERVICE.generate_script_if_needed(
@ -1479,7 +1463,6 @@ async def _generate_loop_task(
local_datetime=datetime.now(context.tz_info).isoformat(),
is_link=is_loop_value_link,
loop_values=loop_values,
non_vision_page_context=await build_non_vision_page_context_if_needed(scraped_page=scraped_page),
)
thought_task_in_loop = await app.DATABASE.observer.create_thought(
task_v2_id=task_v2.observer_cruise_id,
@ -1590,7 +1573,6 @@ async def _generate_extraction_task(
current_url=current_url,
data_extraction_goal=data_extraction_goal,
local_datetime=datetime.now(context.tz_info).isoformat(),
non_vision_page_context=await build_non_vision_page_context_if_needed(scraped_page=scraped_page),
)
max_retries = 3
@ -1981,7 +1963,6 @@ async def _summarize_task_v2(
task_history: list[dict],
context: SkyvernContext,
screenshots: list[bytes] | None = None,
non_vision_page_context: str | None = None,
) -> TaskV2:
thought = await app.DATABASE.observer.create_thought(
task_v2_id=task_v2.observer_cruise_id,
@ -1999,7 +1980,6 @@ async def _summarize_task_v2(
task_history=task_history,
extracted_information_schema=task_v2.extracted_information_schema,
local_datetime=datetime.now(context.tz_info).isoformat(),
non_vision_page_context=non_vision_page_context,
)
task_v2_summary_resp = await app.LLM_API_HANDLER(
prompt=task_v2_summary_prompt,

7
skyvern/utils/prompt_engine.py
View file

@ -50,12 +50,11 @@ CEILING_FALLBACK_KEYS_BY_TEMPLATE: dict[str, list[str]] = {
"previous_extracted_information",
"extracted_information_schema",
"extracted_text",
"non_vision_page_context",
],
"extract-action": ["action_history", "navigation_payload_str", "non_vision_page_context"],
"extract-action-dynamic": ["action_history", "navigation_payload_str", "non_vision_page_context"],
"extract-action": ["action_history", "navigation_payload_str"],
"extract-action-dynamic": ["action_history", "navigation_payload_str"],
"extract-action-static": [],
"data-extraction-summary": ["data_extraction_schema", "non_vision_page_context"],
"data-extraction-summary": ["data_extraction_schema"],
}

10
skyvern/webeye/actions/handler.py
View file

@ -120,7 +120,6 @@ from skyvern.webeye.cdp_download_interceptor import (
is_download_response,
normalize_download_filename,
)
from skyvern.webeye.scraper.non_vision_context import build_non_vision_page_context_if_needed
from skyvern.webeye.scraper.scraped_page import (
CleanupElementTreeFunc,
ElementTreeBuilder,
@ -1405,10 +1404,6 @@ async def handle_sequential_click_for_dropdown(
without_screenshots=True,
action_history=action_history_str,
local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),
non_vision_page_context=await build_non_vision_page_context_if_needed(
scraped_page=scraped_page_after_open,
page=page,
),
lean_compress_long_href=lean_enabled,
lean_compress_image_src=lean_enabled,
lean_strip_url_query_strings=lean_enabled,
@ -4170,7 +4165,6 @@ async def sequentially_select_from_dropdown(
elements="".join(json_to_html(element) for element in secondary_increment_element),
select_history=json.dumps(build_sequential_select_history(select_history)),
local_datetime=datetime.now(ensure_context().tz_info).isoformat(),
non_vision_page_context=await build_non_vision_page_context_if_needed(page=page),
)
llm_api_handler = LLMAPIHandlerFactory.get_override_llm_api_handler(task.llm_key, default=app.LLM_API_HANDLER)
json_response = await llm_api_handler(
@ -4757,7 +4751,6 @@ async def locate_dropdown_menu(
# TODO: better to send untrimmed HTML without skyvern attributes in the future
dropdown_confirm_prompt = prompt_engine.load_prompt(
"opened-dropdown-confirm",
non_vision_page_context=await build_non_vision_page_context_if_needed(page=skyvern_frame.get_frame()),
)
LOG.debug(
"Confirm if it's an opened dropdown menu",
@ -5187,7 +5180,6 @@ async def extract_information_for_navigation_goal(
extracted_text=extracted_text_for_prompt,
error_code_mapping_str=error_code_mapping_str,
local_datetime=local_datetime_str,
non_vision_page_context=await build_non_vision_page_context_if_needed(scraped_page=scraped_page_refreshed),
)
# Self-heal guard: on the second retry onward (``retry_index > 1``) the
@ -5235,7 +5227,6 @@ async def extract_information_for_navigation_goal(
previous_extracted_information=post_ceiling_kwargs["previous_extracted_information"],
llm_key=llm_key_override,
workflow_system_prompt=task.workflow_system_prompt,
non_vision_page_context=post_ceiling_kwargs.get("non_vision_page_context"),
)
if is_retry_step:
# Proactively evict the in-run entry. The cross-run tier will be
@ -5685,7 +5676,6 @@ async def extract_user_defined_errors(
error_code_mapping_str=json.dumps(task.error_code_mapping) if task.error_code_mapping else "{}",
local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),
reasoning=reasoning,
non_vision_page_context=await build_non_vision_page_context_if_needed(scraped_page=scraped_page_refreshed),
)
json_response = await app.EXTRACTION_LLM_API_HANDLER(
prompt=prompt,

84
skyvern/webeye/scraper/domUtils.js
View file

@ -1529,6 +1529,84 @@ async function uniqueId() {
return result;
}
const ENRICHED_ELEMENT_TEXT_LIMIT = 120;
function truncateEnrichedText(value) {
const text = (value || "").replace(/\s+/g, " ").trim();
if (!text) {
return "";
}
return text.length > ENRICHED_ELEMENT_TEXT_LIMIT
? text.slice(0, ENRICHED_ELEMENT_TEXT_LIMIT) + "..."
: text;
}
function textFromElementIds(ids) {
if (!ids) {
return "";
}
return ids
.split(/\s+/)
.map((id) => {
const ref = document.getElementById(id);
return ref ? ref.innerText || ref.textContent || "" : "";
})
.join(" ");
}
function resolveErrorText(element) {
const errMsgId = element.getAttribute("aria-errormessage");
if (errMsgId) {
const text = truncateEnrichedText(textFromElementIds(errMsgId));
if (text) {
return text;
}
}
const describedBy = element.getAttribute("aria-describedby");
if (describedBy) {
return truncateEnrichedText(textFromElementIds(describedBy));
}
return "";
}
function enrichValidationState(attrs, element, elementTagNameLower) {
if (window.GlobalEnableEnrichedElementTree !== true) {
return;
}
if (attrs["aria-invalid"] !== undefined) {
const ariaInvalid = attrs["aria-invalid"];
if (
ariaInvalid === false ||
(typeof ariaInvalid === "string" && ariaInvalid.toLowerCase() === "false")
) {
delete attrs["aria-invalid"];
} else if (
ariaInvalid === true ||
(typeof ariaInvalid === "string" &&
["true", "grammar", "spelling"].includes(ariaInvalid.toLowerCase()))
) {
attrs["aria-invalid"] = true;
attrs["invalid"] = true;
}
}
if (element.validity && element.validity.valid === false) {
attrs["invalid"] = true;
if (element.type !== "password") {
const validationMessage = truncateEnrichedText(element.validationMessage);
if (validationMessage) {
attrs["validationMessage"] = validationMessage;
}
}
}
const errorText = resolveErrorText(element);
if (errorText) {
attrs["errorText"] = errorText;
}
}
async function buildElementObject(
frame,
element,
@ -1553,7 +1631,9 @@ async function buildElementObject(
attr.name === "readonly" ||
attr.name === "aria-readonly" ||
attr.name === "disabled" ||
attr.name === "aria-disabled"
attr.name === "aria-disabled" ||
attr.name === "aria-invalid" ||
attr.name === "aria-expanded"
) {
if (attrValue && attrValue.toLowerCase() === "false") {
attrValue = false;
@ -1637,6 +1717,8 @@ async function buildElementObject(
}
}
enrichValidationState(attrs, element, elementTagNameLower);
let elementObj = {
id: element_id,
frame: frame,

270
skyvern/webeye/scraper/non_vision_context.py
View file

@ -1,270 +0,0 @@
from __future__ import annotations
import json
import re
from typing import Any
import structlog
from playwright.async_api import Frame, Page
from skyvern.constants import SKYVERN_ID_ATTR
from skyvern.forge.sdk.core import skyvern_context
LOG = structlog.get_logger()
MAX_NON_VISION_CONTEXT_CHARS = 20_000
MAX_VISIBLE_TEXT_CHARS = 6_000
MAX_ACCESSIBILITY_NODES = 160
_SENSITIVE_VALUE_RE = re.compile(
r"(password|passcode|one[-_\s]?time[-_\s]?code|otp|token|secret|api[-_\s]?key|access[-_\s]?key|"
r"private[-_\s]?key|cvv|cvc|security[-_\s]?code|ssn|social[-_\s]?security|card[-_\s]?number|"
r"credit[-_\s]?card)",
re.IGNORECASE,
)
_ACCESSIBILITY_CONTEXT_SCRIPT = """
({ skyvernIdAttr, maxNodes, maxVisibleTextChars }) => {
const normalize = (value) => (value || "").replace(/\\s+/g, " ").trim();
const truncate = (value, limit) => {
const text = normalize(value);
return text.length > limit ? text.slice(0, limit) + "..." : text;
};
const isVisible = (el) => {
const style = window.getComputedStyle(el);
if (!style || style.visibility === "hidden" || style.display === "none" || Number(style.opacity) === 0) {
return false;
}
const rect = el.getBoundingClientRect();
return rect.width > 0 && rect.height > 0 && rect.bottom >= 0 && rect.right >= 0
&& rect.top <= window.innerHeight && rect.left <= window.innerWidth;
};
const textByIds = (ids) => normalize(ids.split(/\\s+/).map((id) => {
const ref = document.getElementById(id);
return ref ? (ref.innerText || ref.textContent || "") : "";
}).join(" "));
const labelText = (el) => {
const aria = el.getAttribute("aria-label");
if (aria) return aria;
const labelledBy = el.getAttribute("aria-labelledby");
if (labelledBy) {
const text = textByIds(labelledBy);
if (text) return text;
}
const id = el.getAttribute("id");
if (id) {
const escapedId = window.CSS && CSS.escape ? CSS.escape(id) : id.replace(/["\\\\]/g, "\\\\$&");
const label = document.querySelector(`label[for="${escapedId}"]`);
if (label) return label.innerText || label.textContent || "";
}
const parentLabel = el.closest("label");
if (parentLabel) return parentLabel.innerText || parentLabel.textContent || "";
return el.getAttribute("title") || el.getAttribute("alt") || el.getAttribute("placeholder") || "";
};
const implicitRole = (el) => {
const tag = el.tagName.toLowerCase();
const type = (el.getAttribute("type") || "").toLowerCase();
if (tag === "a" && el.getAttribute("href")) return "link";
if (tag === "button") return "button";
if (tag === "select") return "combobox";
if (tag === "textarea") return "textbox";
if (tag === "input") {
if (type === "checkbox") return "checkbox";
if (type === "radio") return "radio";
if (type === "submit" || type === "button") return "button";
return "textbox";
}
if (/^h[1-6]$/.test(tag)) return "heading";
if (tag === "img") return "img";
return "";
};
const rgb = (value) => {
const match = (value || "").match(/rgba?\\((\\d+),\\s*(\\d+),\\s*(\\d+)/);
if (!match) return null;
return { r: Number(match[1]), g: Number(match[2]), b: Number(match[3]) };
};
const isErrorColor = (value) => {
const color = rgb(value);
return color && color.r > 140 && color.r > color.g * 1.4 && color.r > color.b * 1.4;
};
const selector = [
"a[href]", "button", "input", "select", "textarea", "summary", "option",
"[role]", "[aria-label]", "[aria-labelledby]", "[aria-describedby]",
"[aria-invalid]", "[aria-expanded]", "[aria-selected]", "[aria-checked]",
"[tabindex]:not([tabindex='-1'])", "[data-testid]"
].join(",");
const nodes = [];
for (const el of Array.from(document.querySelectorAll(selector))) {
if (nodes.length >= maxNodes) break;
if (!isVisible(el)) continue;
const style = window.getComputedStyle(el);
const rect = el.getBoundingClientRect();
const tag = el.tagName.toLowerCase();
const role = el.getAttribute("role") || implicitRole(el);
const text = truncate(el.innerText || el.textContent || "", 220);
const name = truncate(labelText(el), 220);
const node = {
skyvern_id: el.getAttribute(skyvernIdAttr) || null,
tag,
role: role || null,
name: name || null,
text: text && text !== name ? text : null,
type: el.getAttribute("type") || null,
value: tag === "input" || tag === "textarea" || tag === "select" ? truncate(el.value, 120) : null,
autocomplete: el.getAttribute("autocomplete") || null,
html_id: el.getAttribute("id") || null,
html_name: el.getAttribute("name") || null,
placeholder: el.getAttribute("placeholder") || null,
disabled: el.disabled === true || el.getAttribute("aria-disabled") === "true" || null,
required: el.required === true || el.getAttribute("aria-required") === "true" || null,
checked: el.checked === true || el.getAttribute("aria-checked") || null,
selected: el.selected === true || el.getAttribute("aria-selected") || null,
expanded: el.getAttribute("aria-expanded") || null,
invalid: el.getAttribute("aria-invalid") || (el.validity && !el.validity.valid) || null,
validation_message: el.validationMessage || null,
error_color: isErrorColor(style.color) ? style.color : null,
rect: {
x: Math.round(rect.x),
y: Math.round(rect.y),
width: Math.round(rect.width),
height: Math.round(rect.height)
}
};
for (const key of Object.keys(node)) {
if (node[key] === null) delete node[key];
}
if (node.name || node.text || node.role || node.skyvern_id || node.placeholder || node.value) {
nodes.push(node);
}
}
return {
title: document.title || "",
url: window.location.href,
viewport: { width: window.innerWidth, height: window.innerHeight },
visible_text: truncate(document.body ? document.body.innerText : "", maxVisibleTextChars),
accessibility_tree: nodes
};
}
"""
def _truncate(text: str | None, max_chars: int) -> str | None:
if not text:
return None
normalized = " ".join(text.split())
if len(normalized) <= max_chars:
return normalized
return normalized[:max_chars] + "..."
def _redact_sensitive_values(page_context: dict[str, Any]) -> bool:
nodes = page_context.get("accessibility_tree")
if not isinstance(nodes, list):
return False
redacted_sensitive_value = False
for node in nodes:
if not isinstance(node, dict) or "value" not in node:
continue
sensitive_fields = [
node.get("type"),
node.get("name"),
node.get("placeholder"),
node.get("autocomplete"),
node.get("html_id"),
node.get("html_name"),
]
haystack = " ".join(str(value) for value in sensitive_fields if value)
if _SENSITIVE_VALUE_RE.search(haystack):
node.pop("value", None)
node["value_redacted"] = True
redacted_sensitive_value = True
return redacted_sensitive_value
def _json_dumps(page_context: dict[str, Any]) -> str:
return json.dumps(page_context, separators=(",", ":"), default=str)
def _bounded_json_context(page_context: dict[str, Any], max_chars: int) -> str:
rendered = _json_dumps(page_context)
if len(rendered) <= max_chars:
return rendered
bounded = dict(page_context)
bounded["truncated"] = True
if isinstance(bounded.get("visible_text"), str):
bounded["visible_text"] = _truncate(bounded["visible_text"], min(MAX_VISIBLE_TEXT_CHARS, max_chars // 4))
nodes = bounded.get("accessibility_tree")
if isinstance(nodes, list):
bounded["accessibility_tree"] = list(nodes)
while bounded["accessibility_tree"] and len(_json_dumps(bounded)) > max_chars:
keep_count = max(0, len(bounded["accessibility_tree"]) // 2)
bounded["accessibility_tree"] = bounded["accessibility_tree"][:keep_count]
rendered = _json_dumps(bounded)
if len(rendered) <= max_chars:
return rendered
fallback: dict[str, Any] = {"truncated": True}
for key in ("url", "title"):
value = bounded.get(key)
if value:
fallback[key] = _truncate(str(value), max_chars // 4)
rendered = _json_dumps(fallback)
return rendered if len(rendered) <= max_chars else _json_dumps({"truncated": True})
async def _safe_page_context(page: Frame | Page | None) -> tuple[dict[str, Any], bool]:
if page is None:
return {}, False
try:
result = await page.evaluate(
_ACCESSIBILITY_CONTEXT_SCRIPT,
{
"skyvernIdAttr": SKYVERN_ID_ATTR,
"maxNodes": MAX_ACCESSIBILITY_NODES,
"maxVisibleTextChars": MAX_VISIBLE_TEXT_CHARS,
},
)
if not isinstance(result, dict):
return {}, False
redacted_sensitive_values = _redact_sensitive_values(result)
if redacted_sensitive_values:
result.pop("visible_text", None)
return result, redacted_sensitive_values
except Exception:
LOG.warning("Failed to build non-vision page context", exc_info=True)
return {}, False
async def build_non_vision_page_context(
*,
scraped_page: Any | None = None,
page: Frame | Page | None = None,
max_chars: int = MAX_NON_VISION_CONTEXT_CHARS,
) -> str | None:
page_context, redacted_sensitive_values = await _safe_page_context(page)
if scraped_page is not None:
page_context.setdefault("url", getattr(scraped_page, "url", None))
extracted_text = _truncate(getattr(scraped_page, "extracted_text", None), MAX_VISIBLE_TEXT_CHARS)
if extracted_text and not page_context.get("visible_text") and not redacted_sensitive_values:
page_context["visible_text"] = extracted_text
if not page_context:
return None
return _bounded_json_context(page_context, max_chars)
async def build_non_vision_page_context_if_needed(
*,
scraped_page: Any | None = None,
page: Frame | Page | None = None,
max_chars: int = MAX_NON_VISION_CONTEXT_CHARS,
) -> str | None:
context = skyvern_context.current()
if not context or not context.llm_accessibility_context_enabled():
return None
return await build_non_vision_page_context(scraped_page=scraped_page, page=page, max_chars=max_chars)

25
skyvern/webeye/scraper/scraper.py
View file

@ -96,6 +96,28 @@ RESERVED_ATTRIBUTES = {
"value",
}
ENRICHED_ONLY_ATTRIBUTES = {
"aria-describedby",
"aria-errormessage",
"aria-expanded",
"aria-haspopup",
"aria-invalid",
"aria-labelledby",
"errorText",
"invalid",
"validationMessage",
}
ENRICHED_RESERVED_ATTRIBUTES = RESERVED_ATTRIBUTES | ENRICHED_ONLY_ATTRIBUTES
def _reserved_attributes_for_context() -> set[str]:
context = skyvern_context.current()
if context and context.enriched_tree_enabled():
return ENRICHED_RESERVED_ATTRIBUTES
return RESERVED_ATTRIBUTES
BASE64_INCLUDE_ATTRIBUTES = {
"href",
"src",
@ -863,11 +885,12 @@ def _trimmed_base64_data(attributes: dict) -> dict:
def _trimmed_attributes(attributes: dict, *, keep_class: bool = False) -> dict:
new_attributes: dict = {}
reserved_attributes = _reserved_attributes_for_context()
for key in attributes:
if key == "role" and attributes[key] in ["listbox", "option"]:
new_attributes[key] = attributes[key]
if key in RESERVED_ATTRIBUTES:
if key in reserved_attributes:
new_attributes[key] = attributes[key]
if keep_class and "class" in attributes:

13
skyvern/webeye/utils/page.py
View file

@ -15,6 +15,7 @@ from playwright.async_api import ElementHandle, Frame, Page
from skyvern.constants import PAGE_CONTENT_TIMEOUT, SKYVERN_DIR
from skyvern.exceptions import FailedToTakeScreenshot
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.trace import apply_context_attrs, traced
@ -697,6 +698,15 @@ class SkyvernFrame:
js_script = "() => removeAllUniqueIds()"
await self.evaluate(frame=self.frame, expression=js_script)
async def _set_enriched_element_tree_flag(self) -> None:
context = skyvern_context.current()
enriched_enabled = bool(context and context.enriched_tree_enabled())
await self.evaluate(
frame=self.frame,
expression="([enabled]) => { window.GlobalEnableEnrichedElementTree = enabled; }",
arg=[enriched_enabled],
)
@traced(name="skyvern.browser.element_tree_from_body")
async def build_tree_from_body(
self,
@ -706,6 +716,7 @@ class SkyvernFrame:
timeout_ms: float = SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,
) -> tuple[list[dict], list[dict]]:
must_included_tags = must_included_tags or []
await self._set_enriched_element_tree_flag()
js_script = "async ([frame_name, frame_index, must_included_tags]) => await buildTreeFromBody(frame_name, frame_index, must_included_tags)"
return await self.evaluate(
frame=self.frame,
@ -720,6 +731,7 @@ class SkyvernFrame:
wait_until_finished: bool = True,
timeout_ms: float = SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,
) -> tuple[list[dict], list[dict]]:
await self._set_enriched_element_tree_flag()
js_script = "async ([wait_until_finished]) => await getIncrementElements(wait_until_finished)"
return await self.evaluate(
frame=self.frame, expression=js_script, timeout_ms=timeout_ms, arg=[wait_until_finished]
@ -733,6 +745,7 @@ class SkyvernFrame:
full_tree: bool = False,
timeout_ms: float = SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,
) -> tuple[list[dict], list[dict]]:
await self._set_enriched_element_tree_flag()
js_script = "async ([starter, frame, full_tree]) => await buildElementTree(starter, frame, full_tree)"
return await self.evaluate(
frame=self.frame, expression=js_script, timeout_ms=timeout_ms, arg=[starter, frame, full_tree]

207
tests/unit/test_disable_llm_screenshots_flag.py
View file

@ -1,207 +0,0 @@
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from skyvern.forge.sdk.api.llm.api_handler_factory import (
_llm_screenshots_enabled_metric,
_llm_screenshots_for_call,
_llm_vision_log_fields,
)
from skyvern.forge.sdk.core.skyvern_context import LLMVisionMode, SkyvernContext, parse_llm_vision_mode
from skyvern.forge.sdk.experimentation import llm_vision_mode
from skyvern.schemas.llm import LLMConfig
def _config(*, supports_vision: bool = True) -> MagicMock:
config = MagicMock(spec=LLMConfig)
config.supports_vision = supports_vision
return config
def _step(retry_index: int) -> SimpleNamespace:
return SimpleNamespace(retry_index=retry_index)
@pytest.mark.parametrize(
("mode", "retry_index", "screenshots_enabled", "accessibility_enabled", "fallback_active"),
[
(LLMVisionMode.CONTROL, 0, True, False, False),
(LLMVisionMode.CONTROL, 1, True, False, False),
(LLMVisionMode.NO_IMAGES_WITH_A11Y, 0, False, True, False),
(LLMVisionMode.NO_IMAGES_WITH_A11Y, 1, False, True, False),
(LLMVisionMode.FALLBACK_WITH_A11Y, 0, False, True, False),
(LLMVisionMode.FALLBACK_WITH_A11Y, 1, True, True, True),
(LLMVisionMode.FALLBACK_WITHOUT_A11Y, 0, False, False, False),
(LLMVisionMode.FALLBACK_WITHOUT_A11Y, 1, True, False, True),
],
)
def test_llm_vision_modes_control_screenshots_and_accessibility_context(
mode: LLMVisionMode,
retry_index: int,
screenshots_enabled: bool,
accessibility_enabled: bool,
fallback_active: bool,
) -> None:
ctx = SkyvernContext(llm_vision_mode=mode, step_retry_index=retry_index)
expected_screenshots = [b"png"] if screenshots_enabled else None
assert _llm_screenshots_for_call([b"png"], _config(), ctx) == expected_screenshots
assert _llm_screenshots_enabled_metric(_config(), ctx) is screenshots_enabled
assert ctx.llm_accessibility_context_enabled() is accessibility_enabled
assert ctx.llm_vision_fallback_active() is fallback_active
log_fields = _llm_vision_log_fields(ctx)
assert log_fields["llm_vision_mode"] == mode.value
assert log_fields["llm_vision_fallback_active"] is fallback_active
assert log_fields["llm_accessibility_context_enabled"] is accessibility_enabled
def test_legacy_disable_llm_screenshots_still_maps_to_no_images_with_a11y() -> None:
ctx = SkyvernContext(disable_llm_screenshots=True)
assert ctx.effective_llm_vision_mode() == LLMVisionMode.NO_IMAGES_WITH_A11Y
assert _llm_screenshots_for_call([b"png"], _config(), ctx) is None
assert _llm_screenshots_enabled_metric(_config(), ctx) is False
assert ctx.llm_accessibility_context_enabled() is True
@pytest.mark.parametrize(
"mode",
[
LLMVisionMode.NO_IMAGES_WITH_A11Y,
LLMVisionMode.FALLBACK_WITH_A11Y,
LLMVisionMode.FALLBACK_WITHOUT_A11Y,
],
)
def test_vision_fallback_prompt_keeps_screenshots_in_no_image_modes(mode: LLMVisionMode) -> None:
ctx = SkyvernContext(llm_vision_mode=mode)
assert _llm_screenshots_for_call([b"png"], _config(), ctx, "extract-text-from-image") == [b"png"]
assert _llm_screenshots_enabled_metric(_config(), ctx, "extract-text-from-image") is True
def test_step_argument_controls_fallback_for_speculative_next_step() -> None:
ctx = SkyvernContext(
llm_vision_mode=LLMVisionMode.FALLBACK_WITHOUT_A11Y,
step_retry_index=2,
)
assert _llm_screenshots_for_call([b"png"], _config(), ctx, step=_step(0)) is None
assert _llm_screenshots_enabled_metric(_config(), ctx, step=_step(0)) is False
assert _llm_vision_log_fields(ctx, _step(0))["llm_vision_fallback_active"] is False
assert _llm_screenshots_for_call([b"png"], _config(), ctx, step=_step(1)) == [b"png"]
assert _llm_screenshots_enabled_metric(_config(), ctx, step=_step(1)) is True
assert _llm_vision_log_fields(ctx, _step(1))["llm_vision_fallback_active"] is True
def test_non_vision_model_never_attaches() -> None:
ctx = SkyvernContext(llm_vision_mode=LLMVisionMode.CONTROL)
assert _llm_screenshots_for_call([b"png"], _config(supports_vision=False), ctx) is None
assert _llm_screenshots_enabled_metric(_config(supports_vision=False), ctx) is False
def test_none_context_treated_as_screenshots_enabled_for_vision() -> None:
assert _llm_screenshots_for_call([b"png"], _config(), None) == [b"png"]
assert _llm_screenshots_enabled_metric(_config(), None) is True
assert _llm_vision_log_fields(None) == {
"llm_vision_mode": LLMVisionMode.CONTROL.value,
"llm_vision_fallback_active": False,
"llm_accessibility_context_enabled": False,
}
def test_none_screenshots_input_still_reports_cohort_when_vision() -> None:
ctx = SkyvernContext(llm_vision_mode=LLMVisionMode.CONTROL)
assert _llm_screenshots_for_call(None, _config(), ctx) is None
assert _llm_screenshots_enabled_metric(_config(), ctx) is True
def test_invalid_or_missing_llm_vision_mode_defaults_to_control() -> None:
assert parse_llm_vision_mode(None) == LLMVisionMode.CONTROL
assert parse_llm_vision_mode("unknown-mode") == LLMVisionMode.CONTROL
assert parse_llm_vision_mode("control") == LLMVisionMode.CONTROL
def test_set_llm_vision_mode_keeps_legacy_disable_flag_in_sync() -> None:
ctx = SkyvernContext(disable_llm_screenshots=True)
ctx.set_llm_vision_mode(LLMVisionMode.CONTROL)
assert ctx.effective_llm_vision_mode() == LLMVisionMode.CONTROL
assert ctx.disable_llm_screenshots is False
@pytest.mark.asyncio
async def test_resolve_llm_vision_mode_for_context_uses_run_distinct_id_and_properties(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls: list[dict] = []
class Provider:
async def get_value_cached(
self,
feature_name: str,
distinct_id: str,
*,
properties: dict[str, str],
) -> str:
calls.append(
{
"feature_name": feature_name,
"distinct_id": distinct_id,
"properties": properties,
}
)
return LLMVisionMode.FALLBACK_WITH_A11Y.value
monkeypatch.setattr(llm_vision_mode.app, "EXPERIMENTATION_PROVIDER", Provider())
monkeypatch.delenv("FORCE_DISABLE_LLM_SCREENSHOTS", raising=False)
ctx = SkyvernContext()
await llm_vision_mode.resolve_llm_vision_mode_for_context(
ctx,
"workflow-run-id",
"organization-id",
workflow_permanent_id="workflow-permanent-id",
)
assert ctx.llm_vision_mode == LLMVisionMode.FALLBACK_WITH_A11Y
assert calls == [
{
"feature_name": "llm_vision_mode",
"distinct_id": "workflow-run-id",
"properties": {
"organization_id": "organization-id",
"workflow_permanent_id": "workflow-permanent-id",
},
}
]
@pytest.mark.asyncio
async def test_resolve_llm_vision_mode_for_context_defaults_invalid_values_to_control(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class Provider:
async def get_value_cached(
self,
_feature_name: str,
_distinct_id: str,
*,
properties: dict[str, str],
) -> str:
assert properties == {"organization_id": "organization-id"}
return "not-a-real-mode"
monkeypatch.setattr(llm_vision_mode.app, "EXPERIMENTATION_PROVIDER", Provider())
monkeypatch.delenv("FORCE_DISABLE_LLM_SCREENSHOTS", raising=False)
ctx = SkyvernContext(llm_vision_mode=LLMVisionMode.NO_IMAGES_WITH_A11Y)
await llm_vision_mode.resolve_llm_vision_mode_for_context(ctx, "task-id", "organization-id")
assert ctx.llm_vision_mode == LLMVisionMode.CONTROL
assert ctx.disable_llm_screenshots is False

145
tests/unit/test_enrich_tree_flag.py Normal file
View file

@ -0,0 +1,145 @@
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from skyvern.forge.sdk.api.llm.api_handler_factory import (
_enrich_tree_log_fields,
_llm_screenshots_enabled_metric,
_llm_screenshots_for_call,
)
from skyvern.forge.sdk.core.skyvern_context import EnrichTreeMode, SkyvernContext, parse_enrich_tree_mode
from skyvern.schemas.llm import LLMConfig
def _config(*, supports_vision: bool = True) -> MagicMock:
config = MagicMock(spec=LLMConfig)
config.supports_vision = supports_vision
return config
def _step(retry_index: int) -> SimpleNamespace:
return SimpleNamespace(retry_index=retry_index)
@pytest.mark.parametrize(
("mode", "retry_index", "screenshots_enabled", "enriched_enabled", "fallback_active"),
[
(EnrichTreeMode.CONTROL, 0, True, False, False),
(EnrichTreeMode.CONTROL, 1, True, False, False),
(EnrichTreeMode.ENRICHED_TREE, 0, True, True, False),
(EnrichTreeMode.ENRICHED_TREE, 1, True, True, False),
(EnrichTreeMode.ENRICHED_TREE_NO_IMAGES, 0, False, True, False),
(EnrichTreeMode.ENRICHED_TREE_NO_IMAGES, 1, False, True, False),
(EnrichTreeMode.ENRICHED_TREE_NO_IMAGES_FALLBACK, 0, False, True, False),
(EnrichTreeMode.ENRICHED_TREE_NO_IMAGES_FALLBACK, 1, True, True, True),
],
)
def test_enrich_tree_modes_control_screenshots_and_enrichment(
mode: EnrichTreeMode,
retry_index: int,
screenshots_enabled: bool,
enriched_enabled: bool,
fallback_active: bool,
) -> None:
ctx = SkyvernContext(enrich_tree_mode=mode, step_retry_index=retry_index)
expected_screenshots = [b"png"] if screenshots_enabled else None
assert _llm_screenshots_for_call([b"png"], _config(), ctx) == expected_screenshots
assert _llm_screenshots_enabled_metric(_config(), ctx) is screenshots_enabled
assert ctx.enriched_tree_enabled() is enriched_enabled
assert ctx.enrich_tree_fallback_active() is fallback_active
log_fields = _enrich_tree_log_fields(ctx)
assert log_fields["enrich_tree_mode"] == mode.value
assert log_fields["enrich_tree_fallback_active"] is fallback_active
assert log_fields["enriched_tree_enabled"] is enriched_enabled
@pytest.mark.parametrize(
"mode",
[
EnrichTreeMode.ENRICHED_TREE_NO_IMAGES,
EnrichTreeMode.ENRICHED_TREE_NO_IMAGES_FALLBACK,
],
)
def test_vision_fallback_prompt_keeps_screenshots_in_no_image_modes(mode: EnrichTreeMode) -> None:
ctx = SkyvernContext(enrich_tree_mode=mode)
assert _llm_screenshots_for_call([b"png"], _config(), ctx, "extract-text-from-image") == [b"png"]
assert _llm_screenshots_enabled_metric(_config(), ctx, "extract-text-from-image") is True
def test_step_argument_controls_fallback_for_speculative_next_step() -> None:
ctx = SkyvernContext(
enrich_tree_mode=EnrichTreeMode.ENRICHED_TREE_NO_IMAGES_FALLBACK,
step_retry_index=2,
)
assert _llm_screenshots_for_call([b"png"], _config(), ctx, step=_step(0)) is None
assert _llm_screenshots_enabled_metric(_config(), ctx, step=_step(0)) is False
assert _enrich_tree_log_fields(ctx, _step(0))["enrich_tree_fallback_active"] is False
assert _llm_screenshots_for_call([b"png"], _config(), ctx, step=_step(1)) == [b"png"]
assert _llm_screenshots_enabled_metric(_config(), ctx, step=_step(1)) is True
assert _enrich_tree_log_fields(ctx, _step(1))["enrich_tree_fallback_active"] is True
def test_non_vision_model_never_attaches() -> None:
ctx = SkyvernContext(enrich_tree_mode=EnrichTreeMode.CONTROL)
assert _llm_screenshots_for_call([b"png"], _config(supports_vision=False), ctx) is None
assert _llm_screenshots_enabled_metric(_config(supports_vision=False), ctx) is False
def test_none_context_treated_as_screenshots_enabled_for_vision() -> None:
assert _llm_screenshots_for_call([b"png"], _config(), None) == [b"png"]
assert _llm_screenshots_enabled_metric(_config(), None) is True
assert _enrich_tree_log_fields(None) == {
"enrich_tree_mode": EnrichTreeMode.CONTROL.value,
"enrich_tree_fallback_active": False,
"enriched_tree_enabled": False,
}
def test_none_screenshots_input_still_reports_cohort_when_vision() -> None:
ctx = SkyvernContext(enrich_tree_mode=EnrichTreeMode.CONTROL)
assert _llm_screenshots_for_call(None, _config(), ctx) is None
assert _llm_screenshots_enabled_metric(_config(), ctx) is True
def test_invalid_or_missing_enrich_tree_mode_defaults_to_control() -> None:
assert parse_enrich_tree_mode(None) == EnrichTreeMode.CONTROL
assert parse_enrich_tree_mode("unknown-mode") == EnrichTreeMode.CONTROL
assert parse_enrich_tree_mode("control") == EnrichTreeMode.CONTROL
def test_extract_action_prompt_uses_llm_screenshots_enabled_not_non_vision() -> None:
from skyvern.forge.prompts import prompt_engine as engine_module
with_screenshots = engine_module.load_prompt(
"extract-action",
llm_screenshots_enabled=True,
elements="<input id='a'>",
navigation_goal="Submit form",
navigation_payload_str="{}",
action_history="",
local_datetime="2026-05-27T12:00:00",
)
without_screenshots = engine_module.load_prompt(
"extract-action",
llm_screenshots_enabled=False,
enriched_tree_enabled=True,
elements="<input id='a' validationMessage='Invalid email'>",
navigation_goal="Submit form",
navigation_payload_str="{}",
action_history="",
local_datetime="2026-05-27T12:00:00",
)
assert "non_vision_page_context" not in with_screenshots
assert "non_vision_page_context" not in without_screenshots
assert "screenshot of the website" in with_screenshots
assert "screenshot of the website" not in without_screenshots
assert "validation messages" in without_screenshots

108
tests/unit/test_enrich_tree_resolver.py Normal file
View file

@ -0,0 +1,108 @@
from types import SimpleNamespace
import pytest
from skyvern.forge.sdk.core.skyvern_context import EnrichTreeMode, SkyvernContext
from skyvern.forge.sdk.experimentation import enrich_tree
@pytest.mark.asyncio
async def test_resolve_enrich_tree_for_context_uses_run_distinct_id_and_properties(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls: list[dict] = []
class Provider:
async def get_value_cached(
self,
feature_name: str,
distinct_id: str,
*,
properties: dict[str, str],
) -> str:
calls.append(
{
"feature_name": feature_name,
"distinct_id": distinct_id,
"properties": properties,
}
)
return EnrichTreeMode.ENRICHED_TREE_NO_IMAGES_FALLBACK.value
monkeypatch.setattr(
enrich_tree,
"app",
SimpleNamespace(EXPERIMENTATION_PROVIDER=Provider()),
)
monkeypatch.delenv("FORCE_DISABLE_LLM_SCREENSHOTS", raising=False)
ctx = SkyvernContext()
await enrich_tree.resolve_enrich_tree_for_context(
ctx,
"workflow-run-id",
"organization-id",
workflow_permanent_id="workflow-permanent-id",
task_url="https://example.test",
)
assert ctx.enrich_tree_mode == EnrichTreeMode.ENRICHED_TREE_NO_IMAGES_FALLBACK
assert calls == [
{
"feature_name": "enrich_tree",
"distinct_id": "workflow-run-id",
"properties": {
"organization_id": "organization-id",
"workflow_permanent_id": "workflow-permanent-id",
"task_url": "https://example.test",
},
}
]
@pytest.mark.asyncio
async def test_resolve_enrich_tree_for_context_defaults_invalid_values_to_control(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class Provider:
async def get_value_cached(
self,
_feature_name: str,
_distinct_id: str,
*,
properties: dict[str, str],
) -> str:
assert properties == {"organization_id": "organization-id"}
return "not-a-real-mode"
monkeypatch.setattr(
enrich_tree,
"app",
SimpleNamespace(EXPERIMENTATION_PROVIDER=Provider()),
)
monkeypatch.delenv("FORCE_DISABLE_LLM_SCREENSHOTS", raising=False)
ctx = SkyvernContext(enrich_tree_mode=EnrichTreeMode.ENRICHED_TREE_NO_IMAGES)
await enrich_tree.resolve_enrich_tree_for_context(ctx, "task-id", "organization-id")
assert ctx.enrich_tree_mode == EnrichTreeMode.CONTROL
@pytest.mark.asyncio
async def test_force_disable_llm_screenshots_maps_to_enriched_tree_no_images(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class Provider:
async def get_value_cached(self, *_args, **_kwargs) -> str:
raise AssertionError("PostHog should not be called when FORCE_DISABLE_LLM_SCREENSHOTS is set")
monkeypatch.setattr(
enrich_tree,
"app",
SimpleNamespace(EXPERIMENTATION_PROVIDER=Provider()),
)
monkeypatch.setenv("FORCE_DISABLE_LLM_SCREENSHOTS", "true")
ctx = SkyvernContext()
await enrich_tree.resolve_enrich_tree_for_context(ctx, "task-id", "organization-id")
assert ctx.enrich_tree_mode == EnrichTreeMode.ENRICHED_TREE_NO_IMAGES

60
tests/unit/test_enriched_element_trim.py Normal file
View file

@ -0,0 +1,60 @@
import pytest
from skyvern.forge.sdk.core.skyvern_context import EnrichTreeMode, SkyvernContext
from skyvern.webeye.scraper import scraper
@pytest.fixture
def sample_attributes() -> dict[str, str]:
return {
"unique_id": "AAAB",
"id": "email",
"name": "email",
"type": "email",
"aria-invalid": "true",
"aria-describedby": "email-error",
"invalid": "true",
"validationMessage": "Please enter a valid email",
"errorText": "Invalid email address",
"aria-expanded": "false",
"data-testid": "should-drop",
}
def test_trimmed_attributes_drops_enriched_fields_in_control(
monkeypatch: pytest.MonkeyPatch,
sample_attributes: dict[str, str],
) -> None:
monkeypatch.setattr(
scraper.skyvern_context,
"current",
lambda: SkyvernContext(enrich_tree_mode=EnrichTreeMode.CONTROL),
)
trimmed = scraper._trimmed_attributes(sample_attributes)
assert trimmed == {
"name": "email",
"type": "email",
}
def test_trimmed_attributes_keeps_enriched_fields_when_enabled(
monkeypatch: pytest.MonkeyPatch,
sample_attributes: dict[str, str],
) -> None:
monkeypatch.setattr(
scraper.skyvern_context,
"current",
lambda: SkyvernContext(enrich_tree_mode=EnrichTreeMode.ENRICHED_TREE),
)
trimmed = scraper._trimmed_attributes(sample_attributes)
assert trimmed["validationMessage"] == "Please enter a valid email"
assert trimmed["errorText"] == "Invalid email address"
assert trimmed["aria-invalid"] == "true"
assert trimmed["aria-describedby"] == "email-error"
assert trimmed["invalid"] == "true"
assert trimmed["aria-expanded"] == "false"
assert "data-testid" not in trimmed

14
tests/unit/test_load_prompt_with_elements_ceiling.py
View file

@ -19,16 +19,13 @@ def test_ceiling_fallback_keys_by_template_has_known_mappings() -> None:
"previous_extracted_information",
"extracted_information_schema",
"extracted_text",
"non_vision_page_context",
]
assert CEILING_FALLBACK_KEYS_BY_TEMPLATE["extract-action"] == [
"action_history",
"navigation_payload_str",
"non_vision_page_context",
]
assert CEILING_FALLBACK_KEYS_BY_TEMPLATE["data-extraction-summary"] == [
"data_extraction_schema",
"non_vision_page_context",
]
@ -112,12 +109,12 @@ def test_load_prompt_with_elements_respects_ceiling_for_small_prompts() -> None:
assert count_tokens(rendered) <= PROMPT_HARD_CEILING_TOKENS
def test_load_prompt_with_elements_tracked_drops_non_vision_context_as_last_resort() -> None:
def test_load_prompt_with_elements_tracked_drops_extracted_text_as_last_resort() -> None:
from skyvern.forge.prompts import prompt_engine as engine_module
from skyvern.utils.prompt_engine import PROMPT_HARD_CEILING_TOKENS, load_prompt_with_elements_tracked
from skyvern.utils.token_counter import count_tokens
oversized_non_vision_context = '{"visible_text":"' + ("UNIQUE_NON_VISION_CONTEXT " * 300_000) + '"}'
oversized_extracted_text = "UNIQUE_EXTRACTED_TEXT " * 300_000
rendered, post_kwargs = load_prompt_with_elements_tracked(
element_tree_builder=_make_element_tree_builder(),
@ -126,17 +123,16 @@ def test_load_prompt_with_elements_tracked_drops_non_vision_context_as_last_reso
data_extraction_goal="Extract documents",
extracted_information_schema=None,
current_url="https://example.test",
extracted_text=None,
extracted_text=oversized_extracted_text,
error_code_mapping_str=None,
navigation_payload=None,
local_datetime="2026-04-14T12:00:00",
previous_extracted_information=None,
non_vision_page_context=oversized_non_vision_context,
)
assert count_tokens(rendered) <= PROMPT_HARD_CEILING_TOKENS
assert "UNIQUE_NON_VISION_CONTEXT" not in rendered
assert post_kwargs["non_vision_page_context"] is None
assert "UNIQUE_EXTRACTED_TEXT" not in rendered
assert post_kwargs["extracted_text"] is None
def test_enforce_prompt_ceiling_tracked_reports_dropped_keys() -> None:

186
tests/unit/test_non_vision_context.py
View file

@ -1,186 +0,0 @@
from __future__ import annotations
import json
from types import SimpleNamespace
import pytest
from skyvern.forge.sdk.core.skyvern_context import LLMVisionMode, SkyvernContext
from skyvern.forge.sdk.prompting import NON_VISION_CONTEXT_HEADER, _with_non_vision_context
from skyvern.webeye.scraper import non_vision_context
def test_accessibility_context_script_keeps_backslash_regex_escaped() -> None:
assert '/["\\\\]/g' in non_vision_context._ACCESSIBILITY_CONTEXT_SCRIPT
assert '/["\\]/g' not in non_vision_context._ACCESSIBILITY_CONTEXT_SCRIPT
class FakePage:
async def evaluate(self, _script: str, args: dict) -> dict:
return {
"title": "Example",
"url": "https://example.test",
"visible_text": "Save changes",
"accessibility_tree": [
{
"skyvern_id": "abc",
"tag": "button",
"role": "button",
"name": "Save",
}
],
"max_nodes": args["maxNodes"],
}
class SensitiveFakePage:
async def evaluate(self, _script: str, _args: dict) -> dict:
return {
"visible_text": "Password hunter2 One-time code 123456 Email user@example.test",
"accessibility_tree": [
{
"tag": "input",
"role": "textbox",
"name": "Password",
"type": "password",
"value": "hunter2",
},
{
"tag": "input",
"role": "textbox",
"name": "One-time code",
"type": "text",
"autocomplete": "one-time-code",
"value": "123456",
},
{
"tag": "input",
"role": "textbox",
"name": "Email",
"type": "email",
"value": "user@example.test",
},
],
}
class LargeFakePage:
async def evaluate(self, _script: str, _args: dict) -> dict:
return {
"title": "Large page",
"url": "https://example.test/large",
"visible_text": "Visible text " * 100,
"accessibility_tree": [
{
"tag": "button",
"role": "button",
"name": f"Button {index}",
"text": "Button text " * 20,
}
for index in range(20)
],
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
"mode",
[
LLMVisionMode.CONTROL,
LLMVisionMode.FALLBACK_WITHOUT_A11Y,
],
)
async def test_build_non_vision_context_if_needed_omits_accessibility_for_modes_without_a11y(
monkeypatch: pytest.MonkeyPatch,
mode: LLMVisionMode,
) -> None:
monkeypatch.setattr(
non_vision_context.skyvern_context,
"current",
lambda: SkyvernContext(llm_vision_mode=mode),
)
rendered = await non_vision_context.build_non_vision_page_context_if_needed(page=FakePage())
assert rendered is None
@pytest.mark.asyncio
@pytest.mark.parametrize(
"mode",
[
LLMVisionMode.NO_IMAGES_WITH_A11Y,
LLMVisionMode.FALLBACK_WITH_A11Y,
],
)
async def test_build_non_vision_context_includes_accessibility_tree_for_a11y_modes(
monkeypatch: pytest.MonkeyPatch,
mode: LLMVisionMode,
) -> None:
monkeypatch.setattr(
non_vision_context.skyvern_context,
"current",
lambda: SkyvernContext(llm_vision_mode=mode),
)
scraped_page = SimpleNamespace(url="https://fallback.test", extracted_text="Fallback text")
rendered = await non_vision_context.build_non_vision_page_context_if_needed(
scraped_page=scraped_page,
page=FakePage(),
)
assert rendered is not None
assert '"accessibility_tree"' in rendered
assert '"skyvern_id":"abc"' in rendered
assert '"visible_text":"Save changes"' in rendered
@pytest.mark.asyncio
async def test_build_non_vision_context_redacts_sensitive_input_values() -> None:
scraped_page = SimpleNamespace(
url="https://example.test/login",
extracted_text="Fallback text with password hunter2 and one-time code 123456",
)
rendered = await non_vision_context.build_non_vision_page_context(
page=SensitiveFakePage(), scraped_page=scraped_page
)
assert rendered is not None
assert "hunter2" not in rendered
assert "123456" not in rendered
assert rendered.count('"value_redacted":true') == 2
assert "user@example.test" in rendered
assert "visible_text" not in json.loads(rendered)
@pytest.mark.asyncio
async def test_build_non_vision_context_truncates_as_valid_json() -> None:
rendered = await non_vision_context.build_non_vision_page_context(page=LargeFakePage(), max_chars=250)
assert rendered is not None
assert len(rendered) <= 250
parsed = json.loads(rendered)
assert parsed["truncated"] is True
def test_prompt_engine_appends_non_vision_context_once() -> None:
rendered = _with_non_vision_context(
"Base prompt",
"skyvern/example",
{"non_vision_page_context": '{"accessibility_tree":[]}'},
)
assert rendered.count(NON_VISION_CONTEXT_HEADER) == 1
assert '{"accessibility_tree":[]}' in rendered
assert _with_non_vision_context(rendered, "skyvern/example", {"non_vision_page_context": "{}"}) == rendered
def test_prompt_engine_does_not_append_non_vision_context_to_static_cache_template() -> None:
rendered = _with_non_vision_context(
"Static prompt",
"skyvern/extract-action-static",
{"non_vision_page_context": '{"accessibility_tree":[]}'},
)
assert rendered == "Static prompt"