diff --git a/skyvern/forge/sdk/routes/streaming/channels/cdp.py b/skyvern/forge/sdk/routes/streaming/channels/cdp.py index 605625478..9000c43af 100644 --- a/skyvern/forge/sdk/routes/streaming/channels/cdp.py +++ b/skyvern/forge/sdk/routes/streaming/channels/cdp.py @@ -26,6 +26,7 @@ from skyvern.webeye.cdp_connection import ( is_local_pbs_cdp_url, resolve_local_pbs_cdp_url, ) +from skyvern.webeye.main_world_eval import evaluate_in_main_world if t.TYPE_CHECKING: from skyvern.forge.sdk.routes.streaming.channels.vnc import VncChannel @@ -185,7 +186,7 @@ class CdpChannel: LOG.info(f"{self.class_name} evaluating js", expression=expression[:100], **self.identity) try: - result = await self.page.evaluate(expression, arg) + result = await evaluate_in_main_world(self.page, expression, arg) LOG.info(f"{self.class_name} evaluated js successfully", **self.identity) return result except Exception: diff --git a/skyvern/forge/sdk/routes/streaming/channels/execution.py b/skyvern/forge/sdk/routes/streaming/channels/execution.py index 1b59aa060..ad4bad250 100644 --- a/skyvern/forge/sdk/routes/streaming/channels/execution.py +++ b/skyvern/forge/sdk/routes/streaming/channels/execution.py @@ -24,6 +24,7 @@ from skyvern.config import settings from skyvern.forge.sdk.routes.streaming.channels.cdp import CdpChannel from skyvern.forge.sdk.routes.streaming.payload_limits import MAX_SCREENSHOT_BYTES from skyvern.forge.sdk.routes.streaming.registries import get_vnc_channel +from skyvern.webeye.main_world_eval import evaluate_in_main_world if t.TYPE_CHECKING: from skyvern.forge.sdk.routes.streaming.channels.message import MessageChannel @@ -279,7 +280,7 @@ class LocalExecutionChannel(ExecutionChannel): # which LocalExecutionChannel intentionally leaves None. if not self.page: raise RuntimeError(f"{self.class_name} evaluate_js: no page available.") - return await self.page.evaluate(expression, arg) + return await evaluate_in_main_world(self.page, expression, arg) async def close(self) -> None: # We don't own the page or context; do not close. diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py index 229a3ee7e..cb2a0d0a4 100644 --- a/skyvern/webeye/actions/handler.py +++ b/skyvern/webeye/actions/handler.py @@ -121,6 +121,7 @@ from skyvern.webeye.cdp_download_interceptor import ( is_download_response, normalize_download_filename, ) +from skyvern.webeye.main_world_eval import evaluate_in_main_world from skyvern.webeye.scraper.scraped_page import ( CleanupElementTreeFunc, ElementTreeBuilder, @@ -3282,7 +3283,7 @@ async def handle_execute_js_action( ) -> list[ActionResult]: import json as _json - result = await page.evaluate(action.js_code) + result = await evaluate_in_main_world(page, action.js_code) if result is None: return [ActionSuccess(data="undefined")] if isinstance(result, str): diff --git a/skyvern/webeye/main_world_eval.py b/skyvern/webeye/main_world_eval.py new file mode 100644 index 000000000..6a2878301 --- /dev/null +++ b/skyvern/webeye/main_world_eval.py @@ -0,0 +1,110 @@ +"""Generic main-world JS evaluation hook. + +When a context has a prefix configured via ``configure_main_world_prefix``, +``evaluate_in_main_world`` runs the script via a single CDP ``Runtime.evaluate`` +call with the prefix prepended, so middleware that inspects script content sees +the prefix intact. With no prefix configured the call falls through to +``page.evaluate`` with no extra CDP overhead. The prefix is opaque text. +""" + +from __future__ import annotations + +import contextlib +import json +import re +import weakref +from typing import Any + +from playwright.async_api import BrowserContext, Page + +_CONTEXT_PREFIXES: weakref.WeakKeyDictionary[BrowserContext, str] = weakref.WeakKeyDictionary() + + +def configure_main_world_prefix(context: BrowserContext, prefix: str) -> None: + """Attach an opaque text prefix to ``context``; prepended to every JS body.""" + _CONTEXT_PREFIXES[context] = prefix + + +def clear_main_world_prefix(context: BrowserContext) -> None: + _CONTEXT_PREFIXES.pop(context, None) + + +def get_main_world_prefix(context: BrowserContext) -> str | None: + return _CONTEXT_PREFIXES.get(context) + + +def _resolve_prefix(page: Page) -> str | None: + return _CONTEXT_PREFIXES.get(page.context) + + +# Conservative: only arrow / function declarations that Playwright would auto-wrap +# as an IIFE. Bare expressions / object literals must NOT be wrapped. +# Limitation: arrow param list is ``[^()]*``, so nested-paren params (e.g. +# ``(a = (1+2)) => a``) don't match — current callers don't use that shape. +_ARROW_FN_RE = re.compile(r"^\s*(async\s+)?(\([^()]*\)|\w+)\s*=>") +_FUNCTION_DECL_RE = re.compile(r"^\s*(async\s+)?function\b") + + +def _is_function_form(expression: str) -> bool: + return bool(_ARROW_FN_RE.match(expression) or _FUNCTION_DECL_RE.match(expression)) + + +def _extract_runtime_result(result: dict[str, Any]) -> Any: + # CDP returns ``value`` for JSON-serialisable types; unserialisable primitives + # (NaN/Infinity/-0/BigInt) fall through to None, matching what page.evaluate + # gives callers that don't opt into custom serialisation. + result_obj = result.get("result") or {} + if "value" in result_obj: + return result_obj["value"] + return None + + +async def evaluate_in_main_world(page: Page, expression: str, arg: Any = None) -> Any: + """Evaluate ``expression`` in the page main world when a prefix is configured. + + No prefix → identical to ``page.evaluate(expression, arg)``. Prefix → single + ``Runtime.evaluate`` call so middleware sees the prefix intact. Function-form + expressions are auto-wrapped as IIFEs; ``arg`` is inlined as a JSON literal. + Non-function expressions drop ``arg`` (mirroring page.evaluate). page.evaluate + is avoided here because Playwright's function-string normalisation requires + an unprefixed function head, which a leading prefix line breaks. + """ + prefix = _resolve_prefix(page) + if prefix is None: + if arg is None: + return await page.evaluate(expression) + return await page.evaluate(expression, arg) + + if _is_function_form(expression): + if arg is None: + wrapped = f"({expression})()" + else: + wrapped = f"({expression})({json.dumps(arg)})" + else: + wrapped = expression + body = f"{prefix}\n{wrapped}" + cdp_session = await page.context.new_cdp_session(page) + try: + result = await cdp_session.send( + "Runtime.evaluate", + { + "expression": body, + "returnByValue": True, + "awaitPromise": True, + }, + ) + finally: + with contextlib.suppress(Exception): + await cdp_session.detach() + + exception_details = result.get("exceptionDetails") + if exception_details: + exception = exception_details.get("exception") or {} + description = ( + exception.get("description") + or exception.get("value") + or exception_details.get("text", "Runtime.evaluate exception") + ) + raise RuntimeError(f"main-world evaluate raised: {description}") + + return _extract_runtime_result(result) diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py index 17e675956..929361e35 100644 --- a/skyvern/webeye/utils/page.py +++ b/skyvern/webeye/utils/page.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import json import time from enum import StrEnum from io import BytesIO @@ -18,6 +19,7 @@ from skyvern.exceptions import FailedToTakeScreenshot from skyvern.forge.sdk.core import skyvern_context from skyvern.forge.sdk.settings_manager import SettingsManager from skyvern.forge.sdk.trace import apply_context_attrs, traced +from skyvern.webeye.main_world_eval import evaluate_in_main_world, get_main_world_prefix LOG = structlog.get_logger() @@ -47,6 +49,29 @@ def _is_navigation_context_lost(error_msg: str) -> bool: return "ReferenceError" in error_msg and "is not defined" in error_msg +def _is_json_inlinable(arg: Any) -> bool: + # ElementHandle / JSHandle aren't JSON-serialisable; those must keep + # Playwright's own marshalling instead of being inlined into Runtime.evaluate. + try: + json.dumps(arg) + except (TypeError, ValueError): + return False + return True + + +async def _dispatch_evaluate(frame: Page | Frame, expression: str, arg: Any | None) -> Any: + # Page + prefix + JSON-safe arg → main-world hook (preserves the marker). + # Iframe Frames and non-JSON args fall back to per-frame evaluate so iframe + # contexts and Playwright handle-marshalling keep working. + if not isinstance(frame, Page): + return await frame.evaluate(expression=expression, arg=arg) + if get_main_world_prefix(frame.context) is None: + return await frame.evaluate(expression=expression, arg=arg) + if arg is not None and not _is_json_inlinable(arg): + return await frame.evaluate(expression=expression, arg=arg) + return await evaluate_in_main_world(frame, expression, arg) + + async def _wait_for_navigation_settle(frame: Page | Frame, timeout_ms: float) -> None: if timeout_ms <= 0: return @@ -288,7 +313,7 @@ class SkyvernFrame: ) -> Any: try: async with asyncio.timeout(timeout_ms / 1000): - return await frame.evaluate(expression=expression, arg=arg) + return await _dispatch_evaluate(frame, expression, arg) except PlaywrightError as e: error_msg = str(e) if not _is_navigation_context_lost(error_msg): @@ -300,6 +325,19 @@ class SkyvernFrame: timeout_ms=timeout_ms, initial_error=error_msg, ) + except RuntimeError as e: + # `evaluate_in_main_world` raises RuntimeError on Runtime.evaluate + # exception payloads; only navigation-context-lost text recovers here. + error_msg = str(e) + if not _is_navigation_context_lost(error_msg): + raise + return await SkyvernFrame._evaluate_with_navigation_recovery( + frame=frame, + expression=expression, + arg=arg, + timeout_ms=timeout_ms, + initial_error=error_msg, + ) except asyncio.TimeoutError: LOG.exception("Skyvern timed out trying to analyze the page", expression=expression) raise TimeoutError("Skyvern timed out trying to analyze the page") @@ -351,14 +389,16 @@ class SkyvernFrame: raise TimeoutError("Skyvern timed out trying to analyze the page") try: async with asyncio.timeout(inject_budget): - await frame.evaluate(expression=JS_FUNCTION_DEFS) + # Same dispatch helper so a prefixed Page re-injects + # JS_FUNCTION_DEFS via Runtime.evaluate (preserving the marker). + await _dispatch_evaluate(frame, JS_FUNCTION_DEFS, None) except asyncio.TimeoutError: LOG.exception( "Skyvern timed out trying to analyze the page during domUtils.js re-injection", expression=expression, ) raise TimeoutError("Skyvern timed out trying to analyze the page") - except PlaywrightError as inject_err: + except (PlaywrightError, RuntimeError) as inject_err: last_error_msg = str(inject_err) if attempt == _NAVIGATION_RECOVERY_MAX_ATTEMPTS or not _is_navigation_context_lost(last_error_msg): LOG.warning( @@ -377,11 +417,11 @@ class SkyvernFrame: raise TimeoutError("Skyvern timed out trying to analyze the page") try: async with asyncio.timeout(retry_budget): - return await frame.evaluate(expression=expression, arg=arg) + return await _dispatch_evaluate(frame, expression, arg) except asyncio.TimeoutError: LOG.exception("Skyvern timed out on retry after JS context re-injection", expression=expression) raise TimeoutError("Skyvern timed out trying to analyze the page") - except PlaywrightError as retry_err: + except (PlaywrightError, RuntimeError) as retry_err: last_error_msg = str(retry_err) if attempt == _NAVIGATION_RECOVERY_MAX_ATTEMPTS or not _is_navigation_context_lost(last_error_msg): raise diff --git a/tests/unit/forge/sdk/routes/streaming/test_message_reify.py b/tests/unit/forge/sdk/routes/streaming/test_message_reify.py index d20f3a364..8105e8a38 100644 --- a/tests/unit/forge/sdk/routes/streaming/test_message_reify.py +++ b/tests/unit/forge/sdk/routes/streaming/test_message_reify.py @@ -241,7 +241,7 @@ class TestLocalExecutionChannel: result = await channel.evaluate_js("() => 'hello'") assert result == "hello" - page.evaluate.assert_awaited_once_with("() => 'hello'", None) + page.evaluate.assert_awaited_once_with("() => 'hello'") @pytest.mark.asyncio async def test_evaluate_js_forwards_arg(self) -> None: diff --git a/tests/unit/webeye/test_main_world_eval.py b/tests/unit/webeye/test_main_world_eval.py new file mode 100644 index 000000000..38d49c06d --- /dev/null +++ b/tests/unit/webeye/test_main_world_eval.py @@ -0,0 +1,290 @@ +"""Tests for the generic main-world evaluation hook.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from skyvern.webeye.main_world_eval import ( + clear_main_world_prefix, + configure_main_world_prefix, + evaluate_in_main_world, + get_main_world_prefix, +) + + +class _MockBrowserContext: + """Hashable stand-in for BrowserContext (WeakKeyDictionary key).""" + + +def _mock_page(prefix: str | None = None) -> tuple[MagicMock, _MockBrowserContext]: + context = _MockBrowserContext() + page = MagicMock() + page.context = context + page.evaluate = AsyncMock() + cdp_session = MagicMock() + cdp_session.send = AsyncMock(return_value={"result": {"value": "ok"}}) + cdp_session.detach = AsyncMock() + page.context_cdp_session = cdp_session + context_mock = MagicMock(wraps=context) + context_mock.new_cdp_session = AsyncMock(return_value=cdp_session) + page.context = context_mock + if prefix is not None: + configure_main_world_prefix(context_mock, prefix) + return page, context_mock + + +@pytest.fixture(autouse=True) +def _reset_prefix_state() -> None: + # WeakKeyDictionary already drops mock contexts when they go out of scope. + return None + + +class TestNoPrefixConfigured: + @pytest.mark.asyncio + async def test_passes_through_to_page_evaluate_no_arg(self) -> None: + page = MagicMock() + page.context = _MockBrowserContext() + page.evaluate = AsyncMock(return_value=42) + + result = await evaluate_in_main_world(page, "() => 42") + + assert result == 42 + page.evaluate.assert_awaited_once_with("() => 42") + + @pytest.mark.asyncio + async def test_passes_through_to_page_evaluate_with_arg(self) -> None: + page = MagicMock() + page.context = _MockBrowserContext() + page.evaluate = AsyncMock(return_value="hi") + + result = await evaluate_in_main_world(page, "(x) => x", "hi") + + assert result == "hi" + page.evaluate.assert_awaited_once_with("(x) => x", "hi") + + +class TestPrefixConfiguredNoArg: + @pytest.mark.asyncio + async def test_function_form_wraps_as_iife_and_uses_runtime_evaluate(self) -> None: + page, context = _mock_page(prefix="// MARKER") + page.context.new_cdp_session.return_value.send = AsyncMock(return_value={"result": {"value": 7}}) + + result = await evaluate_in_main_world(page, "() => 7") + + assert result == 7 + page.evaluate.assert_not_awaited() + page.context.new_cdp_session.assert_awaited_once_with(page) + send_mock = page.context.new_cdp_session.return_value.send + send_mock.assert_awaited_once() + method, params = send_mock.await_args.args + assert method == "Runtime.evaluate" + assert params["expression"].startswith("// MARKER\n") + assert "(() => 7)()" in params["expression"] + assert params["returnByValue"] is True + assert params["awaitPromise"] is True + page.context.new_cdp_session.return_value.detach.assert_awaited_once() + + @pytest.mark.asyncio + async def test_statement_form_passes_through_unchanged_to_runtime_evaluate(self) -> None: + page, context = _mock_page(prefix="/* P */") + page.context.new_cdp_session.return_value.send = AsyncMock(return_value={"result": {"value": None}}) + + await evaluate_in_main_world(page, "window.x = 1;") + + send_mock = page.context.new_cdp_session.return_value.send + method, params = send_mock.await_args.args + assert method == "Runtime.evaluate" + assert params["expression"] == "/* P */\nwindow.x = 1;" + + @pytest.mark.asyncio + async def test_async_arrow_function_wraps_as_iife(self) -> None: + page, _ = _mock_page(prefix="// P") + page.context.new_cdp_session.return_value.send = AsyncMock(return_value={"result": {"value": "done"}}) + + await evaluate_in_main_world(page, "async () => fetch('/x')") + + params = page.context.new_cdp_session.return_value.send.await_args.args[1] + assert "(async () => fetch('/x'))()" in params["expression"] + + @pytest.mark.asyncio + async def test_runtime_evaluate_exception_is_raised(self) -> None: + page, _ = _mock_page(prefix="// P") + page.context.new_cdp_session.return_value.send = AsyncMock( + return_value={ + "exceptionDetails": { + "text": "Uncaught", + "exception": {"description": "ReferenceError: x is not defined"}, + } + } + ) + + with pytest.raises(RuntimeError, match="ReferenceError: x is not defined"): + await evaluate_in_main_world(page, "() => x") + + @pytest.mark.asyncio + async def test_detach_is_called_even_on_failure(self) -> None: + page, _ = _mock_page(prefix="// P") + page.context.new_cdp_session.return_value.send = AsyncMock(side_effect=RuntimeError("boom")) + detach_mock = page.context.new_cdp_session.return_value.detach + + with pytest.raises(RuntimeError, match="boom"): + await evaluate_in_main_world(page, "() => 1") + + detach_mock.assert_awaited_once() + + +class TestPrefixConfiguredWithArg: + @pytest.mark.asyncio + async def test_function_form_with_arg_inlines_arg_via_runtime_evaluate(self) -> None: + page, _ = _mock_page(prefix="// MARK") + page.context.new_cdp_session.return_value.send = AsyncMock(return_value={"result": {"value": "arg-value"}}) + + result = await evaluate_in_main_world(page, "(x) => x", "arg-value") + + assert result == "arg-value" + # No page.evaluate: a leading prefix line breaks Playwright's function-string normalisation. + page.evaluate.assert_not_awaited() + params = page.context.new_cdp_session.return_value.send.await_args.args[1] + assert params["expression"] == '// MARK\n((x) => x)("arg-value")' + + @pytest.mark.asyncio + async def test_function_declaration_form_with_arg_inlines_arg(self) -> None: + page, _ = _mock_page(prefix="// MARK") + page.context.new_cdp_session.return_value.send = AsyncMock(return_value={"result": {"value": 5}}) + + await evaluate_in_main_world(page, "function (x) { return x + 1 }", 4) + + params = page.context.new_cdp_session.return_value.send.await_args.args[1] + assert params["expression"] == "// MARK\n(function (x) { return x + 1 })(4)" + + @pytest.mark.asyncio + async def test_non_function_expression_with_arg_drops_arg(self) -> None: + """page.evaluate ignores extra args for non-function strings; mirror that.""" + page, _ = _mock_page(prefix="// MARK") + page.context.new_cdp_session.return_value.send = AsyncMock(return_value={"result": {"value": 3}}) + + await evaluate_in_main_world(page, "1 + 2", "ignored") + + params = page.context.new_cdp_session.return_value.send.await_args.args[1] + assert params["expression"] == "// MARK\n1 + 2" + + +class TestExpressionShapeRegressions: + """Shapes the earlier prefix-startswith-`(` heuristic mis-wrapped.""" + + @pytest.mark.asyncio + async def test_parenthesised_arithmetic_is_not_wrapped(self) -> None: + page, _ = _mock_page(prefix="// P") + page.context.new_cdp_session.return_value.send = AsyncMock(return_value={"result": {"value": 3}}) + + await evaluate_in_main_world(page, "(1 + 2)") + + params = page.context.new_cdp_session.return_value.send.await_args.args[1] + assert params["expression"] == "// P\n(1 + 2)" # not ((1 + 2))() — that would TypeError + + @pytest.mark.asyncio + async def test_object_literal_in_parens_is_not_wrapped(self) -> None: + page, _ = _mock_page(prefix="// P") + page.context.new_cdp_session.return_value.send = AsyncMock(return_value={"result": {"value": {"foo": 1}}}) + + await evaluate_in_main_world(page, "({foo: 1})") + + params = page.context.new_cdp_session.return_value.send.await_args.args[1] + assert params["expression"] == "// P\n({foo: 1})" # not (({foo: 1}))() — that would TypeError + + @pytest.mark.asyncio + async def test_single_param_arrow_function_wraps_as_iife(self) -> None: + page, _ = _mock_page(prefix="// P") + page.context.new_cdp_session.return_value.send = AsyncMock(return_value={"result": {"value": 1}}) + + await evaluate_in_main_world(page, "x => x") + + params = page.context.new_cdp_session.return_value.send.await_args.args[1] + assert "(x => x)()" in params["expression"] + + @pytest.mark.asyncio + async def test_destructured_param_arrow_function_wraps_as_iife(self) -> None: + page, _ = _mock_page(prefix="// P") + page.context.new_cdp_session.return_value.send = AsyncMock(return_value={"result": {"value": 3}}) + + await evaluate_in_main_world(page, "({a, b}) => a + b") + + params = page.context.new_cdp_session.return_value.send.await_args.args[1] + assert "(({a, b}) => a + b)()" in params["expression"] + + @pytest.mark.asyncio + async def test_default_param_arrow_function_wraps_as_iife(self) -> None: + page, _ = _mock_page(prefix="// P") + page.context.new_cdp_session.return_value.send = AsyncMock(return_value={"result": {"value": 1}}) + + await evaluate_in_main_world(page, "(x = 1) => x") + + params = page.context.new_cdp_session.return_value.send.await_args.args[1] + assert "((x = 1) => x)()" in params["expression"] + + @pytest.mark.asyncio + async def test_named_function_with_arg_inlines_arg(self) -> None: + page, _ = _mock_page(prefix="// P") + page.context.new_cdp_session.return_value.send = AsyncMock(return_value={"result": {"value": 5}}) + + await evaluate_in_main_world(page, "function foo(x) { return x }", 5) + + params = page.context.new_cdp_session.return_value.send.await_args.args[1] + assert "(function foo(x) { return x })(5)" in params["expression"] + + +class TestRuntimeResultDecoding: + @pytest.mark.asyncio + async def test_returns_none_when_runtime_value_absent(self) -> None: + page, _ = _mock_page(prefix="// P") + page.context.new_cdp_session.return_value.send = AsyncMock(return_value={"result": {}}) + + result = await evaluate_in_main_world(page, "() => undefined") + + assert result is None + + @pytest.mark.asyncio + async def test_unserializable_value_falls_through_to_none(self) -> None: + """CDP encodes NaN/Infinity via ``unserializableValue``; callers expect None.""" + page, _ = _mock_page(prefix="// P") + page.context.new_cdp_session.return_value.send = AsyncMock( + return_value={"result": {"type": "number", "unserializableValue": "NaN"}} + ) + + result = await evaluate_in_main_world(page, "() => NaN") + + assert result is None + + @pytest.mark.asyncio + async def test_exception_details_without_description_uses_text_fallback(self) -> None: + page, _ = _mock_page(prefix="// P") + page.context.new_cdp_session.return_value.send = AsyncMock( + return_value={ + "exceptionDetails": { + "text": "Uncaught (in promise)", + "exception": {"type": "object"}, + } + } + ) + + with pytest.raises(RuntimeError, match="Uncaught"): + await evaluate_in_main_world(page, "() => 1") + + +class TestPrefixRegistry: + def test_configure_and_get_round_trip(self) -> None: + context = _MockBrowserContext() + configure_main_world_prefix(context, "// FOO") + assert get_main_world_prefix(context) == "// FOO" + + def test_clear_removes_prefix(self) -> None: + context = _MockBrowserContext() + configure_main_world_prefix(context, "// FOO") + clear_main_world_prefix(context) + assert get_main_world_prefix(context) is None + + def test_get_returns_none_when_unconfigured(self) -> None: + context = _MockBrowserContext() + assert get_main_world_prefix(context) is None diff --git a/tests/unit/webeye/test_skyvern_frame_main_world_routing.py b/tests/unit/webeye/test_skyvern_frame_main_world_routing.py new file mode 100644 index 000000000..303c0c999 --- /dev/null +++ b/tests/unit/webeye/test_skyvern_frame_main_world_routing.py @@ -0,0 +1,245 @@ +"""Tests for SkyvernFrame.evaluate routing into the main-world eval hook.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from playwright.async_api import Frame, Page + +from skyvern.webeye.main_world_eval import ( + clear_main_world_prefix, + configure_main_world_prefix, +) +from skyvern.webeye.utils.page import SkyvernFrame + + +class _HashableContext: + """Hashable + weak-ref-able stand-in for BrowserContext (WeakKeyDictionary key).""" + + +def _make_page_mock(prefix: str | None) -> tuple[MagicMock, _HashableContext]: + context = _HashableContext() + page = MagicMock(spec=Page) + page.context = context + page.evaluate = AsyncMock(return_value="page-evaluate-result") + cdp_session = MagicMock() + cdp_session.send = AsyncMock(return_value={"result": {"value": "runtime-evaluate-result"}}) + cdp_session.detach = AsyncMock() + page.context.new_cdp_session = AsyncMock(return_value=cdp_session) # type: ignore[attr-defined] + if prefix is not None: + configure_main_world_prefix(context, prefix) # type: ignore[arg-type] + return page, context + + +def _make_frame_mock() -> MagicMock: + """A Frame mock that is NOT a Page (iframe-style).""" + frame = MagicMock(spec=Frame) + frame.evaluate = AsyncMock(return_value="frame-evaluate-result") + return frame + + +@pytest.fixture(autouse=True) +def _short_timeout() -> object: + return None + + +class TestSkyvernFrameEvaluateRouting: + @pytest.mark.asyncio + async def test_page_without_prefix_delegates_to_page_evaluate(self) -> None: + page, ctx = _make_page_mock(prefix=None) + + result = await SkyvernFrame.evaluate(page, "() => 1") + + assert result == "page-evaluate-result" + page.evaluate.assert_awaited_once_with(expression="() => 1", arg=None) + page.context.new_cdp_session.assert_not_awaited() # type: ignore[attr-defined] + + @pytest.mark.asyncio + async def test_page_with_prefix_and_no_arg_routes_through_runtime_evaluate(self) -> None: + page, ctx = _make_page_mock(prefix="// MARK") + try: + result = await SkyvernFrame.evaluate(page, "() => 7") + finally: + clear_main_world_prefix(ctx) # type: ignore[arg-type] + + assert result == "runtime-evaluate-result" + page.evaluate.assert_not_awaited() + params = page.context.new_cdp_session.return_value.send.await_args.args[1] # type: ignore[attr-defined] + assert params["expression"].startswith("// MARK\n") + assert "(() => 7)()" in params["expression"] + + @pytest.mark.asyncio + async def test_page_with_prefix_and_json_arg_inlines_arg(self) -> None: + page, ctx = _make_page_mock(prefix="// MARK") + try: + await SkyvernFrame.evaluate(page, "(pos) => __pwCursorMove(pos)", [1.5, 2.5]) + finally: + clear_main_world_prefix(ctx) # type: ignore[arg-type] + + params = page.context.new_cdp_session.return_value.send.await_args.args[1] # type: ignore[attr-defined] + assert "((pos) => __pwCursorMove(pos))([1.5, 2.5])" in params["expression"] + page.evaluate.assert_not_awaited() + + @pytest.mark.asyncio + async def test_page_with_prefix_and_non_json_arg_falls_back_to_page_evaluate(self) -> None: + """ElementHandle args can't be JSON-inlined → keep Playwright marshalling.""" + page, ctx = _make_page_mock(prefix="// MARK") + element_handle_like = MagicMock() # stands in for an ElementHandle/JSHandle + try: + await SkyvernFrame.evaluate(page, "(el) => el.blur()", element_handle_like) + finally: + clear_main_world_prefix(ctx) # type: ignore[arg-type] + + page.evaluate.assert_awaited_once_with(expression="(el) => el.blur()", arg=element_handle_like) + page.context.new_cdp_session.assert_not_awaited() # type: ignore[attr-defined] + + @pytest.mark.asyncio + async def test_frame_target_never_routes_through_runtime_evaluate(self) -> None: + """Iframe Frames have their own world; page-level Runtime.evaluate lands elsewhere.""" + frame = _make_frame_mock() + + await SkyvernFrame.evaluate(frame, "() => 1") + + frame.evaluate.assert_awaited_once_with(expression="() => 1", arg=None) + + @pytest.mark.asyncio + async def test_runtime_evaluate_navigation_context_lost_triggers_recovery(self) -> None: + """Runtime.evaluate context-destroyed must hit recovery, like PlaywrightError.""" + page, ctx = _make_page_mock(prefix="// MARK") + page.context.new_cdp_session.return_value.send = AsyncMock( # type: ignore[attr-defined] + return_value={ + "exceptionDetails": { + "text": "Uncaught", + "exception": {"description": "Error: Execution context was destroyed by a navigation"}, + } + } + ) + + with patch.object( + SkyvernFrame, + "_evaluate_with_navigation_recovery", + new=AsyncMock(return_value="recovered"), + ) as recovery: + try: + result = await SkyvernFrame.evaluate(page, "() => 1") + finally: + clear_main_world_prefix(ctx) # type: ignore[arg-type] + + assert result == "recovered" + recovery.assert_awaited_once() + + @pytest.mark.asyncio + async def test_runtime_evaluate_non_recovery_runtime_error_propagates(self) -> None: + """Non-navigation RuntimeError must propagate, not get swallowed.""" + page, ctx = _make_page_mock(prefix="// MARK") + page.context.new_cdp_session.return_value.send = AsyncMock( # type: ignore[attr-defined] + return_value={ + "exceptionDetails": { + "text": "Uncaught", + "exception": {"description": "TypeError: undefined is not a function"}, + } + } + ) + + try: + with pytest.raises(RuntimeError, match="undefined is not a function"): + await SkyvernFrame.evaluate(page, "() => x.y()") + finally: + clear_main_world_prefix(ctx) # type: ignore[arg-type] + + +class TestNavigationRecoveryRouting: + """Recovery loop must keep using the main-world hook for prefixed Pages, + otherwise the marker is dropped on the post-navigation re-injection + retry.""" + + @pytest.mark.asyncio + async def test_prefixed_page_recovery_uses_runtime_evaluate_throughout(self) -> None: + """For a prefixed Page, the original call, the JS_FUNCTION_DEFS + re-injection, AND the final retry must all run via CDP Runtime.evaluate.""" + page, ctx = _make_page_mock(prefix="// MARK") + page.wait_for_load_state = AsyncMock() + + send_calls: list[dict[str, str]] = [] + + async def fake_send(method: str, params: dict[str, str]) -> dict: + send_calls.append({"method": method, "expression": params["expression"]}) + if len(send_calls) == 1: + return { + "exceptionDetails": { + "text": "Uncaught", + "exception": {"description": "Error: Execution context was destroyed by a navigation"}, + } + } + if len(send_calls) == 2: + return {"result": {"value": None}} # re-injection of JS_FUNCTION_DEFS + return {"result": {"value": "final"}} # final retry + + cdp_session = MagicMock() + cdp_session.send = AsyncMock(side_effect=fake_send) + cdp_session.detach = AsyncMock() + page.context.new_cdp_session = AsyncMock(return_value=cdp_session) # type: ignore[attr-defined] + + try: + result = await SkyvernFrame.evaluate(page, "() => 1", timeout_ms=1000) + finally: + clear_main_world_prefix(ctx) # type: ignore[arg-type] + + assert result == "final" + assert len(send_calls) == 3 + # Every CDP call carried the configured prefix. + for call in send_calls: + assert call["method"] == "Runtime.evaluate" + assert call["expression"].startswith("// MARK\n") + # Recovery must NOT have fallen back to raw page.evaluate for a JSON-safe call. + page.evaluate.assert_not_awaited() + + @pytest.mark.asyncio + async def test_recovery_with_element_handle_arg_preserves_page_evaluate_on_final_retry( + self, + ) -> None: + """ElementHandle arg → final retry keeps page.evaluate (re-injection has + arg=None so it can still go through the main-world hook).""" + from playwright._impl._errors import Error as PlaywrightError + + page, ctx = _make_page_mock(prefix="// MARK") + page.wait_for_load_state = AsyncMock() + + element_handle_like = MagicMock() # non-JSON-inlinable + page.evaluate = AsyncMock( + side_effect=[ + PlaywrightError("Execution context was destroyed by a navigation"), + "final-via-page-evaluate", + ] + ) + + try: + result = await SkyvernFrame.evaluate(page, "(el) => el.blur()", element_handle_like, timeout_ms=1000) + finally: + clear_main_world_prefix(ctx) # type: ignore[arg-type] + + assert result == "final-via-page-evaluate" + # Original call + final retry both went through page.evaluate (non-JSON arg). + # Re-injection used Runtime.evaluate (None arg → main-world hook is safe). + assert page.evaluate.await_count == 2 + page.context.new_cdp_session.assert_awaited_once() # type: ignore[attr-defined] + + @pytest.mark.asyncio + async def test_frame_target_recovery_still_uses_frame_evaluate(self) -> None: + """Iframe Frames keep per-frame evaluate even during recovery.""" + from playwright._impl._errors import Error as PlaywrightError + + frame = _make_frame_mock() + frame.wait_for_load_state = AsyncMock() + frame.evaluate = AsyncMock( + side_effect=[ + PlaywrightError("Execution context was destroyed by a navigation"), + None, + "frame-final", + ] + ) + + result = await SkyvernFrame.evaluate(frame, "() => 1", timeout_ms=1000) + + assert result == "frame-final" + assert frame.evaluate.await_count == 3