mirror of
https://github.com/Skyvern-AI/skyvern.git
synced 2026-04-26 10:41:14 +00:00
486 lines
18 KiB
Python
486 lines
18 KiB
Python
"""Tests for copilot session injection and output contract adapters."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from typing import Any
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
|
|
from skyvern.cli.core import session_manager
|
|
from skyvern.cli.core.result import BrowserContext as MCPBrowserContext
|
|
from skyvern.cli.core.session_manager import SessionState, scoped_session
|
|
from skyvern.forge.sdk.copilot.runtime import AgentContext, mcp_to_copilot
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _reset_session_state() -> None:
|
|
session_manager._current_session.set(None)
|
|
session_manager._global_session = None
|
|
session_manager.set_stateless_http_mode(False)
|
|
|
|
|
|
def _make_stream() -> MagicMock:
|
|
stream = MagicMock()
|
|
stream.is_disconnected = AsyncMock(return_value=False)
|
|
return stream
|
|
|
|
|
|
def _make_ctx(**overrides: Any) -> AgentContext:
|
|
defaults = dict(
|
|
organization_id="org-1",
|
|
workflow_id="wf-1",
|
|
workflow_permanent_id="wfp-1",
|
|
workflow_yaml="",
|
|
browser_session_id="pbs_test_123",
|
|
stream=_make_stream(),
|
|
api_key="sk-test-key",
|
|
)
|
|
defaults.update(overrides)
|
|
return AgentContext(**defaults)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_scoped_session_pushes_and_restores() -> None:
|
|
"""scoped_session sets the ContextVar within scope and restores on exit."""
|
|
# ContextVar starts as None (from fixture reset)
|
|
assert session_manager._current_session.get() is None
|
|
|
|
injected = SessionState(
|
|
browser=MagicMock(),
|
|
context=MCPBrowserContext(mode="cloud_session", session_id="pbs_injected"),
|
|
)
|
|
|
|
async with scoped_session(injected):
|
|
inside = session_manager.get_current_session()
|
|
assert inside is injected
|
|
assert inside.context.session_id == "pbs_injected"
|
|
|
|
after = session_manager._current_session.get()
|
|
# Should be restored to None (the value before scoped_session set it)
|
|
assert after is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_scoped_session_does_not_touch_global() -> None:
|
|
"""scoped_session must NOT mutate _global_session."""
|
|
session_manager._global_session = None
|
|
|
|
injected = SessionState(
|
|
browser=MagicMock(),
|
|
context=MCPBrowserContext(mode="cloud_session", session_id="pbs_test"),
|
|
)
|
|
|
|
async with scoped_session(injected):
|
|
pass
|
|
|
|
assert session_manager._global_session is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_scoped_session_concurrent_isolation() -> None:
|
|
"""Two concurrent scoped_session calls don't interfere with each other."""
|
|
results: dict[str, str | None] = {}
|
|
both_entered = asyncio.Event()
|
|
entered_count = 0
|
|
|
|
async def worker(session_id: str) -> None:
|
|
nonlocal entered_count
|
|
state = SessionState(
|
|
browser=MagicMock(),
|
|
context=MCPBrowserContext(mode="cloud_session", session_id=session_id),
|
|
)
|
|
async with scoped_session(state):
|
|
entered_count += 1
|
|
if entered_count == 2:
|
|
both_entered.set()
|
|
# Wait until BOTH workers are inside their scope before reading —
|
|
# this guarantees ContextVar isolation is the only thing separating them.
|
|
await both_entered.wait()
|
|
current = session_manager.get_current_session()
|
|
results[session_id] = current.context.session_id if current.context else None
|
|
|
|
await asyncio.gather(worker("pbs_a"), worker("pbs_b"))
|
|
|
|
assert results["pbs_a"] == "pbs_a"
|
|
assert results["pbs_b"] == "pbs_b"
|
|
assert session_manager._global_session is None
|
|
|
|
|
|
def test_mcp_to_copilot_success() -> None:
|
|
mcp_result = {
|
|
"ok": True,
|
|
"action": "skyvern_navigate",
|
|
"browser_context": {"mode": "cloud_session", "session_id": "pbs_1"},
|
|
"data": {"url": "https://example.com", "title": "Example"},
|
|
"timing_ms": {"total": 500},
|
|
"artifacts": [],
|
|
}
|
|
result = mcp_to_copilot(mcp_result)
|
|
assert result["ok"] is True
|
|
assert result["data"]["url"] == "https://example.com"
|
|
assert "action" not in result
|
|
assert "browser_context" not in result
|
|
assert "timing_ms" not in result
|
|
assert "artifacts" not in result
|
|
|
|
|
|
def test_mcp_to_copilot_error() -> None:
|
|
mcp_result = {
|
|
"ok": False,
|
|
"error": {"code": "NO_ACTIVE_BROWSER", "message": "No browser", "hint": "Create one"},
|
|
}
|
|
result = mcp_to_copilot(mcp_result)
|
|
assert result["ok"] is False
|
|
assert "No browser" in result["error"]
|
|
assert "Create one" in result["error"]
|
|
|
|
|
|
class TestMcpBrowserContextBridge:
|
|
"""Bridge-specific behavior of mcp_browser_context."""
|
|
|
|
def _install_happy_path_mocks(
|
|
self, monkeypatch: pytest.MonkeyPatch
|
|
) -> tuple[MagicMock, MagicMock, MagicMock, MagicMock, list[Any]]:
|
|
import skyvern.forge.sdk.copilot.runtime as runtime
|
|
|
|
browser_state = MagicMock()
|
|
browser_state.browser_context = MagicMock()
|
|
manager = MagicMock()
|
|
manager.get_browser_state = AsyncMock(return_value=browser_state)
|
|
monkeypatch.setattr(runtime.app, "PERSISTENT_SESSIONS_MANAGER", manager)
|
|
|
|
monkeypatch.setattr(runtime, "get_skyvern", lambda: MagicMock())
|
|
monkeypatch.setattr(runtime, "SkyvernBrowser", lambda *a, **kw: MagicMock())
|
|
monkeypatch.setattr(runtime, "get_active_api_key", lambda: "sk-test-key")
|
|
monkeypatch.setattr(runtime, "hash_api_key_for_cache", lambda k: "hash_" + k)
|
|
|
|
override_token = object()
|
|
override_calls: list[Any] = []
|
|
monkeypatch.setattr(
|
|
runtime, "set_api_key_override", lambda k: (override_calls.append(("set", k)), override_token)[1]
|
|
)
|
|
monkeypatch.setattr(runtime, "reset_api_key_override", lambda t: override_calls.append(("reset", t)))
|
|
|
|
register_mock = MagicMock()
|
|
unregister_mock = MagicMock()
|
|
monkeypatch.setattr(runtime, "register_copilot_session", register_mock)
|
|
monkeypatch.setattr(runtime, "unregister_copilot_session", unregister_mock)
|
|
|
|
return manager, register_mock, unregister_mock, browser_state, override_calls
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_happy_path_registers_session_and_balances_unregister(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
from skyvern.forge.sdk.copilot.runtime import mcp_browser_context
|
|
|
|
_, register_mock, unregister_mock, _, override_calls = self._install_happy_path_mocks(monkeypatch)
|
|
|
|
ctx = _make_ctx()
|
|
async with mcp_browser_context(ctx):
|
|
# Inside the context, the bridge has registered a SessionState whose
|
|
# session_id matches the agent context — this is the public contract
|
|
# that resolve_browser(session_id=...) relies on.
|
|
args = register_mock.call_args.args
|
|
assert args[0] == ctx.browser_session_id
|
|
registered_state = args[1]
|
|
assert isinstance(registered_state, SessionState)
|
|
assert registered_state.context.session_id == ctx.browser_session_id
|
|
|
|
assert register_mock.call_count == 1
|
|
assert unregister_mock.call_count == 1
|
|
unregister_mock.assert_called_with(ctx.browser_session_id)
|
|
# Override installed then reset.
|
|
assert [c[0] for c in override_calls] == ["set", "reset"]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_missing_browser_context_raises_without_leaking_session_id(
|
|
self, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
import skyvern.forge.sdk.copilot.runtime as runtime
|
|
from skyvern.forge.sdk.copilot.runtime import mcp_browser_context
|
|
|
|
manager = MagicMock()
|
|
manager.get_browser_state = AsyncMock(return_value=None)
|
|
monkeypatch.setattr(runtime.app, "PERSISTENT_SESSIONS_MANAGER", manager)
|
|
|
|
ctx = _make_ctx()
|
|
with pytest.raises(RuntimeError, match="No browser context for copilot session") as exc_info:
|
|
async with mcp_browser_context(ctx):
|
|
pytest.fail("should not enter body")
|
|
|
|
# Session id must not leak into the user/LLM-visible exception message.
|
|
assert ctx.browser_session_id not in str(exc_info.value)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_exception_during_yield_still_tears_down(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
from skyvern.forge.sdk.copilot.runtime import mcp_browser_context
|
|
|
|
_, register_mock, unregister_mock, _, override_calls = self._install_happy_path_mocks(monkeypatch)
|
|
|
|
class Boom(RuntimeError):
|
|
pass
|
|
|
|
ctx = _make_ctx()
|
|
with pytest.raises(Boom):
|
|
async with mcp_browser_context(ctx):
|
|
raise Boom("caller raised inside context")
|
|
|
|
# Both teardown paths must fire even when the caller raises.
|
|
assert register_mock.call_count == 1
|
|
assert unregister_mock.call_count == 1
|
|
assert [c[0] for c in override_calls] == ["set", "reset"]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_setup_phase_failure_still_resets_api_key_override(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""If get_skyvern raises AFTER set_api_key_override, the override must
|
|
still be reset so the request-scoped API key does not leak across
|
|
requests."""
|
|
import skyvern.forge.sdk.copilot.runtime as runtime
|
|
from skyvern.forge.sdk.copilot.runtime import mcp_browser_context
|
|
|
|
_, register_mock, unregister_mock, _, override_calls = self._install_happy_path_mocks(monkeypatch)
|
|
|
|
def _raising_get_skyvern() -> Any:
|
|
raise RuntimeError("skyvern client unavailable")
|
|
|
|
monkeypatch.setattr(runtime, "get_skyvern", _raising_get_skyvern)
|
|
|
|
ctx = _make_ctx()
|
|
with pytest.raises(RuntimeError, match="skyvern client unavailable"):
|
|
async with mcp_browser_context(ctx):
|
|
pytest.fail("should not enter body")
|
|
|
|
# Registration never happened because setup failed before register_copilot_session.
|
|
assert register_mock.call_count == 0
|
|
assert unregister_mock.call_count == 0
|
|
# But the override must have been set AND reset.
|
|
assert [c[0] for c in override_calls] == ["set", "reset"]
|
|
|
|
|
|
class TestScreenshotAdapter:
|
|
@pytest.mark.asyncio
|
|
async def test_screenshot_post_hook_reshapes_data_with_url_and_title(self) -> None:
|
|
from skyvern.forge.sdk.copilot.tools import _screenshot_post_hook
|
|
|
|
ctx = _make_ctx()
|
|
raw = {"browser_context": {"url": "https://example.com", "title": "Example"}}
|
|
result = {
|
|
"ok": True,
|
|
"data": {"data": "iVBOR...", "mime": "image/png", "bytes": 1234},
|
|
}
|
|
|
|
adapted = await _screenshot_post_hook(result, raw, ctx)
|
|
|
|
assert adapted["data"]["screenshot_base64"] == "iVBOR..."
|
|
assert adapted["data"]["url"] == "https://example.com"
|
|
assert adapted["data"]["title"] == "Example"
|
|
|
|
|
|
class TestNavigateAdapter:
|
|
@pytest.mark.asyncio
|
|
async def test_navigate_post_hook_lifts_url_and_adds_next_step(self) -> None:
|
|
from skyvern.forge.sdk.copilot.tools import _navigate_post_hook
|
|
|
|
ctx = _make_ctx()
|
|
raw = {"browser_context": {"url": "https://example.com", "title": "Example"}}
|
|
result = {"ok": True, "data": {"url": "https://example.com", "title": "Example"}}
|
|
|
|
adapted = await _navigate_post_hook(result, raw, ctx)
|
|
|
|
assert adapted["ok"] is True
|
|
assert adapted["url"] == "https://example.com"
|
|
assert "next_step" in adapted
|
|
assert "data" not in adapted
|
|
|
|
|
|
class TestClickAdapter:
|
|
@pytest.mark.asyncio
|
|
async def test_click_post_hook_reshapes_data_with_url_and_title(self) -> None:
|
|
from skyvern.forge.sdk.copilot.tools import _click_post_hook
|
|
|
|
ctx = _make_ctx()
|
|
raw = {"browser_context": {"url": "https://ex.com", "title": "Page"}}
|
|
result = {
|
|
"ok": True,
|
|
"data": {"selector": "#btn", "intent": None, "sdk_equivalent": "..."},
|
|
}
|
|
|
|
adapted = await _click_post_hook(result, raw, ctx)
|
|
|
|
assert adapted["data"]["selector"] == "#btn"
|
|
assert adapted["data"]["url"] == "https://ex.com"
|
|
assert adapted["data"]["title"] == "Page"
|
|
assert "sdk_equivalent" not in adapted["data"]
|
|
|
|
|
|
class TestTypeTextAdapter:
|
|
@pytest.mark.asyncio
|
|
async def test_type_text_post_hook_renames_text_length_to_typed_length(self) -> None:
|
|
from skyvern.forge.sdk.copilot.tools import _type_text_post_hook
|
|
|
|
ctx = _make_ctx()
|
|
raw = {"browser_context": {"url": "https://ex.com"}}
|
|
result = {
|
|
"ok": True,
|
|
"data": {"selector": "#email", "text_length": 15, "sdk_equivalent": "..."},
|
|
}
|
|
|
|
adapted = await _type_text_post_hook(result, raw, ctx)
|
|
|
|
assert adapted["data"]["selector"] == "#email"
|
|
assert adapted["data"]["typed_length"] == 15
|
|
assert adapted["data"]["url"] == "https://ex.com"
|
|
assert "text_length" not in adapted["data"]
|
|
|
|
|
|
class TestEvaluateAdapter:
|
|
def test_evaluate_copilot_contract(self) -> None:
|
|
mcp_result = {
|
|
"ok": True,
|
|
"action": "skyvern_evaluate",
|
|
"data": {
|
|
"result": {"title": "Test"},
|
|
"sdk_equivalent": "await page.evaluate(...)",
|
|
},
|
|
"browser_context": {"mode": "cloud_session"},
|
|
}
|
|
result = mcp_to_copilot(mcp_result)
|
|
assert result["data"]["result"] == {"title": "Test"}
|
|
|
|
|
|
class TestUpdateWorkflowDirect:
|
|
@pytest.mark.asyncio
|
|
async def test_calls_internal_with_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""update_workflow uses direct path even when api_key is set."""
|
|
from skyvern.forge.sdk.copilot.tools import _update_workflow
|
|
|
|
ctx = _make_ctx(api_key="sk-test-key", workflow_permanent_id="wpid_abc123")
|
|
|
|
mock_workflow = MagicMock()
|
|
mock_workflow.title = "Test"
|
|
mock_workflow.description = ""
|
|
mock_workflow.workflow_definition = MagicMock()
|
|
mock_workflow.workflow_definition.blocks = [MagicMock(), MagicMock()]
|
|
|
|
monkeypatch.setattr(
|
|
"skyvern.forge.sdk.copilot.tools._process_workflow_yaml",
|
|
lambda **kwargs: mock_workflow,
|
|
)
|
|
|
|
mock_wf_service = MagicMock()
|
|
mock_wf_service.update_workflow_definition = AsyncMock()
|
|
mock_wf_service.get_workflow = AsyncMock(return_value=None)
|
|
monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.app.WORKFLOW_SERVICE", mock_wf_service)
|
|
|
|
yaml_str = "title: Test\nworkflow_definition:\n blocks: []"
|
|
result = await _update_workflow({"workflow_yaml": yaml_str}, ctx)
|
|
|
|
assert result["ok"] is True
|
|
assert result["data"]["block_count"] == 2
|
|
assert result["_workflow"] is mock_workflow
|
|
assert ctx.workflow_yaml == yaml_str
|
|
mock_wf_service.update_workflow_definition.assert_awaited_once()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_calls_internal_without_api_key(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""update_workflow uses direct path when api_key is None."""
|
|
from skyvern.forge.sdk.copilot.tools import _update_workflow
|
|
|
|
ctx = _make_ctx(api_key=None)
|
|
|
|
mock_workflow = MagicMock()
|
|
mock_workflow.title = "Test"
|
|
mock_workflow.description = ""
|
|
mock_workflow.workflow_definition = MagicMock()
|
|
mock_workflow.workflow_definition.blocks = []
|
|
|
|
monkeypatch.setattr(
|
|
"skyvern.forge.sdk.copilot.tools._process_workflow_yaml",
|
|
lambda **kwargs: mock_workflow,
|
|
)
|
|
|
|
mock_wf_service = MagicMock()
|
|
mock_wf_service.update_workflow_definition = AsyncMock()
|
|
mock_wf_service.get_workflow = AsyncMock(return_value=None)
|
|
monkeypatch.setattr("skyvern.forge.sdk.copilot.tools.app.WORKFLOW_SERVICE", mock_wf_service)
|
|
|
|
result = await _update_workflow({"workflow_yaml": "title: Test"}, ctx)
|
|
|
|
assert result["ok"] is True
|
|
mock_wf_service.update_workflow_definition.assert_awaited_once()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_yaml_parse_error_returns_failure(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""YAML parse errors return ok: false."""
|
|
import yaml as _yaml
|
|
|
|
from skyvern.forge.sdk.copilot.tools import _update_workflow
|
|
|
|
ctx = _make_ctx(api_key="sk-test-key")
|
|
|
|
def raise_yaml_error(**kwargs: Any) -> None:
|
|
raise _yaml.YAMLError("bad yaml")
|
|
|
|
monkeypatch.setattr(
|
|
"skyvern.forge.sdk.copilot.tools._process_workflow_yaml",
|
|
raise_yaml_error,
|
|
)
|
|
|
|
result = await _update_workflow({"workflow_yaml": "bad: yaml: {"}, ctx)
|
|
|
|
assert result["ok"] is False
|
|
assert "Workflow validation failed" in result["error"]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_ctx_workflow_yaml_not_updated_on_failure(self, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""ctx.workflow_yaml should NOT be updated when processing fails."""
|
|
from pydantic import ValidationError as _ValidationError
|
|
|
|
from skyvern.forge.sdk.copilot.tools import _update_workflow
|
|
|
|
ctx = _make_ctx(
|
|
api_key="sk-test-key",
|
|
workflow_yaml="original yaml",
|
|
)
|
|
|
|
def raise_validation_error(**kwargs: Any) -> None:
|
|
raise _ValidationError.from_exception_data(
|
|
title="WorkflowCreateYAMLRequest",
|
|
line_errors=[],
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
"skyvern.forge.sdk.copilot.tools._process_workflow_yaml",
|
|
raise_validation_error,
|
|
)
|
|
|
|
await _update_workflow({"workflow_yaml": "new broken yaml"}, ctx)
|
|
|
|
assert ctx.workflow_yaml == "original yaml"
|
|
|
|
|
|
class TestWorkflowUpdatePersistence:
|
|
def test_record_marks_persisted_even_when_workflow_has_zero_blocks(self) -> None:
|
|
from skyvern.forge.sdk.copilot.tools import _record_workflow_update_result
|
|
|
|
ctx = MagicMock()
|
|
ctx.workflow_yaml = "title: Empty workflow"
|
|
ctx.workflow_persisted = False
|
|
|
|
workflow = MagicMock()
|
|
workflow.workflow_definition = MagicMock()
|
|
workflow.workflow_definition.blocks = []
|
|
|
|
_record_workflow_update_result(
|
|
ctx,
|
|
{
|
|
"ok": True,
|
|
"_workflow": workflow,
|
|
"data": {"block_count": 0},
|
|
},
|
|
)
|
|
|
|
assert ctx.last_workflow is workflow
|
|
assert ctx.last_workflow_yaml == "title: Empty workflow"
|
|
assert ctx.workflow_persisted is True
|