Add QA discoverability to MCP instructions and localhost guard for cloud browsers (#4984)

skyvern/cli/mcp_tools/__init__.py

@@ -70,10 +70,17 @@ When the task DOES need a real browser, prefer Skyvern over WebFetch and Playwri
 on sites with CAPTCHAs, pop-ups, login walls, or dynamic content. Playwright primitives require \
 element refs from browser_snapshot; Skyvern accepts natural language intent directly.
 
+## QA Testing
+
+To QA test frontend changes in a real browser, use the `qa_test` prompt or ask the user \
+"Would you like me to test your recent code changes?" Skyvern can read a git diff, generate \
+targeted test cases, open a browser against the dev server, and report pass/fail with screenshots.
+
 ## Quick Start — First Tool to Call
 
 | Task type | First Skyvern tool | Then |
 |-----------|-------------------|------|
+| QA test frontend changes | qa_test prompt | Generates and runs tests from git diff |
 | Visit / explore a website | skyvern_browser_session_create → skyvern_navigate | skyvern_screenshot to see it |
 | Extract data from a page | skyvern_browser_session_create → skyvern_navigate | skyvern_extract with a prompt |
 | Click / fill / interact | skyvern_browser_session_create → skyvern_navigate | skyvern_act or skyvern_click |
@@ -85,6 +92,7 @@ element refs from browser_snapshot; Skyvern accepts natural language intent dire
 
 | User says | Use | Why |
 |-----------|-----|-----|
+| "QA my changes" / "Test my frontend" | qa_test prompt | Reads git diff, generates + runs browser tests |
 | "Go to [url]" / "Visit [site]" | skyvern_navigate | Opens page in real browser |
 | "What's on this page?" | skyvern_screenshot | Visual understanding |
 | "Get / extract / pull data from [site]" | skyvern_extract | AI-powered structured extraction |

skyvern/cli/mcp_tools/_localhost.py

@@ -0,0 +1,25 @@
+"""Localhost URL detection for cloud browser sessions."""
+
+from __future__ import annotations
+
+from urllib.parse import urlparse
+
+_LOCALHOST_HOSTNAMES = frozenset(
+    {
+        "localhost",
+        "127.0.0.1",
+        "0.0.0.0",  # noqa: S104 — detection, not binding
+        "::1",
+        "[::1]",
+    }
+)
+
+
+def is_localhost_url(url: str) -> bool:
+    """Return True if *url* points to a loopback address."""
+    try:
+        parsed = urlparse(url)
+        hostname = (parsed.hostname or "").lower()
+        return hostname in _LOCALHOST_HOSTNAMES
+    except Exception:
+        return False

skyvern/cli/mcp_tools/browser.py

@@ -31,6 +31,7 @@ from ._common import (
     make_result,
     save_artifact,
 )
+from ._localhost import is_localhost_url
 from ._session import BrowserNotAvailableError, get_page, no_browser_error
 
 LOG = logging.getLogger(__name__)
@@ -72,6 +73,20 @@ async def skyvern_navigate(
     except BrowserNotAvailableError:
         return make_result("skyvern_navigate", ok=False, error=no_browser_error())
 
+    if ctx.mode == "cloud_session" and is_localhost_url(url):
+        return make_result(
+            "skyvern_navigate",
+            ok=False,
+            browser_context=ctx,
+            error=make_error(
+                ErrorCode.INVALID_INPUT,
+                "Cloud browsers cannot reach localhost URLs",
+                "Run `pip install skyvern && skyvern browser serve --tunnel` to bridge "
+                "your local dev server to a cloud browser via ngrok. "
+                "Or use `local=true` in skyvern_browser_session_create for a local browser.",
+            ),
+        )
+
     with Timer() as timer:
         try:
             result = await do_navigate(page, url, timeout=timeout, wait_until=wait_until)
@@ -1110,6 +1125,20 @@ async def skyvern_run_task(
     except BrowserNotAvailableError:
         return make_result("skyvern_run_task", ok=False, error=no_browser_error())
 
+    if url and ctx.mode == "cloud_session" and is_localhost_url(url):
+        return make_result(
+            "skyvern_run_task",
+            ok=False,
+            browser_context=ctx,
+            error=make_error(
+                ErrorCode.INVALID_INPUT,
+                "Cloud browsers cannot reach localhost URLs",
+                "Run `pip install skyvern && skyvern browser serve --tunnel` to bridge "
+                "your local dev server to a cloud browser via ngrok. "
+                "Or use `local=true` in skyvern_browser_session_create for a local browser.",
+            ),
+        )
+
     parsed_schema: dict[str, Any] | str | None = None
     if data_extraction_schema is not None:
         try:

tests/unit/test_mcp_localhost_guard.py

@@ -0,0 +1,178 @@
+"""Tests for localhost URL detection and cloud browser guard."""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock
+
+import pytest
+
+from skyvern.cli.core.result import BrowserContext
+from skyvern.cli.mcp_tools import browser as mcp_browser
+from skyvern.cli.mcp_tools._localhost import is_localhost_url
+
+# ---------------------------------------------------------------------------
+# is_localhost_url unit tests
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "http://localhost:3000",
+        "http://localhost:5173/some/path",
+        "https://localhost:8080",
+        "http://localhost",
+        "http://127.0.0.1:8000",
+        "http://127.0.0.1:8000/api/v1/tasks",
+        "https://127.0.0.1",
+        "http://0.0.0.0:3000",
+        "http://[::1]:3000",
+    ],
+)
+def test_is_localhost_url_detects_localhost(url: str) -> None:
+    assert is_localhost_url(url) is True
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "https://example.com",
+        "https://app.skyvern.com",
+        "http://my-localhost-app.com",
+        "https://api.skyvern.com/mcp/",
+        "http://192.168.1.1:3000",
+        "https://10.0.0.1:8080",
+    ],
+)
+def test_is_localhost_url_allows_non_localhost(url: str) -> None:
+    assert is_localhost_url(url) is False
+
+
+def test_is_localhost_url_handles_garbage_input() -> None:
+    assert is_localhost_url("") is False
+    assert is_localhost_url("not a url") is False
+
+
+# ---------------------------------------------------------------------------
+# skyvern_navigate cloud + localhost guard
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_navigate_rejects_localhost_on_cloud_session(monkeypatch: pytest.MonkeyPatch) -> None:
+    page = object()
+    ctx = BrowserContext(mode="cloud_session", session_id="pbs_test")
+    monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+
+    result = await mcp_browser.skyvern_navigate(url="http://localhost:3000")
+
+    assert result["ok"] is False
+    assert result["error"]["code"] == mcp_browser.ErrorCode.INVALID_INPUT
+    assert "localhost" in result["error"]["message"].lower()
+    assert "skyvern browser serve --tunnel" in result["error"]["hint"]
+
+
+@pytest.mark.asyncio
+async def test_navigate_rejects_127_0_0_1_on_cloud_session(monkeypatch: pytest.MonkeyPatch) -> None:
+    page = object()
+    ctx = BrowserContext(mode="cloud_session", session_id="pbs_test")
+    monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+
+    result = await mcp_browser.skyvern_navigate(url="http://127.0.0.1:5173/dashboard")
+
+    assert result["ok"] is False
+    assert result["error"]["code"] == mcp_browser.ErrorCode.INVALID_INPUT
+    assert "localhost" in result["error"]["message"].lower()
+
+
+@pytest.mark.asyncio
+async def test_navigate_allows_localhost_on_local_session(monkeypatch: pytest.MonkeyPatch) -> None:
+    page = AsyncMock()
+    ctx = BrowserContext(mode="local")
+    monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+    monkeypatch.setattr(
+        mcp_browser,
+        "do_navigate",
+        AsyncMock(return_value=AsyncMock(url="http://localhost:3000", title="App")),
+    )
+
+    result = await mcp_browser.skyvern_navigate(url="http://localhost:3000")
+
+    assert result["ok"] is True
+
+
+@pytest.mark.asyncio
+async def test_navigate_allows_localhost_on_cdp_session(monkeypatch: pytest.MonkeyPatch) -> None:
+    page = AsyncMock()
+    ctx = BrowserContext(mode="cdp", cdp_url="ws://localhost:9222")
+    monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+    monkeypatch.setattr(
+        mcp_browser,
+        "do_navigate",
+        AsyncMock(return_value=AsyncMock(url="http://localhost:3000", title="App")),
+    )
+
+    result = await mcp_browser.skyvern_navigate(url="http://localhost:3000")
+
+    assert result["ok"] is True
+
+
+@pytest.mark.asyncio
+async def test_navigate_allows_public_url_on_cloud_session(monkeypatch: pytest.MonkeyPatch) -> None:
+    page = AsyncMock()
+    ctx = BrowserContext(mode="cloud_session", session_id="pbs_test")
+    monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+    monkeypatch.setattr(
+        mcp_browser,
+        "do_navigate",
+        AsyncMock(return_value=AsyncMock(url="https://example.com", title="Example")),
+    )
+
+    result = await mcp_browser.skyvern_navigate(url="https://example.com")
+
+    assert result["ok"] is True
+
+
+# ---------------------------------------------------------------------------
+# skyvern_run_task cloud + localhost guard
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_run_task_rejects_localhost_on_cloud_session(monkeypatch: pytest.MonkeyPatch) -> None:
+    page = object()
+    ctx = BrowserContext(mode="cloud_session", session_id="pbs_test")
+    monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+
+    result = await mcp_browser.skyvern_run_task(
+        prompt="Extract the page title",
+        url="http://localhost:5173",
+    )
+
+    assert result["ok"] is False
+    assert result["error"]["code"] == mcp_browser.ErrorCode.INVALID_INPUT
+    assert "localhost" in result["error"]["message"].lower()
+    assert "skyvern browser serve --tunnel" in result["error"]["hint"]
+
+
+@pytest.mark.asyncio
+async def test_run_task_allows_no_url(monkeypatch: pytest.MonkeyPatch) -> None:
+    """run_task with url=None should not trigger the localhost guard."""
+    page = AsyncMock()
+    page.agent = AsyncMock()
+    page.agent.run_task = AsyncMock(
+        return_value=AsyncMock(
+            run_id="r_1",
+            status="completed",
+            output=None,
+            failure_reason=None,
+            recording_url=None,
+            app_url=None,
+        )
+    )
+    ctx = BrowserContext(mode="cloud_session", session_id="pbs_test")
+    monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+
+    result = await mcp_browser.skyvern_run_task(prompt="Do something on current page")
+
+    assert result["ok"] is True