From 05481c222aa3d8da936fd2a8780597332898456d Mon Sep 17 00:00:00 2001
From: Alessandro <155005371+3clyp50@users.noreply.github.com>
Date: Fri, 3 Jul 2026 01:07:49 +0200
Subject: [PATCH 01/43] Fix browser URL intent routing
Limit the Browser URL-intent handler to web URL schemes so custom Agent Zero schemes fall through to their owning surfaces.
Document the scheme-ownership contract and cover the a0-editor misroute regression in the desktop/editor routing test.
---
plugins/_browser/AGENTS.md | 1 +
plugins/_browser/webui/browser-store.js | 13 +++++++++++++
tests/test_office_canvas_setup.py | 6 ++++++
3 files changed, 20 insertions(+)
diff --git a/plugins/_browser/AGENTS.md b/plugins/_browser/AGENTS.md
index 70211d3a0..15a7ac85c 100644
--- a/plugins/_browser/AGENTS.md
+++ b/plugins/_browser/AGENTS.md
@@ -20,6 +20,7 @@
- Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation.
- Default the visible WebUI Browser to live CDP screencast for responsiveness. Keep lightweight CDP/DOM state snapshots as the fallback transport.
- Keep narrow WebUI Browser controls usable by grouping navigation with Annotate/settings above a full-width address bar.
+- Browser URL-intent handling must only claim web URL schemes and leave custom Agent Zero schemes to their owning surfaces.
- Prefer DOM/CDP browser actions with refs, selectors, frame-chain refs, and screenshots over viewport coordinate input. Coordinates remain a visual fallback.
- Do not hardcode user-specific browser paths or secrets.
diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.js
index c278d63f0..6e70a1891 100644
--- a/plugins/_browser/webui/browser-store.js
+++ b/plugins/_browser/webui/browser-store.js
@@ -2846,8 +2846,21 @@ const model = {
export const store = createStore("browserPage", model);
+const WEB_INTENT_SCHEMES = new Set(["http", "https", "file", "about"]);
+
+function isWebUrlIntent(url = "") {
+ const value = String(url || "").trim();
+ if (!value) return true;
+ const scheme = /^([a-z][a-z0-9+.-]*):/i.exec(value);
+ if (!scheme) return true;
+ return WEB_INTENT_SCHEMES.has(scheme[1].toLowerCase());
+}
+
registerUrlHandler(async (intent = {}) => {
const url = String(intent.url || "").trim();
+ // Custom schemes such as a0-editor: belong to other surfaces; claiming them
+ // here would navigate the browser to an unloadable URL.
+ if (!isWebUrlIntent(url)) return false;
const payload = { url, source: intent.source || "surface-url-intent" };
await openLatestSurface("browser", payload);
return await store.openUrlIntent(url, { source: payload.source });
diff --git a/tests/test_office_canvas_setup.py b/tests/test_office_canvas_setup.py
index a7759a1dc..404e09c5e 100644
--- a/tests/test_office_canvas_setup.py
+++ b/tests/test_office_canvas_setup.py
@@ -592,6 +592,7 @@ def test_desktop_text_open_with_routes_to_editor_surface():
desktop_session = read("plugins", "_desktop", "helpers", "desktop_session.py")
desktop_store = read("plugins", "_desktop", "webui", "desktop-store.js")
editor_store = read("plugins", "_editor", "webui", "editor-store.js")
+ browser_store = read("plugins", "_browser", "webui", "browser-store.js")
assert 'EDITOR_HANDLER_DESKTOP_ID = "agent-zero-editor.desktop"' in desktop_session
assert "def _write_editor_bridge_script" in desktop_session
@@ -607,6 +608,11 @@ def test_desktop_text_open_with_routes_to_editor_surface():
assert "handleEditorUrlIntent" in editor_store
assert 'openLatestSurface("editor"' in editor_store
+ # The browser surface must not claim a0-editor: intents, otherwise the
+ # editor "Open With" handler lands on an unloadable about:blank page.
+ assert "function isWebUrlIntent" in browser_store
+ assert "if (!isWebUrlIntent(url)) return false;" in browser_store
+
def test_office_and_desktop_skills_are_rehomed_and_renamed():
office_skills = PROJECT_ROOT / "plugins" / "_office" / "skills"
From 306012eb00dd90108de7a3ceeb87e7c776b63933 Mon Sep 17 00:00:00 2001
From: Alessandro <155005371+3clyp50@users.noreply.github.com>
Date: Fri, 3 Jul 2026 01:08:02 +0200
Subject: [PATCH 02/43] Default desktop text files to Writer
Set LibreOffice Writer as the Desktop MIME default for Markdown and plain text while keeping Agent Zero Editor as the secondary Open With association.
Update the Desktop DOX contract and extend the routing regression test to pin the Writer-first association order.
---
plugins/_desktop/AGENTS.md | 2 +-
plugins/_desktop/helpers/desktop_session.py | 11 ++++++++---
tests/test_office_canvas_setup.py | 6 ++++++
3 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/plugins/_desktop/AGENTS.md b/plugins/_desktop/AGENTS.md
index 5fc6f449c..f3dc67d12 100644
--- a/plugins/_desktop/AGENTS.md
+++ b/plugins/_desktop/AGENTS.md
@@ -17,7 +17,7 @@
- Keep desktop state injected into prompts accurate and bounded.
- Do not expose desktop routes without the expected auth protections.
- Keep Desktop host visibility tied to an attached modal or canvas host; modal cleanup may preserve the iframe in keepalive, but must not leave stale modal mode behind.
-- Keep Markdown and plain text file open-with handling routed to the Editor surface through the desktop intent bridge; Desktop owns the Xfce launcher/MIME setup, while Editor owns `.md` and `.txt` editing.
+- Keep LibreOffice Writer as the default handler for Markdown and plain text files; keep Agent Zero Editor available as a secondary Open With target through the desktop intent bridge.
## Work Guidance
diff --git a/plugins/_desktop/helpers/desktop_session.py b/plugins/_desktop/helpers/desktop_session.py
index efacb2c41..4acdab0cc 100644
--- a/plugins/_desktop/helpers/desktop_session.py
+++ b/plugins/_desktop/helpers/desktop_session.py
@@ -79,6 +79,7 @@ URL_INTENT_MAX_ITEMS = 50
URL_INTENT_MAX_LENGTH = 8192
URL_HANDLER_DESKTOP_ID = "agent-zero-browser.desktop"
EDITOR_HANDLER_DESKTOP_ID = "agent-zero-editor.desktop"
+WRITER_HANDLER_DESKTOP_ID = "libreoffice-writer.desktop"
SHUTDOWN_HANDLER_DESKTOP_ID = "agent-zero-shutdown.desktop"
SHUTDOWN_PANEL_LAUNCHER_ID = SHUTDOWN_HANDLER_DESKTOP_ID
SHUTDOWN_CONFIRM_SECONDS = 8
@@ -2197,15 +2198,19 @@ def _editor_text_handler_mime_types() -> tuple[str, ...]:
def _write_mimeapps_defaults(path: Path, url_desktop_id: str, editor_desktop_id: str) -> None:
url_associations = ";".join([url_desktop_id, ""])
- editor_associations = ";".join([editor_desktop_id, ""])
+ # LibreOffice Writer stays the default so double-clicks keep the user (and
+ # agents) inside the Desktop; the Agent Zero Editor remains an "Open With"
+ # option. When Writer's desktop entry is missing, xdg falls back through
+ # the added associations to the editor.
+ text_associations = ";".join([WRITER_HANDLER_DESKTOP_ID, editor_desktop_id, ""])
lines = [
"[Default Applications]",
*(f"{mime_type}={url_desktop_id}" for mime_type in _url_handler_mime_types()),
- *(f"{mime_type}={editor_desktop_id}" for mime_type in _editor_text_handler_mime_types()),
+ *(f"{mime_type}={WRITER_HANDLER_DESKTOP_ID}" for mime_type in _editor_text_handler_mime_types()),
"",
"[Added Associations]",
*(f"{mime_type}={url_associations}" for mime_type in _url_handler_mime_types()),
- *(f"{mime_type}={editor_associations}" for mime_type in _editor_text_handler_mime_types()),
+ *(f"{mime_type}={text_associations}" for mime_type in _editor_text_handler_mime_types()),
"",
]
path.parent.mkdir(parents=True, exist_ok=True)
diff --git a/tests/test_office_canvas_setup.py b/tests/test_office_canvas_setup.py
index 404e09c5e..6d29c36d0 100644
--- a/tests/test_office_canvas_setup.py
+++ b/tests/test_office_canvas_setup.py
@@ -608,6 +608,12 @@ def test_desktop_text_open_with_routes_to_editor_surface():
assert "handleEditorUrlIntent" in editor_store
assert 'openLatestSurface("editor"' in editor_store
+ # Text files default to LibreOffice Writer inside the Desktop; the Agent
+ # Zero Editor stays available as a secondary "Open With" association.
+ assert 'WRITER_HANDLER_DESKTOP_ID = "libreoffice-writer.desktop"' in desktop_session
+ assert "{mime_type}={WRITER_HANDLER_DESKTOP_ID}" in desktop_session
+ assert "[WRITER_HANDLER_DESKTOP_ID, editor_desktop_id, " in desktop_session
+
# The browser surface must not claim a0-editor: intents, otherwise the
# editor "Open With" handler lands on an unloadable about:blank page.
assert "function isWebUrlIntent" in browser_store
From 686e039d1b779ecf864d9654bfe175c53d7d3096 Mon Sep 17 00:00:00 2001
From: Alessandro <155005371+3clyp50@users.noreply.github.com>
Date: Fri, 3 Jul 2026 01:25:56 +0200
Subject: [PATCH 03/43] Wait for Browser screencast frames
Replace the WebSocket viewer frame pump's idle poll with read_screencast_frame so live screencast delivery follows the producer cadence instead of a 50 ms sleep cycle.
Keep idle-page timeouts inside the stream loop to avoid screencast restart flashes, and update the Browser regression test to pin the blocking read path and timeout handling.
---
plugins/_browser/api/ws_browser.py | 11 +++++++----
tests/test_browser_agent_regressions.py | 5 ++++-
2 files changed, 11 insertions(+), 5 deletions(-)
diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.py
index 6e24b268c..ec5e137b3 100644
--- a/plugins/_browser/api/ws_browser.py
+++ b/plugins/_browser/api/ws_browser.py
@@ -11,7 +11,7 @@ from helpers.ws_manager import WsResult
from plugins._browser.helpers.runtime import get_runtime, list_runtime_sessions
-FRAME_IDLE_POLL_SECONDS = 0.05
+FRAME_READ_TIMEOUT_SECONDS = 0.5
FRAME_RETRY_DELAY_SECONDS = 0.5
FRAME_STATE_REFRESH_SECONDS = 0.75
SNAPSHOT_STATE_POLL_SECONDS = 0.75
@@ -445,11 +445,14 @@ class WsBrowser(WsHandler):
last_state_refresh = now
try:
- frame = await runtime.call("pop_screencast_frame", stream_id)
+ frame = await runtime.call(
+ "read_screencast_frame",
+ stream_id,
+ timeout=FRAME_READ_TIMEOUT_SECONDS,
+ )
except KeyError:
break
- if frame is None:
- await asyncio.sleep(FRAME_IDLE_POLL_SECONDS)
+ except TimeoutError:
continue
frame["context_id"] = context_id
diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py
index fff617ce5..eef2e141b 100644
--- a/tests/test_browser_agent_regressions.py
+++ b/tests/test_browser_agent_regressions.py
@@ -1162,7 +1162,10 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "initial_viewport = self._viewport_from_data(data)" in ws_browser
assert '"set_viewport"' in ws_browser
assert "start_screencast" in ws_browser
- assert "pop_screencast_frame" in ws_browser
+ assert "read_screencast_frame" in ws_browser
+ assert "FRAME_READ_TIMEOUT_SECONDS" in ws_browser
+ assert "FRAME_IDLE_POLL_SECONDS" not in ws_browser
+ assert "except TimeoutError:" in ws_browser
assert "stop_screencast" in ws_browser
assert "viewer_transport == VIEWER_TRANSPORT_SCREENCAST" in ws_browser
assert '"viewer_transport": viewer_transport' in ws_browser
From 91b3bc9613c7ea314623fc04c634929106d99415 Mon Sep 17 00:00:00 2001
From: Alessandro <155005371+3clyp50@users.noreply.github.com>
Date: Fri, 3 Jul 2026 01:49:07 +0200
Subject: [PATCH 04/43] Slim browser viewer frame transport
Negotiate binary and slim screencast frames so updated clients avoid base64-in-JSON and repeated viewer metadata on every frame.
Clamp screencast capture to the subscriber viewport and DPR, lower streaming JPEG quality while preserving snapshot quality, and teach the client to manage binary frame URLs safely.
Document the browser viewer performance plan and update regression coverage for the new transport contract.
---
docs/AGENTS.md | 1 +
plugins/_browser/AGENTS.md | 1 +
plugins/_browser/api/ws_browser.py | 244 +++++++++++++++++++-----
plugins/_browser/helpers/runtime.py | 10 +-
plugins/_browser/webui/browser-store.js | 205 +++++++++++++++-----
tests/test_browser_agent_regressions.py | 63 +++++-
6 files changed, 417 insertions(+), 107 deletions(-)
diff --git a/docs/AGENTS.md b/docs/AGENTS.md
index 3f3002b01..c81fb55bd 100644
--- a/docs/AGENTS.md
+++ b/docs/AGENTS.md
@@ -9,6 +9,7 @@
- `README.md`, `quickstart.md`, `guides/`, and `setup/` cover user-facing setup and workflows.
- `developer/` covers compact developer references and source handoffs.
+- `plans/` covers implementation plans, migration notes, and staged technical roadmaps.
- `res/` contains documentation images and other documentation assets.
## Local Contracts
diff --git a/plugins/_browser/AGENTS.md b/plugins/_browser/AGENTS.md
index 15a7ac85c..e3f2a137e 100644
--- a/plugins/_browser/AGENTS.md
+++ b/plugins/_browser/AGENTS.md
@@ -19,6 +19,7 @@
- Preserve Playwright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding.
- Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation.
- Default the visible WebUI Browser to live CDP screencast for responsiveness. Keep lightweight CDP/DOM state snapshots as the fallback transport.
+- Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback.
- Keep narrow WebUI Browser controls usable by grouping navigation with Annotate/settings above a full-width address bar.
- Browser URL-intent handling must only claim web URL schemes and leave custom Agent Zero schemes to their owning surfaces.
- Prefer DOM/CDP browser actions with refs, selectors, frame-chain refs, and screenshots over viewport coordinate input. Coordinates remain a visual fallback.
diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.py
index ec5e137b3..827e91df4 100644
--- a/plugins/_browser/api/ws_browser.py
+++ b/plugins/_browser/api/ws_browser.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import base64
import contextlib
import time
from typing import Any, ClassVar
@@ -15,7 +16,8 @@ FRAME_READ_TIMEOUT_SECONDS = 0.5
FRAME_RETRY_DELAY_SECONDS = 0.5
FRAME_STATE_REFRESH_SECONDS = 0.75
SNAPSHOT_STATE_POLL_SECONDS = 0.75
-SCREENCAST_QUALITY = 92
+SCREENCAST_STREAM_QUALITY = 80
+SCREENSHOT_QUALITY = 92
VIEWER_TRANSPORT_SCREENCAST = "screencast"
VIEWER_TRANSPORT_SNAPSHOT = "snapshot"
VIEWER_TRANSPORTS = {VIEWER_TRANSPORT_SCREENCAST, VIEWER_TRANSPORT_SNAPSHOT}
@@ -97,10 +99,21 @@ class WsBrowser(WsHandler):
existing.cancel()
viewer_id = str(data.get("viewer_id") or "")
viewer_transport = self._viewer_transport(data)
+ binary_frames = self._bool(data.get("binary_frames", data.get("binaryFrames")))
+ slim_frames = self._bool(data.get("slim_frames", data.get("slimFrames", binary_frames)))
+ capture_scale = self._capture_scale_from_data(data)
snapshot = None
if runtime:
if viewer_transport == VIEWER_TRANSPORT_SCREENCAST:
- stream_task = self._stream_frames(sid, context_id, active_id, viewer_id)
+ stream_task = self._stream_frames(
+ sid,
+ context_id,
+ active_id,
+ viewer_id,
+ binary_frames=binary_frames,
+ slim_frames=slim_frames,
+ capture_scale=capture_scale,
+ )
else:
stream_task = self._stream_state(sid, context_id, active_id, viewer_id)
self._streams[stream_key] = asyncio.create_task(stream_task)
@@ -115,6 +128,8 @@ class WsBrowser(WsHandler):
"all_browsers": True,
"viewer_id": viewer_id,
"viewer_transport": viewer_transport,
+ "binary_frames": binary_frames,
+ "slim_frames": slim_frames,
}
def _unsubscribe(self, data: dict[str, Any], sid: str) -> dict[str, Any] | WsResult:
@@ -157,9 +172,9 @@ class WsBrowser(WsHandler):
snapshot = None
if active_id:
try:
- quality = int(data.get("quality") or SCREENCAST_QUALITY)
+ quality = int(data.get("quality") or SCREENSHOT_QUALITY)
except (TypeError, ValueError):
- quality = SCREENCAST_QUALITY
+ quality = SCREENSHOT_QUALITY
with contextlib.suppress(Exception):
snapshot = await runtime.call(
"screenshot",
@@ -347,7 +362,7 @@ class WsBrowser(WsHandler):
if not browser_id:
return None
with contextlib.suppress(Exception):
- return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY)
+ return await runtime.call("screenshot", browser_id, quality=SCREENSHOT_QUALITY)
return None
async def _snapshot_for_browser(
@@ -358,7 +373,7 @@ class WsBrowser(WsHandler):
if not browser_id:
return None
with contextlib.suppress(Exception):
- return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY)
+ return await runtime.call("screenshot", browser_id, quality=SCREENSHOT_QUALITY)
return None
async def _all_browser_tabs(self) -> list[dict[str, Any]]:
@@ -377,6 +392,10 @@ class WsBrowser(WsHandler):
context_id: str,
browser_id: int | str | None,
viewer_id: str = "",
+ *,
+ binary_frames: bool = False,
+ slim_frames: bool = False,
+ capture_scale: float = 1.0,
) -> None:
runtime = None
stream_id = None
@@ -397,12 +416,14 @@ class WsBrowser(WsHandler):
browsers = listing.get("browsers") or []
active_id = self._active_browser_id(listing, browser_id)
if not active_id:
- await self._emit_empty_frame(
+ await self._emit_viewer_state(
sid,
context_id,
+ active_id,
browsers=browsers,
viewer_id=viewer_id,
- frame_source=VIEWER_TRANSPORT_SCREENCAST,
+ state=None,
+ viewer_transport=VIEWER_TRANSPORT_SCREENCAST,
)
await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS)
continue
@@ -410,29 +431,26 @@ class WsBrowser(WsHandler):
screencast = await runtime.call(
"start_screencast",
active_id,
- quality=SCREENCAST_QUALITY,
+ quality=SCREENCAST_STREAM_QUALITY,
every_nth_frame=1,
+ capture_scale=capture_scale,
)
stream_id = screencast["stream_id"]
active_id = screencast["browser_id"]
state = screencast.get("state")
- await self.emit_to(
+ await self._emit_viewer_state(
sid,
- "browser_viewer_frame",
- {
- "context_id": context_id,
- "viewer_id": viewer_id,
- "browser_id": active_id,
- "browsers": browsers,
- "image": "",
- "mime": "",
- "state": state,
- "frame_source": "state",
- "viewer_transport": VIEWER_TRANSPORT_SCREENCAST,
- },
+ context_id,
+ active_id,
+ browsers=browsers,
+ viewer_id=viewer_id,
+ state=state,
+ viewer_transport=VIEWER_TRANSPORT_SCREENCAST,
)
last_state_refresh = 0.0
+ last_state_signature = self._state_signature(active_id, browsers)
+ frame_sequence = 0
while True:
now = time.monotonic()
if now - last_state_refresh >= FRAME_STATE_REFRESH_SECONDS:
@@ -442,6 +460,18 @@ class WsBrowser(WsHandler):
if str(active_id) not in browser_ids:
break
state = self._state_for_browser(browsers, active_id, state)
+ state_signature = self._state_signature(active_id, browsers)
+ if state_signature != last_state_signature:
+ await self._emit_viewer_state(
+ sid,
+ context_id,
+ active_id,
+ browsers=browsers,
+ viewer_id=viewer_id,
+ state=state,
+ viewer_transport=VIEWER_TRANSPORT_SCREENCAST,
+ )
+ last_state_signature = state_signature
last_state_refresh = now
try:
@@ -455,14 +485,19 @@ class WsBrowser(WsHandler):
except TimeoutError:
continue
- frame["context_id"] = context_id
- frame["viewer_id"] = viewer_id
- frame["browser_id"] = active_id
- frame["browsers"] = browsers
- frame["state"] = state
- frame["frame_source"] = VIEWER_TRANSPORT_SCREENCAST
- frame["viewer_transport"] = VIEWER_TRANSPORT_SCREENCAST
- await self.emit_to(sid, "browser_viewer_frame", frame)
+ frame_sequence += 1
+ payload = self._frame_payload(
+ frame,
+ context_id=context_id,
+ viewer_id=viewer_id,
+ browser_id=active_id,
+ sequence=frame_sequence,
+ binary_frames=binary_frames,
+ )
+ if not slim_frames:
+ payload["browsers"] = browsers
+ payload["state"] = state
+ await self._emit_to_connected_viewer(sid, "browser_viewer_frame", payload)
except asyncio.CancelledError:
raise
except Exception:
@@ -515,20 +550,14 @@ class WsBrowser(WsHandler):
),
)
if signature != last_signature:
- await self.emit_to(
+ await self._emit_viewer_state(
sid,
- "browser_viewer_frame",
- {
- "context_id": context_id,
- "viewer_id": viewer_id,
- "browser_id": active_id,
- "browsers": browsers,
- "image": "",
- "mime": "",
- "state": state,
- "frame_source": VIEWER_TRANSPORT_SNAPSHOT,
- "viewer_transport": VIEWER_TRANSPORT_SNAPSHOT,
- },
+ context_id,
+ active_id,
+ browsers=browsers,
+ viewer_id=viewer_id,
+ state=state,
+ viewer_transport=VIEWER_TRANSPORT_SNAPSHOT,
)
last_signature = signature
await asyncio.sleep(SNAPSHOT_STATE_POLL_SECONDS)
@@ -567,6 +596,105 @@ class WsBrowser(WsHandler):
return browser
return current_state
+ @staticmethod
+ def _state_signature(
+ active_id: int | str | None,
+ browsers: list[dict[str, Any]],
+ ) -> tuple[str, tuple[tuple[str, str, str, str, bool], ...]]:
+ return (
+ str(active_id or ""),
+ tuple(
+ (
+ str(browser.get("context_id") or ""),
+ str(browser.get("id") or ""),
+ str(browser.get("currentUrl") or ""),
+ str(browser.get("title") or ""),
+ bool(browser.get("loading")),
+ )
+ for browser in browsers
+ ),
+ )
+
+ @staticmethod
+ def _frame_payload(
+ frame: dict[str, Any],
+ *,
+ context_id: str,
+ viewer_id: str,
+ browser_id: int | str,
+ sequence: int,
+ binary_frames: bool,
+ ) -> dict[str, Any]:
+ image = str(frame.get("image") or "")
+ payload: dict[str, Any] = {
+ "context_id": context_id,
+ "viewer_id": viewer_id,
+ "browser_id": browser_id,
+ "seq": sequence,
+ "mime": frame.get("mime") or "image/jpeg",
+ "frame_source": VIEWER_TRANSPORT_SCREENCAST,
+ "viewer_transport": VIEWER_TRANSPORT_SCREENCAST,
+ }
+ dimensions = WsBrowser._frame_dimensions(frame.get("metadata"))
+ if dimensions:
+ payload.update(dimensions)
+ if binary_frames:
+ try:
+ payload["image"] = base64.b64decode(image, validate=False)
+ payload["encoding"] = "binary"
+ except Exception:
+ payload["image"] = image
+ payload["encoding"] = "base64"
+ else:
+ payload["image"] = image
+ payload["encoding"] = "base64"
+ return payload
+
+ @staticmethod
+ def _frame_dimensions(metadata: Any) -> dict[str, int]:
+ if not isinstance(metadata, dict):
+ return {}
+ for width_key, height_key in (
+ ("expectedWidth", "expectedHeight"),
+ ("deviceWidth", "deviceHeight"),
+ ("jpegWidth", "jpegHeight"),
+ ):
+ try:
+ width = int(metadata.get(width_key) or 0)
+ height = int(metadata.get(height_key) or 0)
+ except (TypeError, ValueError):
+ continue
+ if width > 0 and height > 0:
+ return {"width": width, "height": height}
+ return {}
+
+ async def _emit_viewer_state(
+ self,
+ sid: str,
+ context_id: str,
+ browser_id: int | str | None,
+ *,
+ browsers: list[dict[str, Any]] | None = None,
+ viewer_id: str = "",
+ state: dict[str, Any] | None = None,
+ viewer_transport: str = VIEWER_TRANSPORT_SNAPSHOT,
+ ) -> None:
+ await self._emit_to_connected_viewer(
+ sid,
+ "browser_viewer_state",
+ {
+ "context_id": context_id,
+ "active_browser_context_id": context_id,
+ "viewer_id": viewer_id,
+ "browser_id": browser_id,
+ "active_browser_id": browser_id,
+ "browsers": browsers or [],
+ "state": state,
+ "all_browsers": False,
+ "viewer_transport": viewer_transport,
+ },
+ )
+
async def _emit_empty_frame(
self,
sid: str,
@@ -576,7 +704,7 @@ class WsBrowser(WsHandler):
viewer_id: str = "",
frame_source: str = "",
) -> None:
- await self.emit_to(
+ await self._emit_to_connected_viewer(
sid,
"browser_viewer_frame",
{
@@ -592,6 +720,20 @@ class WsBrowser(WsHandler):
},
)
+ async def _emit_to_connected_viewer(
+ self,
+ sid: str,
+ event: str,
+ data: dict[str, Any],
+ ) -> None:
+ manager = getattr(self, "_manager", None)
+ if manager is not None:
+ with manager.lock:
+ connected = (getattr(self, "namespace", "/ws"), sid) in manager.connections
+ if not connected:
+ raise asyncio.CancelledError()
+ await self.emit_to(sid, event, data)
+
@staticmethod
def _viewer_transport(data: dict[str, Any]) -> str:
raw = (
@@ -623,6 +765,20 @@ class WsBrowser(WsHandler):
"height": max(200, min(4096, height)),
}
+ @staticmethod
+ def _capture_scale_from_data(data: dict[str, Any]) -> float:
+ try:
+ scale = float(
+ data.get("device_pixel_ratio")
+ or data.get("devicePixelRatio")
+ or data.get("pixel_ratio")
+ or data.get("pixelRatio")
+ or 1
+ )
+ except (TypeError, ValueError):
+ return 1.0
+ return max(1.0, min(2.0, scale))
+
@staticmethod
def _context_id(data: dict[str, Any]) -> str:
return str(data.get("context_id") or data.get("context") or "").strip()
diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py
index e7455d81a..94524071d 100644
--- a/plugins/_browser/helpers/runtime.py
+++ b/plugins/_browser/helpers/runtime.py
@@ -329,10 +329,14 @@ class _BrowserScreencast:
quality: int,
every_nth_frame: int,
viewport: dict[str, int],
+ capture_scale: float = 1.0,
) -> None:
self.session.on("Page.screencastFrame", self._on_frame)
width = max(320, min(4096, int(viewport.get("width") or DEFAULT_VIEWPORT["width"])))
height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"])))
+ scale = max(1.0, min(2.0, float(capture_scale or 1.0)))
+ max_width = max(320, min(SCREENCAST_MAX_WIDTH, int(round(width * scale))))
+ max_height = max(200, min(SCREENCAST_MAX_HEIGHT, int(round(height * scale))))
self._expected_width = width
self._expected_height = height
with contextlib.suppress(Exception):
@@ -343,8 +347,8 @@ class _BrowserScreencast:
{
"format": "jpeg",
"quality": max(20, min(95, int(quality))),
- "maxWidth": SCREENCAST_MAX_WIDTH,
- "maxHeight": SCREENCAST_MAX_HEIGHT,
+ "maxWidth": max_width,
+ "maxHeight": max_height,
"everyNthFrame": max(1, int(every_nth_frame)),
},
)
@@ -1616,6 +1620,7 @@ class _BrowserRuntimeCore:
*,
quality: int = 78,
every_nth_frame: int = 1,
+ capture_scale: float = 1.0,
) -> dict[str, Any]:
await self.ensure_started()
resolved_id = self._resolve_browser_id(browser_id)
@@ -1634,6 +1639,7 @@ class _BrowserRuntimeCore:
quality=quality,
every_nth_frame=every_nth_frame,
viewport=page.viewport_size or DEFAULT_VIEWPORT,
+ capture_scale=capture_scale,
)
except Exception:
self.screencasts.pop(stream_id, None)
diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.js
index 6e70a1891..a4b3d4525 100644
--- a/plugins/_browser/webui/browser-store.js
+++ b/plugins/_browser/webui/browser-store.js
@@ -34,6 +34,8 @@ const ANNOTATION_DOM_LIMIT = 1200;
const ANNOTATION_TRAY_MARGIN = 10;
const BROWSER_VISUAL_SHORTCUT_KEYS = new Set(["a", "c", "insert", "v", "x", "y", "z"]);
const LOCAL_EDITABLE_SELECTOR = "input, textarea, select, [contenteditable]";
+const BROWSER_BINARY_FRAMES_SUPPORTED = typeof Blob === "function"
+ && typeof globalThis.URL?.createObjectURL === "function";
function makeViewerToken() {
return globalThis.crypto?.randomUUID?.()
@@ -114,6 +116,31 @@ function loadFrameDimensions(src) {
});
}
+function frameImageSource(data = {}) {
+ const image = data?.image;
+ if (!image) return null;
+ const mime = data.mime || "image/jpeg";
+ const isArrayBuffer = typeof ArrayBuffer !== "undefined" && image instanceof ArrayBuffer;
+ const isView = typeof ArrayBuffer !== "undefined" && ArrayBuffer.isView?.(image);
+ const isBlob = typeof Blob !== "undefined" && image instanceof Blob;
+ if (data.encoding === "binary" || isArrayBuffer || isView || isBlob) {
+ if (!BROWSER_BINARY_FRAMES_SUPPORTED) return null;
+ const blob = isBlob ? image : new Blob([image], { type: mime });
+ const src = globalThis.URL.createObjectURL(blob);
+ return {
+ src,
+ objectUrl: src,
+ cleanup: () => globalThis.URL.revokeObjectURL(src),
+ };
+ }
+ if (typeof image !== "string") return null;
+ return {
+ src: `data:${mime};base64,${image}`,
+ objectUrl: "",
+ cleanup: null,
+ };
+}
+
const model = {
loading: true,
error: "",
@@ -146,6 +173,7 @@ const model = {
_lastFrameDimensions: null,
_pendingFrameSrc: "",
_pendingFrameOptions: null,
+ _frameObjectUrl: "",
_frameRenderHandle: null,
_frameRenderCancel: null,
_frameRenderSequence: 0,
@@ -442,7 +470,7 @@ const model = {
this.setActiveBrowserId(null);
this.address = "";
this.frameState = null;
- this.frameSrc = "";
+ this.clearFrameSrc();
if (this.contextId) {
await this.connectViewer();
}
@@ -833,7 +861,7 @@ const model = {
resetRenderedFrame() {
this.cancelFrameRender();
- this.frameSrc = "";
+ this.clearFrameSrc();
this._lastFrameDimensions = null;
this._lastFrameAt = 0;
},
@@ -915,6 +943,25 @@ const model = {
return this.viewerTransport === BROWSER_VIEWER_TRANSPORT_SCREENCAST;
},
+ supportsBinaryFrames() {
+ return BROWSER_BINARY_FRAMES_SUPPORTED;
+ },
+
+ captureDevicePixelRatio() {
+ const value = Number(globalThis.devicePixelRatio || 1);
+ if (!Number.isFinite(value) || value <= 1) return 1;
+ return Math.min(2, value);
+ },
+
+ frameDimensionsFromData(data = null) {
+ const width = Number(data?.width || 0);
+ const height = Number(data?.height || 0);
+ if (Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0) {
+ return { width, height };
+ }
+ return this.frameDimensionsFromMetadata(data?.metadata);
+ },
+
frameDimensionsFromMetadata(metadata = null) {
if (!metadata || typeof metadata !== "object") return null;
const width = Number(metadata.expectedWidth || metadata.deviceWidth || metadata.jpegWidth || 0);
@@ -986,6 +1033,9 @@ const model = {
viewer_id: viewerToken,
create_browser: Boolean(options.createBrowser || options.create_browser),
viewer_transport: this.requestedViewerTransport(),
+ binary_frames: this.supportsBinaryFrames(),
+ slim_frames: true,
+ device_pixel_ratio: this.captureDevicePixelRatio(),
viewport_width: initialViewport?.width,
viewport_height: initialViewport?.height,
},
@@ -1014,9 +1064,9 @@ const model = {
data.active_browser_context_id || contextId,
);
this.applySnapshot(data.snapshot);
- this.connected = true;
- this.browserInstallExpected = false;
- },
+ this.connected = true;
+ this.browserInstallExpected = false;
+ },
async _bindSocketEvents() {
if (!this._frameOff) {
@@ -1028,7 +1078,9 @@ const model = {
}
const incomingContextId = this.normalizeContextId(data.context_id || this.contextId);
const incomingBrowserId = this.normalizeBrowserId(data.browser_id || data.state?.id);
- this.applyBrowserListing(data.browsers || [], incomingContextId, { replaceContext: true });
+ if (Array.isArray(data.browsers)) {
+ this.applyBrowserListing(data.browsers, incomingContextId, { replaceContext: true });
+ }
if (incomingBrowserId && !this.activeBrowserId) {
this.setActiveBrowserId(incomingBrowserId, incomingContextId);
}
@@ -1046,11 +1098,15 @@ const model = {
this.address = data.state.currentUrl;
}
if (data.image) {
+ const frameImage = frameImageSource(data);
+ if (!frameImage?.src) return;
const frameBrowserId = incomingBrowserId || this.activeBrowserId;
- this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`, {
+ this.queueFrameRender(frameImage.src, {
browserId: frameBrowserId,
contextId: incomingContextId,
- dimensions: this.frameDimensionsFromMetadata(data.metadata),
+ dimensions: this.frameDimensionsFromData(data),
+ objectUrl: frameImage.objectUrl,
+ cleanup: frameImage.cleanup,
onAccepted: () => {
if (
this.sameBrowserId(this.switchingBrowserId, frameBrowserId)
@@ -1063,13 +1119,13 @@ const model = {
});
} else if (!data.state) {
this.cancelFrameRender();
- this.frameSrc = "";
+ this.clearFrameSrc();
}
if (!data.image && !data.state) {
if (!this.activeBrowserId) {
this.setActiveBrowserId(null, "");
this.frameState = null;
- this.frameSrc = "";
+ this.clearFrameSrc();
}
}
this._lastFrameAt = Date.now();
@@ -1077,15 +1133,20 @@ const model = {
await websocket.on("browser_viewer_frame", frameHandler);
this._frameOff = () => websocket.off("browser_viewer_frame", frameHandler);
}
- if (!this._stateOff) {
- const stateHandler = ({ data }) => {
- if (data?.context_id !== this.contextId) return;
- if (data?.viewer_id && data.viewer_id !== this._viewerToken) return;
- if (data?.viewer_transport) {
- this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
- }
+ if (!this._stateOff) {
+ const stateHandler = ({ data }) => {
+ if (data?.context_id !== this.contextId) return;
+ if (data?.viewer_id && data.viewer_id !== this._viewerToken) return;
+ if (data?.viewer_transport) {
+ this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
+ }
const commandContextId = this.normalizeContextId(data.active_browser_context_id || data.context_id || this.contextId);
- this.applyBrowserListing(data.browsers || [], commandContextId, { replaceAll: Boolean(data.all_browsers) });
+ if (Array.isArray(data.browsers)) {
+ this.applyBrowserListing(data.browsers, commandContextId, {
+ replaceAll: Boolean(data.all_browsers),
+ replaceContext: !data.all_browsers,
+ });
+ }
const command = String(data.command || "").toLowerCase();
const commandBrowserId = this.normalizeBrowserId(data.browser_id);
const result = data.result || {};
@@ -1102,23 +1163,40 @@ const model = {
|| this.activeBrowserId
|| this.firstBrowserId(resultContextId)
);
- if (
- !this.activeBrowserId
- || command === "open"
- || command === "close"
- || this.sameBrowserTab(commandBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId)
- ) {
- this.setActiveBrowserId(preferredBrowserId, resultContextId);
- }
- this.applyActiveFrameState(resultState || this.browserById(this.activeBrowserId, this.activeBrowserContextId));
- this.applySnapshot(data.snapshot);
- };
+ const stateBrowserId = this.normalizeBrowserId(data.active_browser_id || data.browser_id || data.state?.id);
+ if (
+ stateBrowserId
+ && (
+ !this.activeBrowserId
+ || this.sameBrowserTab(stateBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId)
+ )
+ ) {
+ this.setActiveBrowserId(stateBrowserId, commandContextId);
+ }
+ if (
+ !this.activeBrowserId
+ || command === "open"
+ || command === "close"
+ || this.sameBrowserTab(commandBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId)
+ ) {
+ this.setActiveBrowserId(preferredBrowserId, resultContextId);
+ }
+ this.applyActiveFrameState(
+ resultState
+ || data.state
+ || this.browserById(this.activeBrowserId, this.activeBrowserContextId)
+ );
+ this.applySnapshot(data.snapshot);
+ };
await websocket.on("browser_viewer_state", stateHandler);
this._stateOff = () => websocket.off("browser_viewer_state", stateHandler);
}
},
queueFrameRender(frameSrc, options = {}) {
+ if (this._pendingFrameSrc) {
+ this.releasePendingFrame();
+ }
this._pendingFrameSrc = frameSrc;
this._pendingFrameOptions = options || null;
if (this._frameRenderHandle) return;
@@ -1148,22 +1226,26 @@ const model = {
async renderDecodedFrame(frameSrc, options = {}, sequence = 0, surfaceSequence = this._surfaceOpenSequence) {
if (!frameSrc) {
if (sequence === this._frameRenderSequence) {
- this.frameSrc = "";
+ this.clearFrameSrc();
}
return;
}
const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc);
if (sequence !== this._frameRenderSequence || surfaceSequence !== this._surfaceOpenSequence) {
+ options?.cleanup?.();
return;
}
const viewport = this.currentViewportSize() || this._lastViewport;
if (!this.frameMatchesViewport(dimensions, viewport)) {
this.requestViewportSyncAfterRejectedFrame();
if (!this.shouldAcceptMismatchedFrame(dimensions)) {
+ options?.cleanup?.();
return;
}
}
+ this.releaseRenderedFrameUrl(frameSrc);
this.frameSrc = frameSrc;
+ this._frameObjectUrl = options?.objectUrl || "";
this._lastFrameDimensions = dimensions;
this._lastFrameAt = Date.now();
options?.onAccepted?.();
@@ -1262,9 +1344,26 @@ const model = {
}
this._frameRenderHandle = null;
this._frameRenderCancel = null;
+ this.releasePendingFrame();
+ this._frameRenderSequence += 1;
+ },
+
+ releasePendingFrame() {
+ this._pendingFrameOptions?.cleanup?.();
this._pendingFrameSrc = "";
this._pendingFrameOptions = null;
- this._frameRenderSequence += 1;
+ },
+
+ releaseRenderedFrameUrl(nextSrc = "") {
+ if (this._frameObjectUrl && this._frameObjectUrl !== nextSrc) {
+ globalThis.URL?.revokeObjectURL?.(this._frameObjectUrl);
+ this._frameObjectUrl = "";
+ }
+ },
+
+ clearFrameSrc() {
+ this.releaseRenderedFrameUrl("");
+ this.frameSrc = "";
},
beginCommand() {
@@ -1329,7 +1428,7 @@ const model = {
);
if (!this.activeBrowserId) {
this.frameState = null;
- this.frameSrc = "";
+ this.clearFrameSrc();
}
if (result.state?.currentUrl || result.currentUrl) {
this.address = result.state?.currentUrl || result.currentUrl;
@@ -1438,7 +1537,7 @@ const model = {
this.error = "";
this.switchingBrowserId = targetId;
this.cancelFrameRender();
- this.frameSrc = "";
+ this.clearFrameSrc();
this.frameState = browser || null;
if (!this.addressFocused && browser?.currentUrl) {
this.address = browser.currentUrl;
@@ -1643,35 +1742,35 @@ const model = {
}
},
- applySnapshot(snapshot = null) {
- if (!snapshot?.image) return;
- const snapshotId = this.normalizeBrowserId(snapshot.browser_id || snapshot.state?.id);
+ applySnapshot(snapshot = null) {
+ if (!snapshot?.image) return;
+ const snapshotId = this.normalizeBrowserId(snapshot.browser_id || snapshot.state?.id);
const snapshotContextId = this.normalizeContextId(snapshot.context_id || snapshot.state?.context_id || this.activeBrowserContextId);
- if (
+ if (
snapshotId
&& this.activeBrowserId
&& !this.sameBrowserTab(snapshotId, snapshotContextId, this.activeBrowserId, this.activeBrowserContextId)
) {
- return;
- }
- if (snapshot.state) {
- this.applyActiveFrameState(snapshot.state);
- }
- const frameBrowserId = snapshotId || this.activeBrowserId;
- this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, {
- browserId: frameBrowserId,
+ return;
+ }
+ if (snapshot.state) {
+ this.applyActiveFrameState(snapshot.state);
+ }
+ const frameBrowserId = snapshotId || this.activeBrowserId;
+ this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, {
+ browserId: frameBrowserId,
contextId: snapshotContextId,
- onAccepted: () => {
- if (
+ onAccepted: () => {
+ if (
this.sameBrowserId(this.switchingBrowserId, frameBrowserId)
&& this.normalizeContextId(this.activeBrowserContextId) === snapshotContextId
) {
- this.switchingBrowserId = null;
- }
- this._surfaceSwitching = false;
- },
- });
- },
+ this.switchingBrowserId = null;
+ }
+ this._surfaceSwitching = false;
+ },
+ });
+ },
isSwitchingBrowser() {
return Boolean(
diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py
index eef2e141b..7503efeb3 100644
--- a/tests/test_browser_agent_regressions.py
+++ b/tests/test_browser_agent_regressions.py
@@ -1158,7 +1158,8 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "def _viewer_transport(data: dict[str, Any])" in ws_browser
assert "return VIEWER_TRANSPORT_SNAPSHOT" in ws_browser
assert "self._stream_state" in ws_browser
- assert "SCREENCAST_QUALITY = 92" in ws_browser
+ assert "SCREENCAST_STREAM_QUALITY = 80" in ws_browser
+ assert "SCREENSHOT_QUALITY = 92" in ws_browser
assert "initial_viewport = self._viewport_from_data(data)" in ws_browser
assert '"set_viewport"' in ws_browser
assert "start_screencast" in ws_browser
@@ -1169,7 +1170,7 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "stop_screencast" in ws_browser
assert "viewer_transport == VIEWER_TRANSPORT_SCREENCAST" in ws_browser
assert '"viewer_transport": viewer_transport' in ws_browser
- assert '"viewer_transport": VIEWER_TRANSPORT_SNAPSHOT' in ws_browser
+ assert "viewer_transport: str = VIEWER_TRANSPORT_SNAPSHOT" in ws_browser
assert '"Page.startScreencast"' in runtime
assert '"Page.screencastFrame"' in runtime
assert '"Page.screencastFrameAck"' in runtime
@@ -1180,6 +1181,7 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "await self._stop_screencasts_for_browser(resolved_id)" in runtime
assert "queueFrameRender" in browser_store
assert "requestAnimationFrame" in browser_store
+ assert "function frameImageSource(data = {})" in browser_store
assert 'const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";' in browser_store
assert 'const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";' in browser_store
assert "viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST" in browser_store
@@ -1187,11 +1189,17 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "requestedViewerTransport()" in browser_store
assert "normalizeViewerTransport(value = \"\")" in browser_store
assert "usesScreencastTransport()" in browser_store
+ assert "supportsBinaryFrames()" in browser_store
+ assert "captureDevicePixelRatio()" in browser_store
+ assert "frameDimensionsFromData(data = null)" in browser_store
assert "frameDimensionsFromMetadata(metadata = null)" in browser_store
assert "metadata.expectedWidth || metadata.deviceWidth || metadata.jpegWidth" in browser_store
- assert "dimensions: this.frameDimensionsFromMetadata(data.metadata)" in browser_store
+ assert "dimensions: this.frameDimensionsFromData(data)" in browser_store
assert "const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc)" in browser_store
assert "viewer_transport: this.requestedViewerTransport()" in browser_store
+ assert "binary_frames: this.supportsBinaryFrames()" in browser_store
+ assert "slim_frames: true" in browser_store
+ assert "device_pixel_ratio: this.captureDevicePixelRatio()" in browser_store
assert "viewport_width: initialViewport?.width" in browser_store
assert "viewport_height: initialViewport?.height" in browser_store
assert "restart_stream: restartStream && this.usesScreencastTransport()" in browser_store
@@ -1213,6 +1221,13 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "this.applySnapshot(data.snapshot);" in browser_store
assert "else if (!data.state)" in browser_store
assert '"snapshot": snapshot' in ws_browser
+ assert '"binary_frames": binary_frames' in ws_browser
+ assert '"slim_frames": slim_frames' in ws_browser
+ assert "capture_scale=capture_scale" in ws_browser
+ assert "base64.b64decode(image, validate=False)" in ws_browser
+ assert '"encoding"] = "binary"' in ws_browser
+ assert "def _frame_payload(" in ws_browser
+ assert "def _emit_viewer_state(" in ws_browser
assert 'const BROWSER_SNAPSHOT_META_KEY = "browser_snapshot";' in browser_tool_handler
assert "staticScreenshotUri(kvps)" in browser_tool_handler
assert "/components/modals/image-viewer/image-viewer-store.js" in browser_tool_handler
@@ -1223,9 +1238,9 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "delete displayKvps[BROWSER_SNAPSHOT_META_KEY];" in browser_tool_handler
assert "startBrowserScreenshotPreview(button, image, resolveBrowserPayload)" in browser_tool_handler
assert "FRAME_FALLBACK_SCREENSHOT_SECONDS" not in ws_browser
- assert '"frame_source": "state"' in ws_browser
- assert '"frame_source"] = VIEWER_TRANSPORT_SCREENCAST' in ws_browser
- assert '"viewer_transport"] = VIEWER_TRANSPORT_SCREENCAST' in ws_browser
+ assert '"browser_viewer_state"' in ws_browser
+ assert '"frame_source": VIEWER_TRANSPORT_SCREENCAST' in ws_browser
+ assert '"viewer_transport": VIEWER_TRANSPORT_SCREENCAST' in ws_browser
assert "fallback_screenshot" not in ws_browser
assert "canvas_wheel_screenshot" not in ws_browser
assert "surface_mode: this._mode" not in browser_store
@@ -1234,6 +1249,29 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "image-rendering: auto;" in main_html
+def test_browser_viewer_frame_payload_supports_binary_slim_frames():
+ payload = ws_browser_module.WsBrowser._frame_payload(
+ {
+ "image": SMALL_JPEG_10X10,
+ "mime": "image/jpeg",
+ "metadata": {"expectedWidth": 900, "expectedHeight": 600},
+ },
+ context_id="ctx",
+ viewer_id="viewer",
+ browser_id=1,
+ sequence=3,
+ binary_frames=True,
+ )
+
+ assert payload["encoding"] == "binary"
+ assert payload["image"] == __import__("base64").b64decode(SMALL_JPEG_10X10)
+ assert payload["width"] == 900
+ assert payload["height"] == 600
+ assert payload["seq"] == 3
+ assert "browsers" not in payload
+ assert "state" not in payload
+
+
def test_browser_navigation_errors_stay_inside_native_browser_page():
runtime = (
PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"
@@ -1504,7 +1542,12 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():
mime="image/jpeg",
)
- await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 1118, "height": 662})
+ await screencast.start(
+ quality=92,
+ every_nth_frame=1,
+ viewport={"width": 1118, "height": 662},
+ capture_scale=2,
+ )
session.handlers["Page.screencastFrame"](
{"data": first_image, "metadata": {"deviceWidth": 10}, "sessionId": 1}
)
@@ -1563,6 +1606,10 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():
for index, (method, _params) in enumerate(session.sent)
if method == "Page.startScreencast"
)
+ start_params = session.sent[start_index][1]
+ assert start_params["quality"] == 92
+ assert start_params["maxWidth"] == 2236
+ assert start_params["maxHeight"] == 1324
cdp_viewport_indices = [
index
for index, (method, _params) in enumerate(session.sent)
@@ -2287,7 +2334,7 @@ async def test_browser_viewer_subscribe_returns_initial_snapshot(monkeypatch):
assert result["active_browser_id"] == 1
assert result["snapshot"]["image"] == "jpeg-data"
- assert ("screenshot", (1,), {"quality": ws_browser_module.SCREENCAST_QUALITY}) in calls
+ assert ("screenshot", (1,), {"quality": ws_browser_module.SCREENSHOT_QUALITY}) in calls
await handler.on_disconnect("sid-snapshot")
From 727a840e10c1d877d5d54ce68e3e2ffa0ddef485 Mon Sep 17 00:00:00 2001
From: Alessandro <155005371+3clyp50@users.noreply.github.com>
Date: Fri, 3 Jul 2026 02:15:54 +0200
Subject: [PATCH 05/43] Keep WebUI browser frames on base64 fallback
Disable WebUI binary-frame negotiation until its Socket.IO client reconstructs frame attachments as real Blob, ArrayBuffer, or typed-array values.
Reject placeholder binary payloads instead of wrapping them into broken image blobs, preserving the slim-frame path through base64 frames.
Document the transport constraint and pin the fallback behavior in browser regression coverage.
---
plugins/_browser/AGENTS.md | 2 +-
plugins/_browser/webui/browser-store.js | 10 ++++++----
tests/test_browser_agent_regressions.py | 3 +++
3 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/plugins/_browser/AGENTS.md b/plugins/_browser/AGENTS.md
index e3f2a137e..e80bbe016 100644
--- a/plugins/_browser/AGENTS.md
+++ b/plugins/_browser/AGENTS.md
@@ -19,7 +19,7 @@
- Preserve Playwright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding.
- Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation.
- Default the visible WebUI Browser to live CDP screencast for responsiveness. Keep lightweight CDP/DOM state snapshots as the fallback transport.
-- Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback.
+- Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback. Do not let the WebUI advertise binary frames unless its Socket.IO client reconstructs attachments as real `Blob`, `ArrayBuffer`, or typed-array values.
- Keep narrow WebUI Browser controls usable by grouping navigation with Annotate/settings above a full-width address bar.
- Browser URL-intent handling must only claim web URL schemes and leave custom Agent Zero schemes to their owning surfaces.
- Prefer DOM/CDP browser actions with refs, selectors, frame-chain refs, and screenshots over viewport coordinate input. Coordinates remain a visual fallback.
diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.js
index a4b3d4525..c2fdcd84b 100644
--- a/plugins/_browser/webui/browser-store.js
+++ b/plugins/_browser/webui/browser-store.js
@@ -34,7 +34,8 @@ const ANNOTATION_DOM_LIMIT = 1200;
const ANNOTATION_TRAY_MARGIN = 10;
const BROWSER_VISUAL_SHORTCUT_KEYS = new Set(["a", "c", "insert", "v", "x", "y", "z"]);
const LOCAL_EDITABLE_SELECTOR = "input, textarea, select, [contenteditable]";
-const BROWSER_BINARY_FRAMES_SUPPORTED = typeof Blob === "function"
+const BROWSER_BINARY_FRAME_REQUESTS_ENABLED = false;
+const BROWSER_BINARY_PAYLOADS_SUPPORTED = typeof Blob === "function"
&& typeof globalThis.URL?.createObjectURL === "function";
function makeViewerToken() {
@@ -123,8 +124,8 @@ function frameImageSource(data = {}) {
const isArrayBuffer = typeof ArrayBuffer !== "undefined" && image instanceof ArrayBuffer;
const isView = typeof ArrayBuffer !== "undefined" && ArrayBuffer.isView?.(image);
const isBlob = typeof Blob !== "undefined" && image instanceof Blob;
- if (data.encoding === "binary" || isArrayBuffer || isView || isBlob) {
- if (!BROWSER_BINARY_FRAMES_SUPPORTED) return null;
+ if (isArrayBuffer || isView || isBlob) {
+ if (!BROWSER_BINARY_PAYLOADS_SUPPORTED) return null;
const blob = isBlob ? image : new Blob([image], { type: mime });
const src = globalThis.URL.createObjectURL(blob);
return {
@@ -133,6 +134,7 @@ function frameImageSource(data = {}) {
cleanup: () => globalThis.URL.revokeObjectURL(src),
};
}
+ if (data.encoding === "binary") return null;
if (typeof image !== "string") return null;
return {
src: `data:${mime};base64,${image}`,
@@ -944,7 +946,7 @@ const model = {
},
supportsBinaryFrames() {
- return BROWSER_BINARY_FRAMES_SUPPORTED;
+ return BROWSER_BINARY_FRAME_REQUESTS_ENABLED && BROWSER_BINARY_PAYLOADS_SUPPORTED;
},
captureDevicePixelRatio() {
diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py
index 7503efeb3..2cc355a14 100644
--- a/tests/test_browser_agent_regressions.py
+++ b/tests/test_browser_agent_regressions.py
@@ -1182,6 +1182,9 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "queueFrameRender" in browser_store
assert "requestAnimationFrame" in browser_store
assert "function frameImageSource(data = {})" in browser_store
+ assert "const BROWSER_BINARY_FRAME_REQUESTS_ENABLED = false;" in browser_store
+ assert "const BROWSER_BINARY_PAYLOADS_SUPPORTED = typeof Blob" in browser_store
+ assert "if (data.encoding === \"binary\") return null;" in browser_store
assert 'const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";' in browser_store
assert 'const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";' in browser_store
assert "viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST" in browser_store
From 277bcd0783a5165801f3113c4aa67406690772f9 Mon Sep 17 00:00:00 2001
From: Alessandro <155005371+3clyp50@users.noreply.github.com>
Date: Fri, 3 Jul 2026 03:16:11 +0200
Subject: [PATCH 06/43] Prevent browser frame stretch on resize
Keep the live browser frame proportional while the viewer is being resized by using contain sizing instead of stretching the stale frame to the transient surface shape.
Update the regression guard so the viewer contract pins the proportional render path.
---
plugins/_browser/webui/browser-panel.html | 2 +-
tests/test_browser_agent_regressions.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/plugins/_browser/webui/browser-panel.html b/plugins/_browser/webui/browser-panel.html
index 7ad3902f5..5692dd063 100644
--- a/plugins/_browser/webui/browser-panel.html
+++ b/plugins/_browser/webui/browser-panel.html
@@ -999,7 +999,7 @@
height: 100%;
min-width: 0;
min-height: 0;
- object-fit: fill;
+ object-fit: contain;
image-rendering: auto;
user-select: none;
background: #fff;
diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py
index 2cc355a14..a67de6054 100644
--- a/tests/test_browser_agent_regressions.py
+++ b/tests/test_browser_agent_regressions.py
@@ -1248,7 +1248,7 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "canvas_wheel_screenshot" not in ws_browser
assert "surface_mode: this._mode" not in browser_store
assert "overflow: hidden;" in main_html
- assert "object-fit: fill;" in main_html
+ assert "object-fit: contain;" in main_html
assert "image-rendering: auto;" in main_html
From 69fb0fca2aa3405a5a8154d572e763c165ff9441 Mon Sep 17 00:00:00 2001
From: Alessandro <155005371+3clyp50@users.noreply.github.com>
Date: Fri, 3 Jul 2026 11:06:14 +0200
Subject: [PATCH 07/43] Smooth browser viewer rendering and frame push
Render live browser screencast frames through a canvas/ImageBitmap path while keeping the existing image/data-url path for snapshots and fallback rendering.
Attach WebSocket streams as runtime screencast consumers so delivered frames are pushed to the server loop and acknowledged after emit, removing the per-frame read_screencast_frame runtime round trip.
---
plugins/_browser/AGENTS.md | 3 +
plugins/_browser/api/ws_browser.py | 51 +++++----
plugins/_browser/helpers/runtime.py | 33 +++++-
plugins/_browser/webui/browser-panel.html | 11 +-
plugins/_browser/webui/browser-store.js | 119 +++++++++++++++++---
tests/test_browser_agent_regressions.py | 125 ++++++++++++++++++++--
6 files changed, 296 insertions(+), 46 deletions(-)
diff --git a/plugins/_browser/AGENTS.md b/plugins/_browser/AGENTS.md
index e80bbe016..6a434bbd5 100644
--- a/plugins/_browser/AGENTS.md
+++ b/plugins/_browser/AGENTS.md
@@ -19,6 +19,8 @@
- Preserve Playwright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding.
- Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation.
- Default the visible WebUI Browser to live CDP screencast for responsiveness. Keep lightweight CDP/DOM state snapshots as the fallback transport.
+- Paint live screencast frames through the Browser panel canvas/ImageBitmap path when available; keep the ``/data URL path for snapshots and fallback rendering.
+- Push internal screencast frames from the runtime to the WebSocket consumer after subscription; keep `read/pop_screencast_frame` as fallback/tooling APIs, not the WebUI hot path.
- Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback. Do not let the WebUI advertise binary frames unless its Socket.IO client reconstructs attachments as real `Blob`, `ArrayBuffer`, or typed-array values.
- Keep narrow WebUI Browser controls usable by grouping navigation with Annotate/settings above a full-width address bar.
- Browser URL-intent handling must only claim web URL schemes and leave custom Agent Zero schemes to their owning surfaces.
@@ -36,6 +38,7 @@
## Verification
- Smoke-test browser launch, navigation, DOM capture, and WebUI viewer after runtime changes.
+- For viewer render-path changes, verify the live Browser panel paints a screencast frame on canvas with `frameSrc` empty and snapshots still falling back to the image path.
- Run browser prompt/skill regression tests after changing browser prompt or Browser plugin skills.
## Child DOX Index
diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.py
index 827e91df4..566ec10b8 100644
--- a/plugins/_browser/api/ws_browser.py
+++ b/plugins/_browser/api/ws_browser.py
@@ -451,7 +451,37 @@ class WsBrowser(WsHandler):
last_state_refresh = 0.0
last_state_signature = self._state_signature(active_id, browsers)
frame_sequence = 0
+ server_loop = asyncio.get_running_loop()
+ stop_event = asyncio.Event()
+
+ async def emit_frame(frame: dict[str, Any]) -> None:
+ nonlocal frame_sequence
+ try:
+ frame_sequence += 1
+ payload = self._frame_payload(
+ frame,
+ context_id=context_id,
+ viewer_id=viewer_id,
+ browser_id=active_id,
+ sequence=frame_sequence,
+ binary_frames=binary_frames,
+ )
+ if not slim_frames:
+ payload["browsers"] = browsers
+ payload["state"] = state
+ await self._emit_to_connected_viewer(sid, "browser_viewer_frame", payload)
+ except BaseException:
+ stop_event.set()
+ raise
+
+ def frame_consumer(frame: dict[str, Any]):
+ return asyncio.run_coroutine_threadsafe(emit_frame(frame), server_loop)
+
+ await runtime.call("attach_screencast_consumer", stream_id, frame_consumer)
+
while True:
+ if stop_event.is_set():
+ break
now = time.monotonic()
if now - last_state_refresh >= FRAME_STATE_REFRESH_SECONDS:
listing = await runtime.call("list")
@@ -475,29 +505,10 @@ class WsBrowser(WsHandler):
last_state_refresh = now
try:
- frame = await runtime.call(
- "read_screencast_frame",
- stream_id,
- timeout=FRAME_READ_TIMEOUT_SECONDS,
- )
- except KeyError:
+ await asyncio.wait_for(stop_event.wait(), timeout=FRAME_READ_TIMEOUT_SECONDS)
break
except TimeoutError:
continue
-
- frame_sequence += 1
- payload = self._frame_payload(
- frame,
- context_id=context_id,
- viewer_id=viewer_id,
- browser_id=active_id,
- sequence=frame_sequence,
- binary_frames=binary_frames,
- )
- if not slim_frames:
- payload["browsers"] = browsers
- payload["state"] = state
- await self._emit_to_connected_viewer(sid, "browser_viewer_frame", payload)
except asyncio.CancelledError:
raise
except Exception:
diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py
index 94524071d..6031a7f5f 100644
--- a/plugins/_browser/helpers/runtime.py
+++ b/plugins/_browser/helpers/runtime.py
@@ -317,6 +317,7 @@ class _BrowserScreencast:
self.browser_id = browser_id
self.session = session
self.mime = mime
+ self.frame_consumer: Any | None = None
self.queue = asyncio.Queue(maxsize=1)
self.stopped = False
self._ack_tasks: set[asyncio.Task] = set()
@@ -398,6 +399,12 @@ class _BrowserScreencast:
raise RuntimeError("Browser screencast stopped.")
return frame
+ async def attach_consumer(self, frame_consumer: Any) -> None:
+ self.frame_consumer = frame_consumer
+ frame = await self.pop_frame()
+ if frame:
+ await self._deliver_frame(frame)
+
async def stop(self) -> None:
if self.stopped:
return
@@ -423,6 +430,7 @@ class _BrowserScreencast:
task.add_done_callback(self._ack_tasks.discard)
async def _handle_frame(self, params: dict[str, Any]) -> None:
+ stop_after_ack = False
try:
data = params.get("data") or ""
if data:
@@ -432,7 +440,7 @@ class _BrowserScreencast:
metadata["jpegWidth"], metadata["jpegHeight"] = size
metadata["expectedWidth"] = self._expected_width
metadata["expectedHeight"] = self._expected_height
- self._queue_latest(
+ await self._deliver_frame(
{
"browser_id": self.browser_id,
"mime": self.mime,
@@ -440,6 +448,13 @@ class _BrowserScreencast:
"metadata": metadata,
}
)
+ except asyncio.CancelledError:
+ stop_after_ack = True
+ except Exception:
+ if self.frame_consumer:
+ stop_after_ack = True
+ else:
+ raise
finally:
session_id = params.get("sessionId")
if session_id is not None and not self.stopped:
@@ -448,6 +463,16 @@ class _BrowserScreencast:
"Page.screencastFrameAck",
{"sessionId": int(session_id)},
)
+ if stop_after_ack:
+ self.stopped = True
+
+ async def _deliver_frame(self, frame: dict[str, Any]) -> None:
+ if not self.frame_consumer:
+ self._queue_latest(frame)
+ return
+ future = self.frame_consumer(frame)
+ if future is not None:
+ await asyncio.wrap_future(future)
def _queue_latest(self, frame: dict[str, Any]) -> None:
self._drop_queued_frames()
@@ -1669,6 +1694,12 @@ class _BrowserRuntimeCore:
raise KeyError("Browser screencast is not active.")
return await screencast.pop_frame()
+ async def attach_screencast_consumer(self, stream_id: str, frame_consumer: Any) -> None:
+ screencast = self.screencasts.get(str(stream_id or ""))
+ if not screencast:
+ raise KeyError("Browser screencast is not active.")
+ await screencast.attach_consumer(frame_consumer)
+
async def stop_screencast(self, stream_id: str) -> None:
screencast = self.screencasts.pop(str(stream_id or ""), None)
if screencast:
diff --git a/plugins/_browser/webui/browser-panel.html b/plugins/_browser/webui/browser-panel.html
index 5692dd063..1abcde86f 100644
--- a/plugins/_browser/webui/browser-panel.html
+++ b/plugins/_browser/webui/browser-panel.html
@@ -187,12 +187,17 @@