vrr/agent-zero
2
0
Fork 0
mirror of https://github.com/agent0ai/agent-zero.git synced 2026-07-09 17:08:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Slim browser viewer frame transport

Browse source 
Negotiate binary and slim screencast frames so updated clients avoid base64-in-JSON and repeated viewer metadata on every frame.

Clamp screencast capture to the subscriber viewport and DPR, lower streaming JPEG quality while preserving snapshot quality, and teach the client to manage binary frame URLs safely.

Document the browser viewer performance plan and update regression coverage for the new transport contract.
This commit is contained in:
Alessandro 2026-07-03 01:49:07 +02:00
parent 686e039d1b
commit 91b3bc9613
6 changed files with 417 additions and 107 deletions
Download patch file Download diff file Expand all files Collapse all files

1
docs/AGENTS.md
View file

@ -9,6 +9,7 @@
- `README.md`, `quickstart.md`, `guides/`, and `setup/` cover user-facing setup and workflows.
- `developer/` covers compact developer references and source handoffs.
- `plans/` covers implementation plans, migration notes, and staged technical roadmaps.
- `res/` contains documentation images and other documentation assets.
## Local Contracts

1
plugins/_browser/AGENTS.md
View file

@ -19,6 +19,7 @@
- Preserve Playwright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding.
- Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation.
- Default the visible WebUI Browser to live CDP screencast for responsiveness. Keep lightweight CDP/DOM state snapshots as the fallback transport.
- Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback.
- Keep narrow WebUI Browser controls usable by grouping navigation with Annotate/settings above a full-width address bar.
- Browser URL-intent handling must only claim web URL schemes and leave custom Agent Zero schemes to their owning surfaces.
- Prefer DOM/CDP browser actions with refs, selectors, frame-chain refs, and screenshots over viewport coordinate input. Coordinates remain a visual fallback.

244
plugins/_browser/api/ws_browser.py
View file

@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
import base64
import contextlib
import time
from typing import Any, ClassVar
@ -15,7 +16,8 @@ FRAME_READ_TIMEOUT_SECONDS = 0.5
FRAME_RETRY_DELAY_SECONDS = 0.5
FRAME_STATE_REFRESH_SECONDS = 0.75
SNAPSHOT_STATE_POLL_SECONDS = 0.75
SCREENCAST_QUALITY = 92
SCREENCAST_STREAM_QUALITY = 80
SCREENSHOT_QUALITY = 92
VIEWER_TRANSPORT_SCREENCAST = "screencast"
VIEWER_TRANSPORT_SNAPSHOT = "snapshot"
VIEWER_TRANSPORTS = {VIEWER_TRANSPORT_SCREENCAST, VIEWER_TRANSPORT_SNAPSHOT}
@ -97,10 +99,21 @@ class WsBrowser(WsHandler):
existing.cancel()
viewer_id = str(data.get("viewer_id") or "")
viewer_transport = self._viewer_transport(data)
binary_frames = self._bool(data.get("binary_frames", data.get("binaryFrames")))
slim_frames = self._bool(data.get("slim_frames", data.get("slimFrames", binary_frames)))
capture_scale = self._capture_scale_from_data(data)
snapshot = None
if runtime:
if viewer_transport == VIEWER_TRANSPORT_SCREENCAST:
stream_task = self._stream_frames(sid, context_id, active_id, viewer_id)
stream_task = self._stream_frames(
sid,
context_id,
active_id,
viewer_id,
binary_frames=binary_frames,
slim_frames=slim_frames,
capture_scale=capture_scale,
)
else:
stream_task = self._stream_state(sid, context_id, active_id, viewer_id)
self._streams[stream_key] = asyncio.create_task(stream_task)
@ -115,6 +128,8 @@ class WsBrowser(WsHandler):
"all_browsers": True,
"viewer_id": viewer_id,
"viewer_transport": viewer_transport,
"binary_frames": binary_frames,
"slim_frames": slim_frames,
}
def _unsubscribe(self, data: dict[str, Any], sid: str) -> dict[str, Any] | WsResult:
@ -157,9 +172,9 @@ class WsBrowser(WsHandler):
snapshot = None
if active_id:
try:
quality = int(data.get("quality") or SCREENCAST_QUALITY)
quality = int(data.get("quality") or SCREENSHOT_QUALITY)
except (TypeError, ValueError):
quality = SCREENCAST_QUALITY
quality = SCREENSHOT_QUALITY
with contextlib.suppress(Exception):
snapshot = await runtime.call(
"screenshot",
@ -347,7 +362,7 @@ class WsBrowser(WsHandler):
if not browser_id:
return None
with contextlib.suppress(Exception):
return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY)
return await runtime.call("screenshot", browser_id, quality=SCREENSHOT_QUALITY)
return None
async def _snapshot_for_browser(
@ -358,7 +373,7 @@ class WsBrowser(WsHandler):
if not browser_id:
return None
with contextlib.suppress(Exception):
return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY)
return await runtime.call("screenshot", browser_id, quality=SCREENSHOT_QUALITY)
return None
async def _all_browser_tabs(self) -> list[dict[str, Any]]:
@ -377,6 +392,10 @@ class WsBrowser(WsHandler):
context_id: str,
browser_id: int | str | None,
viewer_id: str = "",
*,
binary_frames: bool = False,
slim_frames: bool = False,
capture_scale: float = 1.0,
) -> None:
runtime = None
stream_id = None
@ -397,12 +416,14 @@ class WsBrowser(WsHandler):
browsers = listing.get("browsers") or []
active_id = self._active_browser_id(listing, browser_id)
if not active_id:
await self._emit_empty_frame(
await self._emit_viewer_state(
sid,
context_id,
active_id,
browsers=browsers,
viewer_id=viewer_id,
frame_source=VIEWER_TRANSPORT_SCREENCAST,
state=None,
viewer_transport=VIEWER_TRANSPORT_SCREENCAST,
)
await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS)
continue
@ -410,29 +431,26 @@ class WsBrowser(WsHandler):
screencast = await runtime.call(
"start_screencast",
active_id,
quality=SCREENCAST_QUALITY,
quality=SCREENCAST_STREAM_QUALITY,
every_nth_frame=1,
capture_scale=capture_scale,
)
stream_id = screencast["stream_id"]
active_id = screencast["browser_id"]
state = screencast.get("state")
await self.emit_to(
await self._emit_viewer_state(
sid,
"browser_viewer_frame",
{
"context_id": context_id,
"viewer_id": viewer_id,
"browser_id": active_id,
"browsers": browsers,
"image": "",
"mime": "",
"state": state,
"frame_source": "state",
"viewer_transport": VIEWER_TRANSPORT_SCREENCAST,
},
context_id,
active_id,
browsers=browsers,
viewer_id=viewer_id,
state=state,
viewer_transport=VIEWER_TRANSPORT_SCREENCAST,
)
last_state_refresh = 0.0
last_state_signature = self._state_signature(active_id, browsers)
frame_sequence = 0
while True:
now = time.monotonic()
if now - last_state_refresh >= FRAME_STATE_REFRESH_SECONDS:
@ -442,6 +460,18 @@ class WsBrowser(WsHandler):
if str(active_id) not in browser_ids:
break
state = self._state_for_browser(browsers, active_id, state)
state_signature = self._state_signature(active_id, browsers)
if state_signature != last_state_signature:
await self._emit_viewer_state(
sid,
context_id,
active_id,
browsers=browsers,
viewer_id=viewer_id,
state=state,
viewer_transport=VIEWER_TRANSPORT_SCREENCAST,
)
last_state_signature = state_signature
last_state_refresh = now
try:
@ -455,14 +485,19 @@ class WsBrowser(WsHandler):
except TimeoutError:
continue
frame["context_id"] = context_id
frame["viewer_id"] = viewer_id
frame["browser_id"] = active_id
frame["browsers"] = browsers
frame["state"] = state
frame["frame_source"] = VIEWER_TRANSPORT_SCREENCAST
frame["viewer_transport"] = VIEWER_TRANSPORT_SCREENCAST
await self.emit_to(sid, "browser_viewer_frame", frame)
frame_sequence += 1
payload = self._frame_payload(
frame,
context_id=context_id,
viewer_id=viewer_id,
browser_id=active_id,
sequence=frame_sequence,
binary_frames=binary_frames,
)
if not slim_frames:
payload["browsers"] = browsers
payload["state"] = state
await self._emit_to_connected_viewer(sid, "browser_viewer_frame", payload)
except asyncio.CancelledError:
raise
except Exception:
@ -515,20 +550,14 @@ class WsBrowser(WsHandler):
),
)
if signature != last_signature:
await self.emit_to(
await self._emit_viewer_state(
sid,
"browser_viewer_frame",
{
"context_id": context_id,
"viewer_id": viewer_id,
"browser_id": active_id,
"browsers": browsers,
"image": "",
"mime": "",
"state": state,
"frame_source": VIEWER_TRANSPORT_SNAPSHOT,
"viewer_transport": VIEWER_TRANSPORT_SNAPSHOT,
},
context_id,
active_id,
browsers=browsers,
viewer_id=viewer_id,
state=state,
viewer_transport=VIEWER_TRANSPORT_SNAPSHOT,
)
last_signature = signature
await asyncio.sleep(SNAPSHOT_STATE_POLL_SECONDS)
@ -567,6 +596,105 @@ class WsBrowser(WsHandler):
return browser
return current_state
@staticmethod
def _state_signature(
active_id: int | str | None,
browsers: list[dict[str, Any]],
) -> tuple[str, tuple[tuple[str, str, str, str, bool], ...]]:
return (
str(active_id or ""),
tuple(
(
str(browser.get("context_id") or ""),
str(browser.get("id") or ""),
str(browser.get("currentUrl") or ""),
str(browser.get("title") or ""),
bool(browser.get("loading")),
)
for browser in browsers
),
)
@staticmethod
def _frame_payload(
frame: dict[str, Any],
*,
context_id: str,
viewer_id: str,
browser_id: int | str,
sequence: int,
binary_frames: bool,
) -> dict[str, Any]:
image = str(frame.get("image") or "")
payload: dict[str, Any] = {
"context_id": context_id,
"viewer_id": viewer_id,
"browser_id": browser_id,
"seq": sequence,
"mime": frame.get("mime") or "image/jpeg",
"frame_source": VIEWER_TRANSPORT_SCREENCAST,
"viewer_transport": VIEWER_TRANSPORT_SCREENCAST,
}
dimensions = WsBrowser._frame_dimensions(frame.get("metadata"))
if dimensions:
payload.update(dimensions)
if binary_frames:
try:
payload["image"] = base64.b64decode(image, validate=False)
payload["encoding"] = "binary"
except Exception:
payload["image"] = image
payload["encoding"] = "base64"
else:
payload["image"] = image
payload["encoding"] = "base64"
return payload
@staticmethod
def _frame_dimensions(metadata: Any) -> dict[str, int]:
if not isinstance(metadata, dict):
return {}
for width_key, height_key in (
("expectedWidth", "expectedHeight"),
("deviceWidth", "deviceHeight"),
("jpegWidth", "jpegHeight"),
):
try:
width = int(metadata.get(width_key) or 0)
height = int(metadata.get(height_key) or 0)
except (TypeError, ValueError):
continue
if width > 0 and height > 0:
return {"width": width, "height": height}
return {}
async def _emit_viewer_state(
self,
sid: str,
context_id: str,
browser_id: int | str | None,
*,
browsers: list[dict[str, Any]] | None = None,
viewer_id: str = "",
state: dict[str, Any] | None = None,
viewer_transport: str = VIEWER_TRANSPORT_SNAPSHOT,
) -> None:
await self._emit_to_connected_viewer(
sid,
"browser_viewer_state",
{
"context_id": context_id,
"active_browser_context_id": context_id,
"viewer_id": viewer_id,
"browser_id": browser_id,
"active_browser_id": browser_id,
"browsers": browsers or [],
"state": state,
"all_browsers": False,
"viewer_transport": viewer_transport,
},
)
async def _emit_empty_frame(
self,
sid: str,
@ -576,7 +704,7 @@ class WsBrowser(WsHandler):
viewer_id: str = "",
frame_source: str = "",
) -> None:
await self.emit_to(
await self._emit_to_connected_viewer(
sid,
"browser_viewer_frame",
{
@ -592,6 +720,20 @@ class WsBrowser(WsHandler):
},
)
async def _emit_to_connected_viewer(
self,
sid: str,
event: str,
data: dict[str, Any],
) -> None:
manager = getattr(self, "_manager", None)
if manager is not None:
with manager.lock:
connected = (getattr(self, "namespace", "/ws"), sid) in manager.connections
if not connected:
raise asyncio.CancelledError()
await self.emit_to(sid, event, data)
@staticmethod
def _viewer_transport(data: dict[str, Any]) -> str:
raw = (
@ -623,6 +765,20 @@ class WsBrowser(WsHandler):
"height": max(200, min(4096, height)),
}
@staticmethod
def _capture_scale_from_data(data: dict[str, Any]) -> float:
try:
scale = float(
data.get("device_pixel_ratio")
or data.get("devicePixelRatio")
or data.get("pixel_ratio")
or data.get("pixelRatio")
or 1
)
except (TypeError, ValueError):
return 1.0
return max(1.0, min(2.0, scale))
@staticmethod
def _context_id(data: dict[str, Any]) -> str:
return str(data.get("context_id") or data.get("context") or "").strip()

10
plugins/_browser/helpers/runtime.py
View file

@ -329,10 +329,14 @@ class _BrowserScreencast:
quality: int,
every_nth_frame: int,
viewport: dict[str, int],
capture_scale: float = 1.0,
) -> None:
self.session.on("Page.screencastFrame", self._on_frame)
width = max(320, min(4096, int(viewport.get("width") or DEFAULT_VIEWPORT["width"])))
height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"])))
scale = max(1.0, min(2.0, float(capture_scale or 1.0)))
max_width = max(320, min(SCREENCAST_MAX_WIDTH, int(round(width * scale))))
max_height = max(200, min(SCREENCAST_MAX_HEIGHT, int(round(height * scale))))
self._expected_width = width
self._expected_height = height
with contextlib.suppress(Exception):
@ -343,8 +347,8 @@ class _BrowserScreencast:
{
"format": "jpeg",
"quality": max(20, min(95, int(quality))),
"maxWidth": SCREENCAST_MAX_WIDTH,
"maxHeight": SCREENCAST_MAX_HEIGHT,
"maxWidth": max_width,
"maxHeight": max_height,
"everyNthFrame": max(1, int(every_nth_frame)),
},
)
@ -1616,6 +1620,7 @@ class _BrowserRuntimeCore:
*,
quality: int = 78,
every_nth_frame: int = 1,
capture_scale: float = 1.0,
) -> dict[str, Any]:
await self.ensure_started()
resolved_id = self._resolve_browser_id(browser_id)
@ -1634,6 +1639,7 @@ class _BrowserRuntimeCore:
quality=quality,
every_nth_frame=every_nth_frame,
viewport=page.viewport_size or DEFAULT_VIEWPORT,
capture_scale=capture_scale,
)
except Exception:
self.screencasts.pop(stream_id, None)

205
plugins/_browser/webui/browser-store.js
View file

@ -34,6 +34,8 @@ const ANNOTATION_DOM_LIMIT = 1200;
const ANNOTATION_TRAY_MARGIN = 10;
const BROWSER_VISUAL_SHORTCUT_KEYS = new Set(["a", "c", "insert", "v", "x", "y", "z"]);
const LOCAL_EDITABLE_SELECTOR = "input, textarea, select, [contenteditable]";
const BROWSER_BINARY_FRAMES_SUPPORTED = typeof Blob === "function"
&& typeof globalThis.URL?.createObjectURL === "function";
function makeViewerToken() {
return globalThis.crypto?.randomUUID?.()
@ -114,6 +116,31 @@ function loadFrameDimensions(src) {
});
}
function frameImageSource(data = {}) {
const image = data?.image;
if (!image) return null;
const mime = data.mime || "image/jpeg";
const isArrayBuffer = typeof ArrayBuffer !== "undefined" && image instanceof ArrayBuffer;
const isView = typeof ArrayBuffer !== "undefined" && ArrayBuffer.isView?.(image);
const isBlob = typeof Blob !== "undefined" && image instanceof Blob;
if (data.encoding === "binary" || isArrayBuffer || isView || isBlob) {
if (!BROWSER_BINARY_FRAMES_SUPPORTED) return null;
const blob = isBlob ? image : new Blob([image], { type: mime });
const src = globalThis.URL.createObjectURL(blob);
return {
src,
objectUrl: src,
cleanup: () => globalThis.URL.revokeObjectURL(src),
};
}
if (typeof image !== "string") return null;
return {
src: `data:${mime};base64,${image}`,
objectUrl: "",
cleanup: null,
};
}
const model = {
loading: true,
error: "",
@ -146,6 +173,7 @@ const model = {
_lastFrameDimensions: null,
_pendingFrameSrc: "",
_pendingFrameOptions: null,
_frameObjectUrl: "",
_frameRenderHandle: null,
_frameRenderCancel: null,
_frameRenderSequence: 0,
@ -442,7 +470,7 @@ const model = {
this.setActiveBrowserId(null);
this.address = "";
this.frameState = null;
this.frameSrc = "";
this.clearFrameSrc();
if (this.contextId) {
await this.connectViewer();
}
@ -833,7 +861,7 @@ const model = {
resetRenderedFrame() {
this.cancelFrameRender();
this.frameSrc = "";
this.clearFrameSrc();
this._lastFrameDimensions = null;
this._lastFrameAt = 0;
},
@ -915,6 +943,25 @@ const model = {
return this.viewerTransport === BROWSER_VIEWER_TRANSPORT_SCREENCAST;
},
supportsBinaryFrames() {
return BROWSER_BINARY_FRAMES_SUPPORTED;
},
captureDevicePixelRatio() {
const value = Number(globalThis.devicePixelRatio || 1);
if (!Number.isFinite(value) || value <= 1) return 1;
return Math.min(2, value);
},
frameDimensionsFromData(data = null) {
const width = Number(data?.width || 0);
const height = Number(data?.height || 0);
if (Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0) {
return { width, height };
}
return this.frameDimensionsFromMetadata(data?.metadata);
},
frameDimensionsFromMetadata(metadata = null) {
if (!metadata || typeof metadata !== "object") return null;
const width = Number(metadata.expectedWidth || metadata.deviceWidth || metadata.jpegWidth || 0);
@ -986,6 +1033,9 @@ const model = {
viewer_id: viewerToken,
create_browser: Boolean(options.createBrowser || options.create_browser),
viewer_transport: this.requestedViewerTransport(),
binary_frames: this.supportsBinaryFrames(),
slim_frames: true,
device_pixel_ratio: this.captureDevicePixelRatio(),
viewport_width: initialViewport?.width,
viewport_height: initialViewport?.height,
},
@ -1014,9 +1064,9 @@ const model = {
data.active_browser_context_id || contextId,
);
this.applySnapshot(data.snapshot);
this.connected = true;
this.browserInstallExpected = false;
},
this.connected = true;
this.browserInstallExpected = false;
},
async _bindSocketEvents() {
if (!this._frameOff) {
@ -1028,7 +1078,9 @@ const model = {
}
const incomingContextId = this.normalizeContextId(data.context_id || this.contextId);
const incomingBrowserId = this.normalizeBrowserId(data.browser_id || data.state?.id);
this.applyBrowserListing(data.browsers || [], incomingContextId, { replaceContext: true });
if (Array.isArray(data.browsers)) {
this.applyBrowserListing(data.browsers, incomingContextId, { replaceContext: true });
}
if (incomingBrowserId && !this.activeBrowserId) {
this.setActiveBrowserId(incomingBrowserId, incomingContextId);
}
@ -1046,11 +1098,15 @@ const model = {
this.address = data.state.currentUrl;
}
if (data.image) {
const frameImage = frameImageSource(data);
if (!frameImage?.src) return;
const frameBrowserId = incomingBrowserId || this.activeBrowserId;
this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`, {
this.queueFrameRender(frameImage.src, {
browserId: frameBrowserId,
contextId: incomingContextId,
dimensions: this.frameDimensionsFromMetadata(data.metadata),
dimensions: this.frameDimensionsFromData(data),
objectUrl: frameImage.objectUrl,
cleanup: frameImage.cleanup,
onAccepted: () => {
if (
this.sameBrowserId(this.switchingBrowserId, frameBrowserId)
@ -1063,13 +1119,13 @@ const model = {
});
} else if (!data.state) {
this.cancelFrameRender();
this.frameSrc = "";
this.clearFrameSrc();
}
if (!data.image && !data.state) {
if (!this.activeBrowserId) {
this.setActiveBrowserId(null, "");
this.frameState = null;
this.frameSrc = "";
this.clearFrameSrc();
}
}
this._lastFrameAt = Date.now();
@ -1077,15 +1133,20 @@ const model = {
await websocket.on("browser_viewer_frame", frameHandler);
this._frameOff = () => websocket.off("browser_viewer_frame", frameHandler);
}
if (!this._stateOff) {
const stateHandler = ({ data }) => {
if (data?.context_id !== this.contextId) return;
if (data?.viewer_id && data.viewer_id !== this._viewerToken) return;
if (data?.viewer_transport) {
this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
}
if (!this._stateOff) {
const stateHandler = ({ data }) => {
if (data?.context_id !== this.contextId) return;
if (data?.viewer_id && data.viewer_id !== this._viewerToken) return;
if (data?.viewer_transport) {
this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
}
const commandContextId = this.normalizeContextId(data.active_browser_context_id || data.context_id || this.contextId);
this.applyBrowserListing(data.browsers || [], commandContextId, { replaceAll: Boolean(data.all_browsers) });
if (Array.isArray(data.browsers)) {
this.applyBrowserListing(data.browsers, commandContextId, {
replaceAll: Boolean(data.all_browsers),
replaceContext: !data.all_browsers,
});
}
const command = String(data.command || "").toLowerCase();
const commandBrowserId = this.normalizeBrowserId(data.browser_id);
const result = data.result || {};
@ -1102,23 +1163,40 @@ const model = {
|| this.activeBrowserId
|| this.firstBrowserId(resultContextId)
);
if (
!this.activeBrowserId
|| command === "open"
|| command === "close"
|| this.sameBrowserTab(commandBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId)
) {
this.setActiveBrowserId(preferredBrowserId, resultContextId);
}
this.applyActiveFrameState(resultState || this.browserById(this.activeBrowserId, this.activeBrowserContextId));
this.applySnapshot(data.snapshot);
};
const stateBrowserId = this.normalizeBrowserId(data.active_browser_id || data.browser_id || data.state?.id);
if (
stateBrowserId
&& (
!this.activeBrowserId
|| this.sameBrowserTab(stateBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId)
)
) {
this.setActiveBrowserId(stateBrowserId, commandContextId);
}
if (
!this.activeBrowserId
|| command === "open"
|| command === "close"
|| this.sameBrowserTab(commandBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId)
) {
this.setActiveBrowserId(preferredBrowserId, resultContextId);
}
this.applyActiveFrameState(
resultState
|| data.state
|| this.browserById(this.activeBrowserId, this.activeBrowserContextId)
);
this.applySnapshot(data.snapshot);
};
await websocket.on("browser_viewer_state", stateHandler);
this._stateOff = () => websocket.off("browser_viewer_state", stateHandler);
}
},
queueFrameRender(frameSrc, options = {}) {
if (this._pendingFrameSrc) {
this.releasePendingFrame();
}
this._pendingFrameSrc = frameSrc;
this._pendingFrameOptions = options || null;
if (this._frameRenderHandle) return;
@ -1148,22 +1226,26 @@ const model = {
async renderDecodedFrame(frameSrc, options = {}, sequence = 0, surfaceSequence = this._surfaceOpenSequence) {
if (!frameSrc) {
if (sequence === this._frameRenderSequence) {
this.frameSrc = "";
this.clearFrameSrc();
}
return;
}
const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc);
if (sequence !== this._frameRenderSequence || surfaceSequence !== this._surfaceOpenSequence) {
options?.cleanup?.();
return;
}
const viewport = this.currentViewportSize() || this._lastViewport;
if (!this.frameMatchesViewport(dimensions, viewport)) {
this.requestViewportSyncAfterRejectedFrame();
if (!this.shouldAcceptMismatchedFrame(dimensions)) {
options?.cleanup?.();
return;
}
}
this.releaseRenderedFrameUrl(frameSrc);
this.frameSrc = frameSrc;
this._frameObjectUrl = options?.objectUrl || "";
this._lastFrameDimensions = dimensions;
this._lastFrameAt = Date.now();
options?.onAccepted?.();
@ -1262,9 +1344,26 @@ const model = {
}
this._frameRenderHandle = null;
this._frameRenderCancel = null;
this.releasePendingFrame();
this._frameRenderSequence += 1;
},
releasePendingFrame() {
this._pendingFrameOptions?.cleanup?.();
this._pendingFrameSrc = "";
this._pendingFrameOptions = null;
this._frameRenderSequence += 1;
},
releaseRenderedFrameUrl(nextSrc = "") {
if (this._frameObjectUrl && this._frameObjectUrl !== nextSrc) {
globalThis.URL?.revokeObjectURL?.(this._frameObjectUrl);
this._frameObjectUrl = "";
}
},
clearFrameSrc() {
this.releaseRenderedFrameUrl("");
this.frameSrc = "";
},
beginCommand() {
@ -1329,7 +1428,7 @@ const model = {
);
if (!this.activeBrowserId) {
this.frameState = null;
this.frameSrc = "";
this.clearFrameSrc();
}
if (result.state?.currentUrl || result.currentUrl) {
this.address = result.state?.currentUrl || result.currentUrl;
@ -1438,7 +1537,7 @@ const model = {
this.error = "";
this.switchingBrowserId = targetId;
this.cancelFrameRender();
this.frameSrc = "";
this.clearFrameSrc();
this.frameState = browser || null;
if (!this.addressFocused && browser?.currentUrl) {
this.address = browser.currentUrl;
@ -1643,35 +1742,35 @@ const model = {
}
},
applySnapshot(snapshot = null) {
if (!snapshot?.image) return;
const snapshotId = this.normalizeBrowserId(snapshot.browser_id || snapshot.state?.id);
applySnapshot(snapshot = null) {
if (!snapshot?.image) return;
const snapshotId = this.normalizeBrowserId(snapshot.browser_id || snapshot.state?.id);
const snapshotContextId = this.normalizeContextId(snapshot.context_id || snapshot.state?.context_id || this.activeBrowserContextId);
if (
if (
snapshotId
&& this.activeBrowserId
&& !this.sameBrowserTab(snapshotId, snapshotContextId, this.activeBrowserId, this.activeBrowserContextId)
) {
return;
}
if (snapshot.state) {
this.applyActiveFrameState(snapshot.state);
}
const frameBrowserId = snapshotId || this.activeBrowserId;
this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, {
browserId: frameBrowserId,
return;
}
if (snapshot.state) {
this.applyActiveFrameState(snapshot.state);
}
const frameBrowserId = snapshotId || this.activeBrowserId;
this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, {
browserId: frameBrowserId,
contextId: snapshotContextId,
onAccepted: () => {
if (
onAccepted: () => {
if (
this.sameBrowserId(this.switchingBrowserId, frameBrowserId)
&& this.normalizeContextId(this.activeBrowserContextId) === snapshotContextId
) {
this.switchingBrowserId = null;
}
this._surfaceSwitching = false;
},
});
},
this.switchingBrowserId = null;
}
this._surfaceSwitching = false;
},
});
},
isSwitchingBrowser() {
return Boolean(

63
tests/test_browser_agent_regressions.py
View file

@ -1158,7 +1158,8 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "def _viewer_transport(data: dict[str, Any])" in ws_browser
assert "return VIEWER_TRANSPORT_SNAPSHOT" in ws_browser
assert "self._stream_state" in ws_browser
assert "SCREENCAST_QUALITY = 92" in ws_browser
assert "SCREENCAST_STREAM_QUALITY = 80" in ws_browser
assert "SCREENSHOT_QUALITY = 92" in ws_browser
assert "initial_viewport = self._viewport_from_data(data)" in ws_browser
assert '"set_viewport"' in ws_browser
assert "start_screencast" in ws_browser
@ -1169,7 +1170,7 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "stop_screencast" in ws_browser
assert "viewer_transport == VIEWER_TRANSPORT_SCREENCAST" in ws_browser
assert '"viewer_transport": viewer_transport' in ws_browser
assert '"viewer_transport": VIEWER_TRANSPORT_SNAPSHOT' in ws_browser
assert "viewer_transport: str = VIEWER_TRANSPORT_SNAPSHOT" in ws_browser
assert '"Page.startScreencast"' in runtime
assert '"Page.screencastFrame"' in runtime
assert '"Page.screencastFrameAck"' in runtime
@ -1180,6 +1181,7 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "await self._stop_screencasts_for_browser(resolved_id)" in runtime
assert "queueFrameRender" in browser_store
assert "requestAnimationFrame" in browser_store
assert "function frameImageSource(data = {})" in browser_store
assert 'const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";' in browser_store
assert 'const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";' in browser_store
assert "viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST" in browser_store
@ -1187,11 +1189,17 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "requestedViewerTransport()" in browser_store
assert "normalizeViewerTransport(value = \"\")" in browser_store
assert "usesScreencastTransport()" in browser_store
assert "supportsBinaryFrames()" in browser_store
assert "captureDevicePixelRatio()" in browser_store
assert "frameDimensionsFromData(data = null)" in browser_store
assert "frameDimensionsFromMetadata(metadata = null)" in browser_store
assert "metadata.expectedWidth || metadata.deviceWidth || metadata.jpegWidth" in browser_store
assert "dimensions: this.frameDimensionsFromMetadata(data.metadata)" in browser_store
assert "dimensions: this.frameDimensionsFromData(data)" in browser_store
assert "const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc)" in browser_store
assert "viewer_transport: this.requestedViewerTransport()" in browser_store
assert "binary_frames: this.supportsBinaryFrames()" in browser_store
assert "slim_frames: true" in browser_store
assert "device_pixel_ratio: this.captureDevicePixelRatio()" in browser_store
assert "viewport_width: initialViewport?.width" in browser_store
assert "viewport_height: initialViewport?.height" in browser_store
assert "restart_stream: restartStream && this.usesScreencastTransport()" in browser_store
@ -1213,6 +1221,13 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "this.applySnapshot(data.snapshot);" in browser_store
assert "else if (!data.state)" in browser_store
assert '"snapshot": snapshot' in ws_browser
assert '"binary_frames": binary_frames' in ws_browser
assert '"slim_frames": slim_frames' in ws_browser
assert "capture_scale=capture_scale" in ws_browser
assert "base64.b64decode(image, validate=False)" in ws_browser
assert '"encoding"] = "binary"' in ws_browser
assert "def _frame_payload(" in ws_browser
assert "def _emit_viewer_state(" in ws_browser
assert 'const BROWSER_SNAPSHOT_META_KEY = "browser_snapshot";' in browser_tool_handler
assert "staticScreenshotUri(kvps)" in browser_tool_handler
assert "/components/modals/image-viewer/image-viewer-store.js" in browser_tool_handler
@ -1223,9 +1238,9 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "delete displayKvps[BROWSER_SNAPSHOT_META_KEY];" in browser_tool_handler
assert "startBrowserScreenshotPreview(button, image, resolveBrowserPayload)" in browser_tool_handler
assert "FRAME_FALLBACK_SCREENSHOT_SECONDS" not in ws_browser
assert '"frame_source": "state"' in ws_browser
assert '"frame_source"] = VIEWER_TRANSPORT_SCREENCAST' in ws_browser
assert '"viewer_transport"] = VIEWER_TRANSPORT_SCREENCAST' in ws_browser
assert '"browser_viewer_state"' in ws_browser
assert '"frame_source": VIEWER_TRANSPORT_SCREENCAST' in ws_browser
assert '"viewer_transport": VIEWER_TRANSPORT_SCREENCAST' in ws_browser
assert "fallback_screenshot" not in ws_browser
assert "canvas_wheel_screenshot" not in ws_browser
assert "surface_mode: this._mode" not in browser_store
@ -1234,6 +1249,29 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
assert "image-rendering: auto;" in main_html
def test_browser_viewer_frame_payload_supports_binary_slim_frames():
payload = ws_browser_module.WsBrowser._frame_payload(
{
"image": SMALL_JPEG_10X10,
"mime": "image/jpeg",
"metadata": {"expectedWidth": 900, "expectedHeight": 600},
},
context_id="ctx",
viewer_id="viewer",
browser_id=1,
sequence=3,
binary_frames=True,
)
assert payload["encoding"] == "binary"
assert payload["image"] == __import__("base64").b64decode(SMALL_JPEG_10X10)
assert payload["width"] == 900
assert payload["height"] == 600
assert payload["seq"] == 3
assert "browsers" not in payload
assert "state" not in payload
def test_browser_navigation_errors_stay_inside_native_browser_page():
runtime = (
PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"
@ -1504,7 +1542,12 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():
mime="image/jpeg",
)
await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 1118, "height": 662})
await screencast.start(
quality=92,
every_nth_frame=1,
viewport={"width": 1118, "height": 662},
capture_scale=2,
)
session.handlers["Page.screencastFrame"](
{"data": first_image, "metadata": {"deviceWidth": 10}, "sessionId": 1}
)
@ -1563,6 +1606,10 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():
for index, (method, _params) in enumerate(session.sent)
if method == "Page.startScreencast"
)
start_params = session.sent[start_index][1]
assert start_params["quality"] == 92
assert start_params["maxWidth"] == 2236
assert start_params["maxHeight"] == 1324
cdp_viewport_indices = [
index
for index, (method, _params) in enumerate(session.sent)
@ -2287,7 +2334,7 @@ async def test_browser_viewer_subscribe_returns_initial_snapshot(monkeypatch):
assert result["active_browser_id"] == 1
assert result["snapshot"]["image"] == "jpeg-data"
assert ("screenshot", (1,), {"quality": ws_browser_module.SCREENCAST_QUALITY}) in calls
assert ("screenshot", (1,), {"quality": ws_browser_module.SCREENSHOT_QUALITY}) in calls
await handler.on_disconnect("sid-snapshot")