From 91b3bc9613c7ea314623fc04c634929106d99415 Mon Sep 17 00:00:00 2001 From: Alessandro <155005371+3clyp50@users.noreply.github.com> Date: Fri, 3 Jul 2026 01:49:07 +0200 Subject: [PATCH] Slim browser viewer frame transport Negotiate binary and slim screencast frames so updated clients avoid base64-in-JSON and repeated viewer metadata on every frame. Clamp screencast capture to the subscriber viewport and DPR, lower streaming JPEG quality while preserving snapshot quality, and teach the client to manage binary frame URLs safely. Document the browser viewer performance plan and update regression coverage for the new transport contract. --- docs/AGENTS.md | 1 + plugins/_browser/AGENTS.md | 1 + plugins/_browser/api/ws_browser.py | 244 +++++++++++++++++++----- plugins/_browser/helpers/runtime.py | 10 +- plugins/_browser/webui/browser-store.js | 205 +++++++++++++++----- tests/test_browser_agent_regressions.py | 63 +++++- 6 files changed, 417 insertions(+), 107 deletions(-) diff --git a/docs/AGENTS.md b/docs/AGENTS.md index 3f3002b01..c81fb55bd 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -9,6 +9,7 @@ - `README.md`, `quickstart.md`, `guides/`, and `setup/` cover user-facing setup and workflows. - `developer/` covers compact developer references and source handoffs. +- `plans/` covers implementation plans, migration notes, and staged technical roadmaps. - `res/` contains documentation images and other documentation assets. ## Local Contracts diff --git a/plugins/_browser/AGENTS.md b/plugins/_browser/AGENTS.md index 15a7ac85c..e3f2a137e 100644 --- a/plugins/_browser/AGENTS.md +++ b/plugins/_browser/AGENTS.md @@ -19,6 +19,7 @@ - Preserve Playwright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding. - Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation. - Default the visible WebUI Browser to live CDP screencast for responsiveness. Keep lightweight CDP/DOM state snapshots as the fallback transport. +- Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback. - Keep narrow WebUI Browser controls usable by grouping navigation with Annotate/settings above a full-width address bar. - Browser URL-intent handling must only claim web URL schemes and leave custom Agent Zero schemes to their owning surfaces. - Prefer DOM/CDP browser actions with refs, selectors, frame-chain refs, and screenshots over viewport coordinate input. Coordinates remain a visual fallback. diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.py index ec5e137b3..827e91df4 100644 --- a/plugins/_browser/api/ws_browser.py +++ b/plugins/_browser/api/ws_browser.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import base64 import contextlib import time from typing import Any, ClassVar @@ -15,7 +16,8 @@ FRAME_READ_TIMEOUT_SECONDS = 0.5 FRAME_RETRY_DELAY_SECONDS = 0.5 FRAME_STATE_REFRESH_SECONDS = 0.75 SNAPSHOT_STATE_POLL_SECONDS = 0.75 -SCREENCAST_QUALITY = 92 +SCREENCAST_STREAM_QUALITY = 80 +SCREENSHOT_QUALITY = 92 VIEWER_TRANSPORT_SCREENCAST = "screencast" VIEWER_TRANSPORT_SNAPSHOT = "snapshot" VIEWER_TRANSPORTS = {VIEWER_TRANSPORT_SCREENCAST, VIEWER_TRANSPORT_SNAPSHOT} @@ -97,10 +99,21 @@ class WsBrowser(WsHandler): existing.cancel() viewer_id = str(data.get("viewer_id") or "") viewer_transport = self._viewer_transport(data) + binary_frames = self._bool(data.get("binary_frames", data.get("binaryFrames"))) + slim_frames = self._bool(data.get("slim_frames", data.get("slimFrames", binary_frames))) + capture_scale = self._capture_scale_from_data(data) snapshot = None if runtime: if viewer_transport == VIEWER_TRANSPORT_SCREENCAST: - stream_task = self._stream_frames(sid, context_id, active_id, viewer_id) + stream_task = self._stream_frames( + sid, + context_id, + active_id, + viewer_id, + binary_frames=binary_frames, + slim_frames=slim_frames, + capture_scale=capture_scale, + ) else: stream_task = self._stream_state(sid, context_id, active_id, viewer_id) self._streams[stream_key] = asyncio.create_task(stream_task) @@ -115,6 +128,8 @@ class WsBrowser(WsHandler): "all_browsers": True, "viewer_id": viewer_id, "viewer_transport": viewer_transport, + "binary_frames": binary_frames, + "slim_frames": slim_frames, } def _unsubscribe(self, data: dict[str, Any], sid: str) -> dict[str, Any] | WsResult: @@ -157,9 +172,9 @@ class WsBrowser(WsHandler): snapshot = None if active_id: try: - quality = int(data.get("quality") or SCREENCAST_QUALITY) + quality = int(data.get("quality") or SCREENSHOT_QUALITY) except (TypeError, ValueError): - quality = SCREENCAST_QUALITY + quality = SCREENSHOT_QUALITY with contextlib.suppress(Exception): snapshot = await runtime.call( "screenshot", @@ -347,7 +362,7 @@ class WsBrowser(WsHandler): if not browser_id: return None with contextlib.suppress(Exception): - return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY) + return await runtime.call("screenshot", browser_id, quality=SCREENSHOT_QUALITY) return None async def _snapshot_for_browser( @@ -358,7 +373,7 @@ class WsBrowser(WsHandler): if not browser_id: return None with contextlib.suppress(Exception): - return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY) + return await runtime.call("screenshot", browser_id, quality=SCREENSHOT_QUALITY) return None async def _all_browser_tabs(self) -> list[dict[str, Any]]: @@ -377,6 +392,10 @@ class WsBrowser(WsHandler): context_id: str, browser_id: int | str | None, viewer_id: str = "", + *, + binary_frames: bool = False, + slim_frames: bool = False, + capture_scale: float = 1.0, ) -> None: runtime = None stream_id = None @@ -397,12 +416,14 @@ class WsBrowser(WsHandler): browsers = listing.get("browsers") or [] active_id = self._active_browser_id(listing, browser_id) if not active_id: - await self._emit_empty_frame( + await self._emit_viewer_state( sid, context_id, + active_id, browsers=browsers, viewer_id=viewer_id, - frame_source=VIEWER_TRANSPORT_SCREENCAST, + state=None, + viewer_transport=VIEWER_TRANSPORT_SCREENCAST, ) await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS) continue @@ -410,29 +431,26 @@ class WsBrowser(WsHandler): screencast = await runtime.call( "start_screencast", active_id, - quality=SCREENCAST_QUALITY, + quality=SCREENCAST_STREAM_QUALITY, every_nth_frame=1, + capture_scale=capture_scale, ) stream_id = screencast["stream_id"] active_id = screencast["browser_id"] state = screencast.get("state") - await self.emit_to( + await self._emit_viewer_state( sid, - "browser_viewer_frame", - { - "context_id": context_id, - "viewer_id": viewer_id, - "browser_id": active_id, - "browsers": browsers, - "image": "", - "mime": "", - "state": state, - "frame_source": "state", - "viewer_transport": VIEWER_TRANSPORT_SCREENCAST, - }, + context_id, + active_id, + browsers=browsers, + viewer_id=viewer_id, + state=state, + viewer_transport=VIEWER_TRANSPORT_SCREENCAST, ) last_state_refresh = 0.0 + last_state_signature = self._state_signature(active_id, browsers) + frame_sequence = 0 while True: now = time.monotonic() if now - last_state_refresh >= FRAME_STATE_REFRESH_SECONDS: @@ -442,6 +460,18 @@ class WsBrowser(WsHandler): if str(active_id) not in browser_ids: break state = self._state_for_browser(browsers, active_id, state) + state_signature = self._state_signature(active_id, browsers) + if state_signature != last_state_signature: + await self._emit_viewer_state( + sid, + context_id, + active_id, + browsers=browsers, + viewer_id=viewer_id, + state=state, + viewer_transport=VIEWER_TRANSPORT_SCREENCAST, + ) + last_state_signature = state_signature last_state_refresh = now try: @@ -455,14 +485,19 @@ class WsBrowser(WsHandler): except TimeoutError: continue - frame["context_id"] = context_id - frame["viewer_id"] = viewer_id - frame["browser_id"] = active_id - frame["browsers"] = browsers - frame["state"] = state - frame["frame_source"] = VIEWER_TRANSPORT_SCREENCAST - frame["viewer_transport"] = VIEWER_TRANSPORT_SCREENCAST - await self.emit_to(sid, "browser_viewer_frame", frame) + frame_sequence += 1 + payload = self._frame_payload( + frame, + context_id=context_id, + viewer_id=viewer_id, + browser_id=active_id, + sequence=frame_sequence, + binary_frames=binary_frames, + ) + if not slim_frames: + payload["browsers"] = browsers + payload["state"] = state + await self._emit_to_connected_viewer(sid, "browser_viewer_frame", payload) except asyncio.CancelledError: raise except Exception: @@ -515,20 +550,14 @@ class WsBrowser(WsHandler): ), ) if signature != last_signature: - await self.emit_to( + await self._emit_viewer_state( sid, - "browser_viewer_frame", - { - "context_id": context_id, - "viewer_id": viewer_id, - "browser_id": active_id, - "browsers": browsers, - "image": "", - "mime": "", - "state": state, - "frame_source": VIEWER_TRANSPORT_SNAPSHOT, - "viewer_transport": VIEWER_TRANSPORT_SNAPSHOT, - }, + context_id, + active_id, + browsers=browsers, + viewer_id=viewer_id, + state=state, + viewer_transport=VIEWER_TRANSPORT_SNAPSHOT, ) last_signature = signature await asyncio.sleep(SNAPSHOT_STATE_POLL_SECONDS) @@ -567,6 +596,105 @@ class WsBrowser(WsHandler): return browser return current_state + @staticmethod + def _state_signature( + active_id: int | str | None, + browsers: list[dict[str, Any]], + ) -> tuple[str, tuple[tuple[str, str, str, str, bool], ...]]: + return ( + str(active_id or ""), + tuple( + ( + str(browser.get("context_id") or ""), + str(browser.get("id") or ""), + str(browser.get("currentUrl") or ""), + str(browser.get("title") or ""), + bool(browser.get("loading")), + ) + for browser in browsers + ), + ) + + @staticmethod + def _frame_payload( + frame: dict[str, Any], + *, + context_id: str, + viewer_id: str, + browser_id: int | str, + sequence: int, + binary_frames: bool, + ) -> dict[str, Any]: + image = str(frame.get("image") or "") + payload: dict[str, Any] = { + "context_id": context_id, + "viewer_id": viewer_id, + "browser_id": browser_id, + "seq": sequence, + "mime": frame.get("mime") or "image/jpeg", + "frame_source": VIEWER_TRANSPORT_SCREENCAST, + "viewer_transport": VIEWER_TRANSPORT_SCREENCAST, + } + dimensions = WsBrowser._frame_dimensions(frame.get("metadata")) + if dimensions: + payload.update(dimensions) + if binary_frames: + try: + payload["image"] = base64.b64decode(image, validate=False) + payload["encoding"] = "binary" + except Exception: + payload["image"] = image + payload["encoding"] = "base64" + else: + payload["image"] = image + payload["encoding"] = "base64" + return payload + + @staticmethod + def _frame_dimensions(metadata: Any) -> dict[str, int]: + if not isinstance(metadata, dict): + return {} + for width_key, height_key in ( + ("expectedWidth", "expectedHeight"), + ("deviceWidth", "deviceHeight"), + ("jpegWidth", "jpegHeight"), + ): + try: + width = int(metadata.get(width_key) or 0) + height = int(metadata.get(height_key) or 0) + except (TypeError, ValueError): + continue + if width > 0 and height > 0: + return {"width": width, "height": height} + return {} + + async def _emit_viewer_state( + self, + sid: str, + context_id: str, + browser_id: int | str | None, + *, + browsers: list[dict[str, Any]] | None = None, + viewer_id: str = "", + state: dict[str, Any] | None = None, + viewer_transport: str = VIEWER_TRANSPORT_SNAPSHOT, + ) -> None: + await self._emit_to_connected_viewer( + sid, + "browser_viewer_state", + { + "context_id": context_id, + "active_browser_context_id": context_id, + "viewer_id": viewer_id, + "browser_id": browser_id, + "active_browser_id": browser_id, + "browsers": browsers or [], + "state": state, + "all_browsers": False, + "viewer_transport": viewer_transport, + }, + ) + async def _emit_empty_frame( self, sid: str, @@ -576,7 +704,7 @@ class WsBrowser(WsHandler): viewer_id: str = "", frame_source: str = "", ) -> None: - await self.emit_to( + await self._emit_to_connected_viewer( sid, "browser_viewer_frame", { @@ -592,6 +720,20 @@ class WsBrowser(WsHandler): }, ) + async def _emit_to_connected_viewer( + self, + sid: str, + event: str, + data: dict[str, Any], + ) -> None: + manager = getattr(self, "_manager", None) + if manager is not None: + with manager.lock: + connected = (getattr(self, "namespace", "/ws"), sid) in manager.connections + if not connected: + raise asyncio.CancelledError() + await self.emit_to(sid, event, data) + @staticmethod def _viewer_transport(data: dict[str, Any]) -> str: raw = ( @@ -623,6 +765,20 @@ class WsBrowser(WsHandler): "height": max(200, min(4096, height)), } + @staticmethod + def _capture_scale_from_data(data: dict[str, Any]) -> float: + try: + scale = float( + data.get("device_pixel_ratio") + or data.get("devicePixelRatio") + or data.get("pixel_ratio") + or data.get("pixelRatio") + or 1 + ) + except (TypeError, ValueError): + return 1.0 + return max(1.0, min(2.0, scale)) + @staticmethod def _context_id(data: dict[str, Any]) -> str: return str(data.get("context_id") or data.get("context") or "").strip() diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py index e7455d81a..94524071d 100644 --- a/plugins/_browser/helpers/runtime.py +++ b/plugins/_browser/helpers/runtime.py @@ -329,10 +329,14 @@ class _BrowserScreencast: quality: int, every_nth_frame: int, viewport: dict[str, int], + capture_scale: float = 1.0, ) -> None: self.session.on("Page.screencastFrame", self._on_frame) width = max(320, min(4096, int(viewport.get("width") or DEFAULT_VIEWPORT["width"]))) height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"]))) + scale = max(1.0, min(2.0, float(capture_scale or 1.0))) + max_width = max(320, min(SCREENCAST_MAX_WIDTH, int(round(width * scale)))) + max_height = max(200, min(SCREENCAST_MAX_HEIGHT, int(round(height * scale)))) self._expected_width = width self._expected_height = height with contextlib.suppress(Exception): @@ -343,8 +347,8 @@ class _BrowserScreencast: { "format": "jpeg", "quality": max(20, min(95, int(quality))), - "maxWidth": SCREENCAST_MAX_WIDTH, - "maxHeight": SCREENCAST_MAX_HEIGHT, + "maxWidth": max_width, + "maxHeight": max_height, "everyNthFrame": max(1, int(every_nth_frame)), }, ) @@ -1616,6 +1620,7 @@ class _BrowserRuntimeCore: *, quality: int = 78, every_nth_frame: int = 1, + capture_scale: float = 1.0, ) -> dict[str, Any]: await self.ensure_started() resolved_id = self._resolve_browser_id(browser_id) @@ -1634,6 +1639,7 @@ class _BrowserRuntimeCore: quality=quality, every_nth_frame=every_nth_frame, viewport=page.viewport_size or DEFAULT_VIEWPORT, + capture_scale=capture_scale, ) except Exception: self.screencasts.pop(stream_id, None) diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.js index 6e70a1891..a4b3d4525 100644 --- a/plugins/_browser/webui/browser-store.js +++ b/plugins/_browser/webui/browser-store.js @@ -34,6 +34,8 @@ const ANNOTATION_DOM_LIMIT = 1200; const ANNOTATION_TRAY_MARGIN = 10; const BROWSER_VISUAL_SHORTCUT_KEYS = new Set(["a", "c", "insert", "v", "x", "y", "z"]); const LOCAL_EDITABLE_SELECTOR = "input, textarea, select, [contenteditable]"; +const BROWSER_BINARY_FRAMES_SUPPORTED = typeof Blob === "function" + && typeof globalThis.URL?.createObjectURL === "function"; function makeViewerToken() { return globalThis.crypto?.randomUUID?.() @@ -114,6 +116,31 @@ function loadFrameDimensions(src) { }); } +function frameImageSource(data = {}) { + const image = data?.image; + if (!image) return null; + const mime = data.mime || "image/jpeg"; + const isArrayBuffer = typeof ArrayBuffer !== "undefined" && image instanceof ArrayBuffer; + const isView = typeof ArrayBuffer !== "undefined" && ArrayBuffer.isView?.(image); + const isBlob = typeof Blob !== "undefined" && image instanceof Blob; + if (data.encoding === "binary" || isArrayBuffer || isView || isBlob) { + if (!BROWSER_BINARY_FRAMES_SUPPORTED) return null; + const blob = isBlob ? image : new Blob([image], { type: mime }); + const src = globalThis.URL.createObjectURL(blob); + return { + src, + objectUrl: src, + cleanup: () => globalThis.URL.revokeObjectURL(src), + }; + } + if (typeof image !== "string") return null; + return { + src: `data:${mime};base64,${image}`, + objectUrl: "", + cleanup: null, + }; +} + const model = { loading: true, error: "", @@ -146,6 +173,7 @@ const model = { _lastFrameDimensions: null, _pendingFrameSrc: "", _pendingFrameOptions: null, + _frameObjectUrl: "", _frameRenderHandle: null, _frameRenderCancel: null, _frameRenderSequence: 0, @@ -442,7 +470,7 @@ const model = { this.setActiveBrowserId(null); this.address = ""; this.frameState = null; - this.frameSrc = ""; + this.clearFrameSrc(); if (this.contextId) { await this.connectViewer(); } @@ -833,7 +861,7 @@ const model = { resetRenderedFrame() { this.cancelFrameRender(); - this.frameSrc = ""; + this.clearFrameSrc(); this._lastFrameDimensions = null; this._lastFrameAt = 0; }, @@ -915,6 +943,25 @@ const model = { return this.viewerTransport === BROWSER_VIEWER_TRANSPORT_SCREENCAST; }, + supportsBinaryFrames() { + return BROWSER_BINARY_FRAMES_SUPPORTED; + }, + + captureDevicePixelRatio() { + const value = Number(globalThis.devicePixelRatio || 1); + if (!Number.isFinite(value) || value <= 1) return 1; + return Math.min(2, value); + }, + + frameDimensionsFromData(data = null) { + const width = Number(data?.width || 0); + const height = Number(data?.height || 0); + if (Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0) { + return { width, height }; + } + return this.frameDimensionsFromMetadata(data?.metadata); + }, + frameDimensionsFromMetadata(metadata = null) { if (!metadata || typeof metadata !== "object") return null; const width = Number(metadata.expectedWidth || metadata.deviceWidth || metadata.jpegWidth || 0); @@ -986,6 +1033,9 @@ const model = { viewer_id: viewerToken, create_browser: Boolean(options.createBrowser || options.create_browser), viewer_transport: this.requestedViewerTransport(), + binary_frames: this.supportsBinaryFrames(), + slim_frames: true, + device_pixel_ratio: this.captureDevicePixelRatio(), viewport_width: initialViewport?.width, viewport_height: initialViewport?.height, }, @@ -1014,9 +1064,9 @@ const model = { data.active_browser_context_id || contextId, ); this.applySnapshot(data.snapshot); - this.connected = true; - this.browserInstallExpected = false; - }, + this.connected = true; + this.browserInstallExpected = false; + }, async _bindSocketEvents() { if (!this._frameOff) { @@ -1028,7 +1078,9 @@ const model = { } const incomingContextId = this.normalizeContextId(data.context_id || this.contextId); const incomingBrowserId = this.normalizeBrowserId(data.browser_id || data.state?.id); - this.applyBrowserListing(data.browsers || [], incomingContextId, { replaceContext: true }); + if (Array.isArray(data.browsers)) { + this.applyBrowserListing(data.browsers, incomingContextId, { replaceContext: true }); + } if (incomingBrowserId && !this.activeBrowserId) { this.setActiveBrowserId(incomingBrowserId, incomingContextId); } @@ -1046,11 +1098,15 @@ const model = { this.address = data.state.currentUrl; } if (data.image) { + const frameImage = frameImageSource(data); + if (!frameImage?.src) return; const frameBrowserId = incomingBrowserId || this.activeBrowserId; - this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`, { + this.queueFrameRender(frameImage.src, { browserId: frameBrowserId, contextId: incomingContextId, - dimensions: this.frameDimensionsFromMetadata(data.metadata), + dimensions: this.frameDimensionsFromData(data), + objectUrl: frameImage.objectUrl, + cleanup: frameImage.cleanup, onAccepted: () => { if ( this.sameBrowserId(this.switchingBrowserId, frameBrowserId) @@ -1063,13 +1119,13 @@ const model = { }); } else if (!data.state) { this.cancelFrameRender(); - this.frameSrc = ""; + this.clearFrameSrc(); } if (!data.image && !data.state) { if (!this.activeBrowserId) { this.setActiveBrowserId(null, ""); this.frameState = null; - this.frameSrc = ""; + this.clearFrameSrc(); } } this._lastFrameAt = Date.now(); @@ -1077,15 +1133,20 @@ const model = { await websocket.on("browser_viewer_frame", frameHandler); this._frameOff = () => websocket.off("browser_viewer_frame", frameHandler); } - if (!this._stateOff) { - const stateHandler = ({ data }) => { - if (data?.context_id !== this.contextId) return; - if (data?.viewer_id && data.viewer_id !== this._viewerToken) return; - if (data?.viewer_transport) { - this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport); - } + if (!this._stateOff) { + const stateHandler = ({ data }) => { + if (data?.context_id !== this.contextId) return; + if (data?.viewer_id && data.viewer_id !== this._viewerToken) return; + if (data?.viewer_transport) { + this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport); + } const commandContextId = this.normalizeContextId(data.active_browser_context_id || data.context_id || this.contextId); - this.applyBrowserListing(data.browsers || [], commandContextId, { replaceAll: Boolean(data.all_browsers) }); + if (Array.isArray(data.browsers)) { + this.applyBrowserListing(data.browsers, commandContextId, { + replaceAll: Boolean(data.all_browsers), + replaceContext: !data.all_browsers, + }); + } const command = String(data.command || "").toLowerCase(); const commandBrowserId = this.normalizeBrowserId(data.browser_id); const result = data.result || {}; @@ -1102,23 +1163,40 @@ const model = { || this.activeBrowserId || this.firstBrowserId(resultContextId) ); - if ( - !this.activeBrowserId - || command === "open" - || command === "close" - || this.sameBrowserTab(commandBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId) - ) { - this.setActiveBrowserId(preferredBrowserId, resultContextId); - } - this.applyActiveFrameState(resultState || this.browserById(this.activeBrowserId, this.activeBrowserContextId)); - this.applySnapshot(data.snapshot); - }; + const stateBrowserId = this.normalizeBrowserId(data.active_browser_id || data.browser_id || data.state?.id); + if ( + stateBrowserId + && ( + !this.activeBrowserId + || this.sameBrowserTab(stateBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId) + ) + ) { + this.setActiveBrowserId(stateBrowserId, commandContextId); + } + if ( + !this.activeBrowserId + || command === "open" + || command === "close" + || this.sameBrowserTab(commandBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId) + ) { + this.setActiveBrowserId(preferredBrowserId, resultContextId); + } + this.applyActiveFrameState( + resultState + || data.state + || this.browserById(this.activeBrowserId, this.activeBrowserContextId) + ); + this.applySnapshot(data.snapshot); + }; await websocket.on("browser_viewer_state", stateHandler); this._stateOff = () => websocket.off("browser_viewer_state", stateHandler); } }, queueFrameRender(frameSrc, options = {}) { + if (this._pendingFrameSrc) { + this.releasePendingFrame(); + } this._pendingFrameSrc = frameSrc; this._pendingFrameOptions = options || null; if (this._frameRenderHandle) return; @@ -1148,22 +1226,26 @@ const model = { async renderDecodedFrame(frameSrc, options = {}, sequence = 0, surfaceSequence = this._surfaceOpenSequence) { if (!frameSrc) { if (sequence === this._frameRenderSequence) { - this.frameSrc = ""; + this.clearFrameSrc(); } return; } const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc); if (sequence !== this._frameRenderSequence || surfaceSequence !== this._surfaceOpenSequence) { + options?.cleanup?.(); return; } const viewport = this.currentViewportSize() || this._lastViewport; if (!this.frameMatchesViewport(dimensions, viewport)) { this.requestViewportSyncAfterRejectedFrame(); if (!this.shouldAcceptMismatchedFrame(dimensions)) { + options?.cleanup?.(); return; } } + this.releaseRenderedFrameUrl(frameSrc); this.frameSrc = frameSrc; + this._frameObjectUrl = options?.objectUrl || ""; this._lastFrameDimensions = dimensions; this._lastFrameAt = Date.now(); options?.onAccepted?.(); @@ -1262,9 +1344,26 @@ const model = { } this._frameRenderHandle = null; this._frameRenderCancel = null; + this.releasePendingFrame(); + this._frameRenderSequence += 1; + }, + + releasePendingFrame() { + this._pendingFrameOptions?.cleanup?.(); this._pendingFrameSrc = ""; this._pendingFrameOptions = null; - this._frameRenderSequence += 1; + }, + + releaseRenderedFrameUrl(nextSrc = "") { + if (this._frameObjectUrl && this._frameObjectUrl !== nextSrc) { + globalThis.URL?.revokeObjectURL?.(this._frameObjectUrl); + this._frameObjectUrl = ""; + } + }, + + clearFrameSrc() { + this.releaseRenderedFrameUrl(""); + this.frameSrc = ""; }, beginCommand() { @@ -1329,7 +1428,7 @@ const model = { ); if (!this.activeBrowserId) { this.frameState = null; - this.frameSrc = ""; + this.clearFrameSrc(); } if (result.state?.currentUrl || result.currentUrl) { this.address = result.state?.currentUrl || result.currentUrl; @@ -1438,7 +1537,7 @@ const model = { this.error = ""; this.switchingBrowserId = targetId; this.cancelFrameRender(); - this.frameSrc = ""; + this.clearFrameSrc(); this.frameState = browser || null; if (!this.addressFocused && browser?.currentUrl) { this.address = browser.currentUrl; @@ -1643,35 +1742,35 @@ const model = { } }, - applySnapshot(snapshot = null) { - if (!snapshot?.image) return; - const snapshotId = this.normalizeBrowserId(snapshot.browser_id || snapshot.state?.id); + applySnapshot(snapshot = null) { + if (!snapshot?.image) return; + const snapshotId = this.normalizeBrowserId(snapshot.browser_id || snapshot.state?.id); const snapshotContextId = this.normalizeContextId(snapshot.context_id || snapshot.state?.context_id || this.activeBrowserContextId); - if ( + if ( snapshotId && this.activeBrowserId && !this.sameBrowserTab(snapshotId, snapshotContextId, this.activeBrowserId, this.activeBrowserContextId) ) { - return; - } - if (snapshot.state) { - this.applyActiveFrameState(snapshot.state); - } - const frameBrowserId = snapshotId || this.activeBrowserId; - this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, { - browserId: frameBrowserId, + return; + } + if (snapshot.state) { + this.applyActiveFrameState(snapshot.state); + } + const frameBrowserId = snapshotId || this.activeBrowserId; + this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, { + browserId: frameBrowserId, contextId: snapshotContextId, - onAccepted: () => { - if ( + onAccepted: () => { + if ( this.sameBrowserId(this.switchingBrowserId, frameBrowserId) && this.normalizeContextId(this.activeBrowserContextId) === snapshotContextId ) { - this.switchingBrowserId = null; - } - this._surfaceSwitching = false; - }, - }); - }, + this.switchingBrowserId = null; + } + this._surfaceSwitching = false; + }, + }); + }, isSwitchingBrowser() { return Boolean( diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py index eef2e141b..7503efeb3 100644 --- a/tests/test_browser_agent_regressions.py +++ b/tests/test_browser_agent_regressions.py @@ -1158,7 +1158,8 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback(): assert "def _viewer_transport(data: dict[str, Any])" in ws_browser assert "return VIEWER_TRANSPORT_SNAPSHOT" in ws_browser assert "self._stream_state" in ws_browser - assert "SCREENCAST_QUALITY = 92" in ws_browser + assert "SCREENCAST_STREAM_QUALITY = 80" in ws_browser + assert "SCREENSHOT_QUALITY = 92" in ws_browser assert "initial_viewport = self._viewport_from_data(data)" in ws_browser assert '"set_viewport"' in ws_browser assert "start_screencast" in ws_browser @@ -1169,7 +1170,7 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback(): assert "stop_screencast" in ws_browser assert "viewer_transport == VIEWER_TRANSPORT_SCREENCAST" in ws_browser assert '"viewer_transport": viewer_transport' in ws_browser - assert '"viewer_transport": VIEWER_TRANSPORT_SNAPSHOT' in ws_browser + assert "viewer_transport: str = VIEWER_TRANSPORT_SNAPSHOT" in ws_browser assert '"Page.startScreencast"' in runtime assert '"Page.screencastFrame"' in runtime assert '"Page.screencastFrameAck"' in runtime @@ -1180,6 +1181,7 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback(): assert "await self._stop_screencasts_for_browser(resolved_id)" in runtime assert "queueFrameRender" in browser_store assert "requestAnimationFrame" in browser_store + assert "function frameImageSource(data = {})" in browser_store assert 'const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";' in browser_store assert 'const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";' in browser_store assert "viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST" in browser_store @@ -1187,11 +1189,17 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback(): assert "requestedViewerTransport()" in browser_store assert "normalizeViewerTransport(value = \"\")" in browser_store assert "usesScreencastTransport()" in browser_store + assert "supportsBinaryFrames()" in browser_store + assert "captureDevicePixelRatio()" in browser_store + assert "frameDimensionsFromData(data = null)" in browser_store assert "frameDimensionsFromMetadata(metadata = null)" in browser_store assert "metadata.expectedWidth || metadata.deviceWidth || metadata.jpegWidth" in browser_store - assert "dimensions: this.frameDimensionsFromMetadata(data.metadata)" in browser_store + assert "dimensions: this.frameDimensionsFromData(data)" in browser_store assert "const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc)" in browser_store assert "viewer_transport: this.requestedViewerTransport()" in browser_store + assert "binary_frames: this.supportsBinaryFrames()" in browser_store + assert "slim_frames: true" in browser_store + assert "device_pixel_ratio: this.captureDevicePixelRatio()" in browser_store assert "viewport_width: initialViewport?.width" in browser_store assert "viewport_height: initialViewport?.height" in browser_store assert "restart_stream: restartStream && this.usesScreencastTransport()" in browser_store @@ -1213,6 +1221,13 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback(): assert "this.applySnapshot(data.snapshot);" in browser_store assert "else if (!data.state)" in browser_store assert '"snapshot": snapshot' in ws_browser + assert '"binary_frames": binary_frames' in ws_browser + assert '"slim_frames": slim_frames' in ws_browser + assert "capture_scale=capture_scale" in ws_browser + assert "base64.b64decode(image, validate=False)" in ws_browser + assert '"encoding"] = "binary"' in ws_browser + assert "def _frame_payload(" in ws_browser + assert "def _emit_viewer_state(" in ws_browser assert 'const BROWSER_SNAPSHOT_META_KEY = "browser_snapshot";' in browser_tool_handler assert "staticScreenshotUri(kvps)" in browser_tool_handler assert "/components/modals/image-viewer/image-viewer-store.js" in browser_tool_handler @@ -1223,9 +1238,9 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback(): assert "delete displayKvps[BROWSER_SNAPSHOT_META_KEY];" in browser_tool_handler assert "startBrowserScreenshotPreview(button, image, resolveBrowserPayload)" in browser_tool_handler assert "FRAME_FALLBACK_SCREENSHOT_SECONDS" not in ws_browser - assert '"frame_source": "state"' in ws_browser - assert '"frame_source"] = VIEWER_TRANSPORT_SCREENCAST' in ws_browser - assert '"viewer_transport"] = VIEWER_TRANSPORT_SCREENCAST' in ws_browser + assert '"browser_viewer_state"' in ws_browser + assert '"frame_source": VIEWER_TRANSPORT_SCREENCAST' in ws_browser + assert '"viewer_transport": VIEWER_TRANSPORT_SCREENCAST' in ws_browser assert "fallback_screenshot" not in ws_browser assert "canvas_wheel_screenshot" not in ws_browser assert "surface_mode: this._mode" not in browser_store @@ -1234,6 +1249,29 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback(): assert "image-rendering: auto;" in main_html +def test_browser_viewer_frame_payload_supports_binary_slim_frames(): + payload = ws_browser_module.WsBrowser._frame_payload( + { + "image": SMALL_JPEG_10X10, + "mime": "image/jpeg", + "metadata": {"expectedWidth": 900, "expectedHeight": 600}, + }, + context_id="ctx", + viewer_id="viewer", + browser_id=1, + sequence=3, + binary_frames=True, + ) + + assert payload["encoding"] == "binary" + assert payload["image"] == __import__("base64").b64decode(SMALL_JPEG_10X10) + assert payload["width"] == 900 + assert payload["height"] == 600 + assert payload["seq"] == 3 + assert "browsers" not in payload + assert "state" not in payload + + def test_browser_navigation_errors_stay_inside_native_browser_page(): runtime = ( PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py" @@ -1504,7 +1542,12 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames(): mime="image/jpeg", ) - await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 1118, "height": 662}) + await screencast.start( + quality=92, + every_nth_frame=1, + viewport={"width": 1118, "height": 662}, + capture_scale=2, + ) session.handlers["Page.screencastFrame"]( {"data": first_image, "metadata": {"deviceWidth": 10}, "sessionId": 1} ) @@ -1563,6 +1606,10 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames(): for index, (method, _params) in enumerate(session.sent) if method == "Page.startScreencast" ) + start_params = session.sent[start_index][1] + assert start_params["quality"] == 92 + assert start_params["maxWidth"] == 2236 + assert start_params["maxHeight"] == 1324 cdp_viewport_indices = [ index for index, (method, _params) in enumerate(session.sent) @@ -2287,7 +2334,7 @@ async def test_browser_viewer_subscribe_returns_initial_snapshot(monkeypatch): assert result["active_browser_id"] == 1 assert result["snapshot"]["image"] == "jpeg-data" - assert ("screenshot", (1,), {"quality": ws_browser_module.SCREENCAST_QUALITY}) in calls + assert ("screenshot", (1,), {"quality": ws_browser_module.SCREENSHOT_QUALITY}) in calls await handler.on_disconnect("sid-snapshot")