diff --git a/docs/AGENTS.md b/docs/AGENTS.md index 3f3002b01..c81fb55bd 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -9,6 +9,7 @@ - `README.md`, `quickstart.md`, `guides/`, and `setup/` cover user-facing setup and workflows. - `developer/` covers compact developer references and source handoffs. +- `plans/` covers implementation plans, migration notes, and staged technical roadmaps. - `res/` contains documentation images and other documentation assets. ## Local Contracts diff --git a/plugins/_browser/AGENTS.md b/plugins/_browser/AGENTS.md index 15a7ac85c..e3f2a137e 100644 --- a/plugins/_browser/AGENTS.md +++ b/plugins/_browser/AGENTS.md @@ -19,6 +19,7 @@ - Preserve Playwright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding. - Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation. - Default the visible WebUI Browser to live CDP screencast for responsiveness. Keep lightweight CDP/DOM state snapshots as the fallback transport. +- Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback. - Keep narrow WebUI Browser controls usable by grouping navigation with Annotate/settings above a full-width address bar. - Browser URL-intent handling must only claim web URL schemes and leave custom Agent Zero schemes to their owning surfaces. - Prefer DOM/CDP browser actions with refs, selectors, frame-chain refs, and screenshots over viewport coordinate input. Coordinates remain a visual fallback. diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.py index ec5e137b3..827e91df4 100644 --- a/plugins/_browser/api/ws_browser.py +++ b/plugins/_browser/api/ws_browser.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import base64 import contextlib import time from typing import Any, ClassVar @@ -15,7 +16,8 @@ FRAME_READ_TIMEOUT_SECONDS = 0.5 FRAME_RETRY_DELAY_SECONDS = 0.5 FRAME_STATE_REFRESH_SECONDS = 0.75 SNAPSHOT_STATE_POLL_SECONDS = 0.75 -SCREENCAST_QUALITY = 92 +SCREENCAST_STREAM_QUALITY = 80 +SCREENSHOT_QUALITY = 92 VIEWER_TRANSPORT_SCREENCAST = "screencast" VIEWER_TRANSPORT_SNAPSHOT = "snapshot" VIEWER_TRANSPORTS = {VIEWER_TRANSPORT_SCREENCAST, VIEWER_TRANSPORT_SNAPSHOT} @@ -97,10 +99,21 @@ class WsBrowser(WsHandler): existing.cancel() viewer_id = str(data.get("viewer_id") or "") viewer_transport = self._viewer_transport(data) + binary_frames = self._bool(data.get("binary_frames", data.get("binaryFrames"))) + slim_frames = self._bool(data.get("slim_frames", data.get("slimFrames", binary_frames))) + capture_scale = self._capture_scale_from_data(data) snapshot = None if runtime: if viewer_transport == VIEWER_TRANSPORT_SCREENCAST: - stream_task = self._stream_frames(sid, context_id, active_id, viewer_id) + stream_task = self._stream_frames( + sid, + context_id, + active_id, + viewer_id, + binary_frames=binary_frames, + slim_frames=slim_frames, + capture_scale=capture_scale, + ) else: stream_task = self._stream_state(sid, context_id, active_id, viewer_id) self._streams[stream_key] = asyncio.create_task(stream_task) @@ -115,6 +128,8 @@ class WsBrowser(WsHandler): "all_browsers": True, "viewer_id": viewer_id, "viewer_transport": viewer_transport, + "binary_frames": binary_frames, + "slim_frames": slim_frames, } def _unsubscribe(self, data: dict[str, Any], sid: str) -> dict[str, Any] | WsResult: @@ -157,9 +172,9 @@ class WsBrowser(WsHandler): snapshot = None if active_id: try: - quality = int(data.get("quality") or SCREENCAST_QUALITY) + quality = int(data.get("quality") or SCREENSHOT_QUALITY) except (TypeError, ValueError): - quality = SCREENCAST_QUALITY + quality = SCREENSHOT_QUALITY with contextlib.suppress(Exception): snapshot = await runtime.call( "screenshot", @@ -347,7 +362,7 @@ class WsBrowser(WsHandler): if not browser_id: return None with contextlib.suppress(Exception): - return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY) + return await runtime.call("screenshot", browser_id, quality=SCREENSHOT_QUALITY) return None async def _snapshot_for_browser( @@ -358,7 +373,7 @@ class WsBrowser(WsHandler): if not browser_id: return None with contextlib.suppress(Exception): - return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY) + return await runtime.call("screenshot", browser_id, quality=SCREENSHOT_QUALITY) return None async def _all_browser_tabs(self) -> list[dict[str, Any]]: @@ -377,6 +392,10 @@ class WsBrowser(WsHandler): context_id: str, browser_id: int | str | None, viewer_id: str = "", + *, + binary_frames: bool = False, + slim_frames: bool = False, + capture_scale: float = 1.0, ) -> None: runtime = None stream_id = None @@ -397,12 +416,14 @@ class WsBrowser(WsHandler): browsers = listing.get("browsers") or [] active_id = self._active_browser_id(listing, browser_id) if not active_id: - await self._emit_empty_frame( + await self._emit_viewer_state( sid, context_id, + active_id, browsers=browsers, viewer_id=viewer_id, - frame_source=VIEWER_TRANSPORT_SCREENCAST, + state=None, + viewer_transport=VIEWER_TRANSPORT_SCREENCAST, ) await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS) continue @@ -410,29 +431,26 @@ class WsBrowser(WsHandler): screencast = await runtime.call( "start_screencast", active_id, - quality=SCREENCAST_QUALITY, + quality=SCREENCAST_STREAM_QUALITY, every_nth_frame=1, + capture_scale=capture_scale, ) stream_id = screencast["stream_id"] active_id = screencast["browser_id"] state = screencast.get("state") - await self.emit_to( + await self._emit_viewer_state( sid, - "browser_viewer_frame", - { - "context_id": context_id, - "viewer_id": viewer_id, - "browser_id": active_id, - "browsers": browsers, - "image": "", - "mime": "", - "state": state, - "frame_source": "state", - "viewer_transport": VIEWER_TRANSPORT_SCREENCAST, - }, + context_id, + active_id, + browsers=browsers, + viewer_id=viewer_id, + state=state, + viewer_transport=VIEWER_TRANSPORT_SCREENCAST, ) last_state_refresh = 0.0 + last_state_signature = self._state_signature(active_id, browsers) + frame_sequence = 0 while True: now = time.monotonic() if now - last_state_refresh >= FRAME_STATE_REFRESH_SECONDS: @@ -442,6 +460,18 @@ class WsBrowser(WsHandler): if str(active_id) not in browser_ids: break state = self._state_for_browser(browsers, active_id, state) + state_signature = self._state_signature(active_id, browsers) + if state_signature != last_state_signature: + await self._emit_viewer_state( + sid, + context_id, + active_id, + browsers=browsers, + viewer_id=viewer_id, + state=state, + viewer_transport=VIEWER_TRANSPORT_SCREENCAST, + ) + last_state_signature = state_signature last_state_refresh = now try: @@ -455,14 +485,19 @@ class WsBrowser(WsHandler): except TimeoutError: continue - frame["context_id"] = context_id - frame["viewer_id"] = viewer_id - frame["browser_id"] = active_id - frame["browsers"] = browsers - frame["state"] = state - frame["frame_source"] = VIEWER_TRANSPORT_SCREENCAST - frame["viewer_transport"] = VIEWER_TRANSPORT_SCREENCAST - await self.emit_to(sid, "browser_viewer_frame", frame) + frame_sequence += 1 + payload = self._frame_payload( + frame, + context_id=context_id, + viewer_id=viewer_id, + browser_id=active_id, + sequence=frame_sequence, + binary_frames=binary_frames, + ) + if not slim_frames: + payload["browsers"] = browsers + payload["state"] = state + await self._emit_to_connected_viewer(sid, "browser_viewer_frame", payload) except asyncio.CancelledError: raise except Exception: @@ -515,20 +550,14 @@ class WsBrowser(WsHandler): ), ) if signature != last_signature: - await self.emit_to( + await self._emit_viewer_state( sid, - "browser_viewer_frame", - { - "context_id": context_id, - "viewer_id": viewer_id, - "browser_id": active_id, - "browsers": browsers, - "image": "", - "mime": "", - "state": state, - "frame_source": VIEWER_TRANSPORT_SNAPSHOT, - "viewer_transport": VIEWER_TRANSPORT_SNAPSHOT, - }, + context_id, + active_id, + browsers=browsers, + viewer_id=viewer_id, + state=state, + viewer_transport=VIEWER_TRANSPORT_SNAPSHOT, ) last_signature = signature await asyncio.sleep(SNAPSHOT_STATE_POLL_SECONDS) @@ -567,6 +596,105 @@ class WsBrowser(WsHandler): return browser return current_state + @staticmethod + def _state_signature( + active_id: int | str | None, + browsers: list[dict[str, Any]], + ) -> tuple[str, tuple[tuple[str, str, str, str, bool], ...]]: + return ( + str(active_id or ""), + tuple( + ( + str(browser.get("context_id") or ""), + str(browser.get("id") or ""), + str(browser.get("currentUrl") or ""), + str(browser.get("title") or ""), + bool(browser.get("loading")), + ) + for browser in browsers + ), + ) + + @staticmethod + def _frame_payload( + frame: dict[str, Any], + *, + context_id: str, + viewer_id: str, + browser_id: int | str, + sequence: int, + binary_frames: bool, + ) -> dict[str, Any]: + image = str(frame.get("image") or "") + payload: dict[str, Any] = { + "context_id": context_id, + "viewer_id": viewer_id, + "browser_id": browser_id, + "seq": sequence, + "mime": frame.get("mime") or "image/jpeg", + "frame_source": VIEWER_TRANSPORT_SCREENCAST, + "viewer_transport": VIEWER_TRANSPORT_SCREENCAST, + } + dimensions = WsBrowser._frame_dimensions(frame.get("metadata")) + if dimensions: + payload.update(dimensions) + if binary_frames: + try: + payload["image"] = base64.b64decode(image, validate=False) + payload["encoding"] = "binary" + except Exception: + payload["image"] = image + payload["encoding"] = "base64" + else: + payload["image"] = image + payload["encoding"] = "base64" + return payload + + @staticmethod + def _frame_dimensions(metadata: Any) -> dict[str, int]: + if not isinstance(metadata, dict): + return {} + for width_key, height_key in ( + ("expectedWidth", "expectedHeight"), + ("deviceWidth", "deviceHeight"), + ("jpegWidth", "jpegHeight"), + ): + try: + width = int(metadata.get(width_key) or 0) + height = int(metadata.get(height_key) or 0) + except (TypeError, ValueError): + continue + if width > 0 and height > 0: + return {"width": width, "height": height} + return {} + + async def _emit_viewer_state( + self, + sid: str, + context_id: str, + browser_id: int | str | None, + *, + browsers: list[dict[str, Any]] | None = None, + viewer_id: str = "", + state: dict[str, Any] | None = None, + viewer_transport: str = VIEWER_TRANSPORT_SNAPSHOT, + ) -> None: + await self._emit_to_connected_viewer( + sid, + "browser_viewer_state", + { + "context_id": context_id, + "active_browser_context_id": context_id, + "viewer_id": viewer_id, + "browser_id": browser_id, + "active_browser_id": browser_id, + "browsers": browsers or [], + "state": state, + "all_browsers": False, + "viewer_transport": viewer_transport, + }, + ) + async def _emit_empty_frame( self, sid: str, @@ -576,7 +704,7 @@ class WsBrowser(WsHandler): viewer_id: str = "", frame_source: str = "", ) -> None: - await self.emit_to( + await self._emit_to_connected_viewer( sid, "browser_viewer_frame", { @@ -592,6 +720,20 @@ class WsBrowser(WsHandler): }, ) + async def _emit_to_connected_viewer( + self, + sid: str, + event: str, + data: dict[str, Any], + ) -> None: + manager = getattr(self, "_manager", None) + if manager is not None: + with manager.lock: + connected = (getattr(self, "namespace", "/ws"), sid) in manager.connections + if not connected: + raise asyncio.CancelledError() + await self.emit_to(sid, event, data) + @staticmethod def _viewer_transport(data: dict[str, Any]) -> str: raw = ( @@ -623,6 +765,20 @@ class WsBrowser(WsHandler): "height": max(200, min(4096, height)), } + @staticmethod + def _capture_scale_from_data(data: dict[str, Any]) -> float: + try: + scale = float( + data.get("device_pixel_ratio") + or data.get("devicePixelRatio") + or data.get("pixel_ratio") + or data.get("pixelRatio") + or 1 + ) + except (TypeError, ValueError): + return 1.0 + return max(1.0, min(2.0, scale)) + @staticmethod def _context_id(data: dict[str, Any]) -> str: return str(data.get("context_id") or data.get("context") or "").strip() diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py index e7455d81a..94524071d 100644 --- a/plugins/_browser/helpers/runtime.py +++ b/plugins/_browser/helpers/runtime.py @@ -329,10 +329,14 @@ class _BrowserScreencast: quality: int, every_nth_frame: int, viewport: dict[str, int], + capture_scale: float = 1.0, ) -> None: self.session.on("Page.screencastFrame", self._on_frame) width = max(320, min(4096, int(viewport.get("width") or DEFAULT_VIEWPORT["width"]))) height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"]))) + scale = max(1.0, min(2.0, float(capture_scale or 1.0))) + max_width = max(320, min(SCREENCAST_MAX_WIDTH, int(round(width * scale)))) + max_height = max(200, min(SCREENCAST_MAX_HEIGHT, int(round(height * scale)))) self._expected_width = width self._expected_height = height with contextlib.suppress(Exception): @@ -343,8 +347,8 @@ class _BrowserScreencast: { "format": "jpeg", "quality": max(20, min(95, int(quality))), - "maxWidth": SCREENCAST_MAX_WIDTH, - "maxHeight": SCREENCAST_MAX_HEIGHT, + "maxWidth": max_width, + "maxHeight": max_height, "everyNthFrame": max(1, int(every_nth_frame)), }, ) @@ -1616,6 +1620,7 @@ class _BrowserRuntimeCore: *, quality: int = 78, every_nth_frame: int = 1, + capture_scale: float = 1.0, ) -> dict[str, Any]: await self.ensure_started() resolved_id = self._resolve_browser_id(browser_id) @@ -1634,6 +1639,7 @@ class _BrowserRuntimeCore: quality=quality, every_nth_frame=every_nth_frame, viewport=page.viewport_size or DEFAULT_VIEWPORT, + capture_scale=capture_scale, ) except Exception: self.screencasts.pop(stream_id, None) diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.js index 6e70a1891..a4b3d4525 100644 --- a/plugins/_browser/webui/browser-store.js +++ b/plugins/_browser/webui/browser-store.js @@ -34,6 +34,8 @@ const ANNOTATION_DOM_LIMIT = 1200; const ANNOTATION_TRAY_MARGIN = 10; const BROWSER_VISUAL_SHORTCUT_KEYS = new Set(["a", "c", "insert", "v", "x", "y", "z"]); const LOCAL_EDITABLE_SELECTOR = "input, textarea, select, [contenteditable]"; +const BROWSER_BINARY_FRAMES_SUPPORTED = typeof Blob === "function" + && typeof globalThis.URL?.createObjectURL === "function"; function makeViewerToken() { return globalThis.crypto?.randomUUID?.() @@ -114,6 +116,31 @@ function loadFrameDimensions(src) { }); } +function frameImageSource(data = {}) { + const image = data?.image; + if (!image) return null; + const mime = data.mime || "image/jpeg"; + const isArrayBuffer = typeof ArrayBuffer !== "undefined" && image instanceof ArrayBuffer; + const isView = typeof ArrayBuffer !== "undefined" && ArrayBuffer.isView?.(image); + const isBlob = typeof Blob !== "undefined" && image instanceof Blob; + if (data.encoding === "binary" || isArrayBuffer || isView || isBlob) { + if (!BROWSER_BINARY_FRAMES_SUPPORTED) return null; + const blob = isBlob ? image : new Blob([image], { type: mime }); + const src = globalThis.URL.createObjectURL(blob); + return { + src, + objectUrl: src, + cleanup: () => globalThis.URL.revokeObjectURL(src), + }; + } + if (typeof image !== "string") return null; + return { + src: `data:${mime};base64,${image}`, + objectUrl: "", + cleanup: null, + }; +} + const model = { loading: true, error: "", @@ -146,6 +173,7 @@ const model = { _lastFrameDimensions: null, _pendingFrameSrc: "", _pendingFrameOptions: null, + _frameObjectUrl: "", _frameRenderHandle: null, _frameRenderCancel: null, _frameRenderSequence: 0, @@ -442,7 +470,7 @@ const model = { this.setActiveBrowserId(null); this.address = ""; this.frameState = null; - this.frameSrc = ""; + this.clearFrameSrc(); if (this.contextId) { await this.connectViewer(); } @@ -833,7 +861,7 @@ const model = { resetRenderedFrame() { this.cancelFrameRender(); - this.frameSrc = ""; + this.clearFrameSrc(); this._lastFrameDimensions = null; this._lastFrameAt = 0; }, @@ -915,6 +943,25 @@ const model = { return this.viewerTransport === BROWSER_VIEWER_TRANSPORT_SCREENCAST; }, + supportsBinaryFrames() { + return BROWSER_BINARY_FRAMES_SUPPORTED; + }, + + captureDevicePixelRatio() { + const value = Number(globalThis.devicePixelRatio || 1); + if (!Number.isFinite(value) || value <= 1) return 1; + return Math.min(2, value); + }, + + frameDimensionsFromData(data = null) { + const width = Number(data?.width || 0); + const height = Number(data?.height || 0); + if (Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0) { + return { width, height }; + } + return this.frameDimensionsFromMetadata(data?.metadata); + }, + frameDimensionsFromMetadata(metadata = null) { if (!metadata || typeof metadata !== "object") return null; const width = Number(metadata.expectedWidth || metadata.deviceWidth || metadata.jpegWidth || 0); @@ -986,6 +1033,9 @@ const model = { viewer_id: viewerToken, create_browser: Boolean(options.createBrowser || options.create_browser), viewer_transport: this.requestedViewerTransport(), + binary_frames: this.supportsBinaryFrames(), + slim_frames: true, + device_pixel_ratio: this.captureDevicePixelRatio(), viewport_width: initialViewport?.width, viewport_height: initialViewport?.height, }, @@ -1014,9 +1064,9 @@ const model = { data.active_browser_context_id || contextId, ); this.applySnapshot(data.snapshot); - this.connected = true; - this.browserInstallExpected = false; - }, + this.connected = true; + this.browserInstallExpected = false; + }, async _bindSocketEvents() { if (!this._frameOff) { @@ -1028,7 +1078,9 @@ const model = { } const incomingContextId = this.normalizeContextId(data.context_id || this.contextId); const incomingBrowserId = this.normalizeBrowserId(data.browser_id || data.state?.id); - this.applyBrowserListing(data.browsers || [], incomingContextId, { replaceContext: true }); + if (Array.isArray(data.browsers)) { + this.applyBrowserListing(data.browsers, incomingContextId, { replaceContext: true }); + } if (incomingBrowserId && !this.activeBrowserId) { this.setActiveBrowserId(incomingBrowserId, incomingContextId); } @@ -1046,11 +1098,15 @@ const model = { this.address = data.state.currentUrl; } if (data.image) { + const frameImage = frameImageSource(data); + if (!frameImage?.src) return; const frameBrowserId = incomingBrowserId || this.activeBrowserId; - this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`, { + this.queueFrameRender(frameImage.src, { browserId: frameBrowserId, contextId: incomingContextId, - dimensions: this.frameDimensionsFromMetadata(data.metadata), + dimensions: this.frameDimensionsFromData(data), + objectUrl: frameImage.objectUrl, + cleanup: frameImage.cleanup, onAccepted: () => { if ( this.sameBrowserId(this.switchingBrowserId, frameBrowserId) @@ -1063,13 +1119,13 @@ const model = { }); } else if (!data.state) { this.cancelFrameRender(); - this.frameSrc = ""; + this.clearFrameSrc(); } if (!data.image && !data.state) { if (!this.activeBrowserId) { this.setActiveBrowserId(null, ""); this.frameState = null; - this.frameSrc = ""; + this.clearFrameSrc(); } } this._lastFrameAt = Date.now(); @@ -1077,15 +1133,20 @@ const model = { await websocket.on("browser_viewer_frame", frameHandler); this._frameOff = () => websocket.off("browser_viewer_frame", frameHandler); } - if (!this._stateOff) { - const stateHandler = ({ data }) => { - if (data?.context_id !== this.contextId) return; - if (data?.viewer_id && data.viewer_id !== this._viewerToken) return; - if (data?.viewer_transport) { - this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport); - } + if (!this._stateOff) { + const stateHandler = ({ data }) => { + if (data?.context_id !== this.contextId) return; + if (data?.viewer_id && data.viewer_id !== this._viewerToken) return; + if (data?.viewer_transport) { + this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport); + } const commandContextId = this.normalizeContextId(data.active_browser_context_id || data.context_id || this.contextId); - this.applyBrowserListing(data.browsers || [], commandContextId, { replaceAll: Boolean(data.all_browsers) }); + if (Array.isArray(data.browsers)) { + this.applyBrowserListing(data.browsers, commandContextId, { + replaceAll: Boolean(data.all_browsers), + replaceContext: !data.all_browsers, + }); + } const command = String(data.command || "").toLowerCase(); const commandBrowserId = this.normalizeBrowserId(data.browser_id); const result = data.result || {}; @@ -1102,23 +1163,40 @@ const model = { || this.activeBrowserId || this.firstBrowserId(resultContextId) ); - if ( - !this.activeBrowserId - || command === "open" - || command === "close" - || this.sameBrowserTab(commandBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId) - ) { - this.setActiveBrowserId(preferredBrowserId, resultContextId); - } - this.applyActiveFrameState(resultState || this.browserById(this.activeBrowserId, this.activeBrowserContextId)); - this.applySnapshot(data.snapshot); - }; + const stateBrowserId = this.normalizeBrowserId(data.active_browser_id || data.browser_id || data.state?.id); + if ( + stateBrowserId + && ( + !this.activeBrowserId + || this.sameBrowserTab(stateBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId) + ) + ) { + this.setActiveBrowserId(stateBrowserId, commandContextId); + } + if ( + !this.activeBrowserId + || command === "open" + || command === "close" + || this.sameBrowserTab(commandBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId) + ) { + this.setActiveBrowserId(preferredBrowserId, resultContextId); + } + this.applyActiveFrameState( + resultState + || data.state + || this.browserById(this.activeBrowserId, this.activeBrowserContextId) + ); + this.applySnapshot(data.snapshot); + }; await websocket.on("browser_viewer_state", stateHandler); this._stateOff = () => websocket.off("browser_viewer_state", stateHandler); } }, queueFrameRender(frameSrc, options = {}) { + if (this._pendingFrameSrc) { + this.releasePendingFrame(); + } this._pendingFrameSrc = frameSrc; this._pendingFrameOptions = options || null; if (this._frameRenderHandle) return; @@ -1148,22 +1226,26 @@ const model = { async renderDecodedFrame(frameSrc, options = {}, sequence = 0, surfaceSequence = this._surfaceOpenSequence) { if (!frameSrc) { if (sequence === this._frameRenderSequence) { - this.frameSrc = ""; + this.clearFrameSrc(); } return; } const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc); if (sequence !== this._frameRenderSequence || surfaceSequence !== this._surfaceOpenSequence) { + options?.cleanup?.(); return; } const viewport = this.currentViewportSize() || this._lastViewport; if (!this.frameMatchesViewport(dimensions, viewport)) { this.requestViewportSyncAfterRejectedFrame(); if (!this.shouldAcceptMismatchedFrame(dimensions)) { + options?.cleanup?.(); return; } } + this.releaseRenderedFrameUrl(frameSrc); this.frameSrc = frameSrc; + this._frameObjectUrl = options?.objectUrl || ""; this._lastFrameDimensions = dimensions; this._lastFrameAt = Date.now(); options?.onAccepted?.(); @@ -1262,9 +1344,26 @@ const model = { } this._frameRenderHandle = null; this._frameRenderCancel = null; + this.releasePendingFrame(); + this._frameRenderSequence += 1; + }, + + releasePendingFrame() { + this._pendingFrameOptions?.cleanup?.(); this._pendingFrameSrc = ""; this._pendingFrameOptions = null; - this._frameRenderSequence += 1; + }, + + releaseRenderedFrameUrl(nextSrc = "") { + if (this._frameObjectUrl && this._frameObjectUrl !== nextSrc) { + globalThis.URL?.revokeObjectURL?.(this._frameObjectUrl); + this._frameObjectUrl = ""; + } + }, + + clearFrameSrc() { + this.releaseRenderedFrameUrl(""); + this.frameSrc = ""; }, beginCommand() { @@ -1329,7 +1428,7 @@ const model = { ); if (!this.activeBrowserId) { this.frameState = null; - this.frameSrc = ""; + this.clearFrameSrc(); } if (result.state?.currentUrl || result.currentUrl) { this.address = result.state?.currentUrl || result.currentUrl; @@ -1438,7 +1537,7 @@ const model = { this.error = ""; this.switchingBrowserId = targetId; this.cancelFrameRender(); - this.frameSrc = ""; + this.clearFrameSrc(); this.frameState = browser || null; if (!this.addressFocused && browser?.currentUrl) { this.address = browser.currentUrl; @@ -1643,35 +1742,35 @@ const model = { } }, - applySnapshot(snapshot = null) { - if (!snapshot?.image) return; - const snapshotId = this.normalizeBrowserId(snapshot.browser_id || snapshot.state?.id); + applySnapshot(snapshot = null) { + if (!snapshot?.image) return; + const snapshotId = this.normalizeBrowserId(snapshot.browser_id || snapshot.state?.id); const snapshotContextId = this.normalizeContextId(snapshot.context_id || snapshot.state?.context_id || this.activeBrowserContextId); - if ( + if ( snapshotId && this.activeBrowserId && !this.sameBrowserTab(snapshotId, snapshotContextId, this.activeBrowserId, this.activeBrowserContextId) ) { - return; - } - if (snapshot.state) { - this.applyActiveFrameState(snapshot.state); - } - const frameBrowserId = snapshotId || this.activeBrowserId; - this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, { - browserId: frameBrowserId, + return; + } + if (snapshot.state) { + this.applyActiveFrameState(snapshot.state); + } + const frameBrowserId = snapshotId || this.activeBrowserId; + this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, { + browserId: frameBrowserId, contextId: snapshotContextId, - onAccepted: () => { - if ( + onAccepted: () => { + if ( this.sameBrowserId(this.switchingBrowserId, frameBrowserId) && this.normalizeContextId(this.activeBrowserContextId) === snapshotContextId ) { - this.switchingBrowserId = null; - } - this._surfaceSwitching = false; - }, - }); - }, + this.switchingBrowserId = null; + } + this._surfaceSwitching = false; + }, + }); + }, isSwitchingBrowser() { return Boolean( diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py index eef2e141b..7503efeb3 100644 --- a/tests/test_browser_agent_regressions.py +++ b/tests/test_browser_agent_regressions.py @@ -1158,7 +1158,8 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback(): assert "def _viewer_transport(data: dict[str, Any])" in ws_browser assert "return VIEWER_TRANSPORT_SNAPSHOT" in ws_browser assert "self._stream_state" in ws_browser - assert "SCREENCAST_QUALITY = 92" in ws_browser + assert "SCREENCAST_STREAM_QUALITY = 80" in ws_browser + assert "SCREENSHOT_QUALITY = 92" in ws_browser assert "initial_viewport = self._viewport_from_data(data)" in ws_browser assert '"set_viewport"' in ws_browser assert "start_screencast" in ws_browser @@ -1169,7 +1170,7 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback(): assert "stop_screencast" in ws_browser assert "viewer_transport == VIEWER_TRANSPORT_SCREENCAST" in ws_browser assert '"viewer_transport": viewer_transport' in ws_browser - assert '"viewer_transport": VIEWER_TRANSPORT_SNAPSHOT' in ws_browser + assert "viewer_transport: str = VIEWER_TRANSPORT_SNAPSHOT" in ws_browser assert '"Page.startScreencast"' in runtime assert '"Page.screencastFrame"' in runtime assert '"Page.screencastFrameAck"' in runtime @@ -1180,6 +1181,7 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback(): assert "await self._stop_screencasts_for_browser(resolved_id)" in runtime assert "queueFrameRender" in browser_store assert "requestAnimationFrame" in browser_store + assert "function frameImageSource(data = {})" in browser_store assert 'const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";' in browser_store assert 'const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";' in browser_store assert "viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST" in browser_store @@ -1187,11 +1189,17 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback(): assert "requestedViewerTransport()" in browser_store assert "normalizeViewerTransport(value = \"\")" in browser_store assert "usesScreencastTransport()" in browser_store + assert "supportsBinaryFrames()" in browser_store + assert "captureDevicePixelRatio()" in browser_store + assert "frameDimensionsFromData(data = null)" in browser_store assert "frameDimensionsFromMetadata(metadata = null)" in browser_store assert "metadata.expectedWidth || metadata.deviceWidth || metadata.jpegWidth" in browser_store - assert "dimensions: this.frameDimensionsFromMetadata(data.metadata)" in browser_store + assert "dimensions: this.frameDimensionsFromData(data)" in browser_store assert "const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc)" in browser_store assert "viewer_transport: this.requestedViewerTransport()" in browser_store + assert "binary_frames: this.supportsBinaryFrames()" in browser_store + assert "slim_frames: true" in browser_store + assert "device_pixel_ratio: this.captureDevicePixelRatio()" in browser_store assert "viewport_width: initialViewport?.width" in browser_store assert "viewport_height: initialViewport?.height" in browser_store assert "restart_stream: restartStream && this.usesScreencastTransport()" in browser_store @@ -1213,6 +1221,13 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback(): assert "this.applySnapshot(data.snapshot);" in browser_store assert "else if (!data.state)" in browser_store assert '"snapshot": snapshot' in ws_browser + assert '"binary_frames": binary_frames' in ws_browser + assert '"slim_frames": slim_frames' in ws_browser + assert "capture_scale=capture_scale" in ws_browser + assert "base64.b64decode(image, validate=False)" in ws_browser + assert '"encoding"] = "binary"' in ws_browser + assert "def _frame_payload(" in ws_browser + assert "def _emit_viewer_state(" in ws_browser assert 'const BROWSER_SNAPSHOT_META_KEY = "browser_snapshot";' in browser_tool_handler assert "staticScreenshotUri(kvps)" in browser_tool_handler assert "/components/modals/image-viewer/image-viewer-store.js" in browser_tool_handler @@ -1223,9 +1238,9 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback(): assert "delete displayKvps[BROWSER_SNAPSHOT_META_KEY];" in browser_tool_handler assert "startBrowserScreenshotPreview(button, image, resolveBrowserPayload)" in browser_tool_handler assert "FRAME_FALLBACK_SCREENSHOT_SECONDS" not in ws_browser - assert '"frame_source": "state"' in ws_browser - assert '"frame_source"] = VIEWER_TRANSPORT_SCREENCAST' in ws_browser - assert '"viewer_transport"] = VIEWER_TRANSPORT_SCREENCAST' in ws_browser + assert '"browser_viewer_state"' in ws_browser + assert '"frame_source": VIEWER_TRANSPORT_SCREENCAST' in ws_browser + assert '"viewer_transport": VIEWER_TRANSPORT_SCREENCAST' in ws_browser assert "fallback_screenshot" not in ws_browser assert "canvas_wheel_screenshot" not in ws_browser assert "surface_mode: this._mode" not in browser_store @@ -1234,6 +1249,29 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback(): assert "image-rendering: auto;" in main_html +def test_browser_viewer_frame_payload_supports_binary_slim_frames(): + payload = ws_browser_module.WsBrowser._frame_payload( + { + "image": SMALL_JPEG_10X10, + "mime": "image/jpeg", + "metadata": {"expectedWidth": 900, "expectedHeight": 600}, + }, + context_id="ctx", + viewer_id="viewer", + browser_id=1, + sequence=3, + binary_frames=True, + ) + + assert payload["encoding"] == "binary" + assert payload["image"] == __import__("base64").b64decode(SMALL_JPEG_10X10) + assert payload["width"] == 900 + assert payload["height"] == 600 + assert payload["seq"] == 3 + assert "browsers" not in payload + assert "state" not in payload + + def test_browser_navigation_errors_stay_inside_native_browser_page(): runtime = ( PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py" @@ -1504,7 +1542,12 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames(): mime="image/jpeg", ) - await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 1118, "height": 662}) + await screencast.start( + quality=92, + every_nth_frame=1, + viewport={"width": 1118, "height": 662}, + capture_scale=2, + ) session.handlers["Page.screencastFrame"]( {"data": first_image, "metadata": {"deviceWidth": 10}, "sessionId": 1} ) @@ -1563,6 +1606,10 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames(): for index, (method, _params) in enumerate(session.sent) if method == "Page.startScreencast" ) + start_params = session.sent[start_index][1] + assert start_params["quality"] == 92 + assert start_params["maxWidth"] == 2236 + assert start_params["maxHeight"] == 1324 cdp_viewport_indices = [ index for index, (method, _params) in enumerate(session.sent) @@ -2287,7 +2334,7 @@ async def test_browser_viewer_subscribe_returns_initial_snapshot(monkeypatch): assert result["active_browser_id"] == 1 assert result["snapshot"]["image"] == "jpeg-data" - assert ("screenshot", (1,), {"quality": ws_browser_module.SCREENCAST_QUALITY}) in calls + assert ("screenshot", (1,), {"quality": ws_browser_module.SCREENSHOT_QUALITY}) in calls await handler.on_disconnect("sid-snapshot")