From decb05a6823b4a04fddafa2357ea8155c766e638 Mon Sep 17 00:00:00 2001 From: Alessandro <155005371+3clyp50@users.noreply.github.com> Date: Tue, 28 Apr 2026 03:50:38 +0200 Subject: [PATCH] Stabilize browser viewer viewport rendering Decode browser frames before display and only render frames that match the active viewer viewport, avoiding stretched stale screencast images during startup and resize. Keep rejecting mismatched CDP screencast frames on the backend, extend canvas viewport settling, and cover the behavior with browser regression tests. Include small browser panel CSS polish. --- plugins/_browser/helpers/runtime.py | 10 +- plugins/_browser/webui/browser-panel.html | 2 +- plugins/_browser/webui/browser-store.js | 127 +++++++++++++++++++--- tests/test_browser_agent_regressions.py | 72 +++++++++--- 4 files changed, 177 insertions(+), 34 deletions(-) diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py index db459de91..3111b3068 100644 --- a/plugins/_browser/helpers/runtime.py +++ b/plugins/_browser/helpers/runtime.py @@ -111,7 +111,6 @@ class _BrowserScreencast: self._ack_tasks: set[asyncio.Task] = set() self._expected_width = 0 self._expected_height = 0 - self._dimension_mismatches = 0 async def start( self, @@ -125,7 +124,6 @@ class _BrowserScreencast: height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"]))) self._expected_width = width self._expected_height = height - self._dimension_mismatches = 0 with contextlib.suppress(Exception): await self.session.send("Page.enable") await self.session.send( @@ -229,10 +227,12 @@ class _BrowserScreencast: if not size: return True width, height = size - if abs(width - self._expected_width) <= 2 and abs(height - self._expected_height) <= 2: + if ( + abs(width - self._expected_width) <= VIEWPORT_SIZE_TOLERANCE + and abs(height - self._expected_height) <= VIEWPORT_SIZE_TOLERANCE + ): return True - self._dimension_mismatches += 1 - return self._dimension_mismatches > 10 + return False @staticmethod def _jpeg_size(data: str) -> tuple[int, int] | None: diff --git a/plugins/_browser/webui/browser-panel.html b/plugins/_browser/webui/browser-panel.html index e10baaaf5..3ff50ef5d 100644 --- a/plugins/_browser/webui/browser-panel.html +++ b/plugins/_browser/webui/browser-panel.html @@ -620,7 +620,7 @@ align-items: end; gap: 6px; min-width: 0; - padding-bottom: 1px; + padding-bottom: 7px; } .browser-annotate-toggle { diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.js index b7db50928..bb12b161e 100644 --- a/plugins/_browser/webui/browser-store.js +++ b/plugins/_browser/webui/browser-store.js @@ -13,8 +13,9 @@ const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000; const BROWSER_CONFIG_REFRESH_MS = 15000; const VIEWPORT_SYNC_DEBOUNCE_MS = 220; const VIEWPORT_SYNC_SIZE_TOLERANCE = 4; -const CANVAS_VIEWPORT_SETTLE_MS = 260; -const SURFACE_VIEWPORT_STABLE_FRAMES = 2; +const CANVAS_VIEWPORT_SETTLE_MS = 520; +const SURFACE_VIEWPORT_STABLE_FRAMES = 4; +const FRAME_REJECT_SYNC_COOLDOWN_MS = 600; const ANNOTATION_DRAG_THRESHOLD = 6; const ANNOTATION_MAX_COMMENTS = 24; const ANNOTATION_DOM_LIMIT = 1200; @@ -55,6 +56,34 @@ function nextAnimationFrame() { }); } +function loadFrameDimensions(src) { + return new Promise((resolve) => { + if (!src) { + resolve(null); + return; + } + + const image = new Image(); + let settled = false; + const finish = (dimensions) => { + if (settled) return; + settled = true; + resolve(dimensions); + }; + + image.onload = () => finish({ + width: image.naturalWidth || 0, + height: image.naturalHeight || 0, + }); + image.onerror = () => finish(null); + image.src = src; + + if (image.complete) { + image.onload(); + } + }); +} + const model = { loading: true, error: "", @@ -79,13 +108,17 @@ const model = { _frameOff: null, _stateOff: null, _lastFrameAt: 0, + _lastFrameDimensions: null, _pendingFrameSrc: "", + _pendingFrameOptions: null, _frameRenderHandle: null, _frameRenderCancel: null, + _frameRenderSequence: 0, _floatingCleanup: null, _stageElement: null, _stageResizeObserver: null, _viewportSyncTimer: null, + _lastFrameRejectSyncAt: 0, _lastViewportKey: "", _lastViewport: null, _annotationPointer: null, @@ -466,6 +499,7 @@ const model = { resetRenderedFrame() { this.cancelFrameRender(); this.frameSrc = ""; + this._lastFrameDimensions = null; this._lastFrameAt = 0; }, @@ -599,11 +633,16 @@ const model = { this.address = data.state.currentUrl; } if (data.image) { - this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`); - if (this.sameBrowserId(this.switchingBrowserId, incomingBrowserId || this.activeBrowserId)) { - this.switchingBrowserId = null; - } - this._surfaceSwitching = false; + const frameBrowserId = incomingBrowserId || this.activeBrowserId; + this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`, { + browserId: frameBrowserId, + onAccepted: () => { + if (this.sameBrowserId(this.switchingBrowserId, frameBrowserId)) { + this.switchingBrowserId = null; + } + this._surfaceSwitching = false; + }, + }); } else { this.cancelFrameRender(); if (!data.state) { @@ -654,8 +693,9 @@ const model = { } }, - queueFrameRender(frameSrc) { + queueFrameRender(frameSrc, options = {}) { this._pendingFrameSrc = frameSrc; + this._pendingFrameOptions = options || null; if (this._frameRenderHandle) return; const schedule = globalThis.requestAnimationFrame?.bind(globalThis); if (schedule) { @@ -670,8 +710,59 @@ const model = { flushFrameRender() { this._frameRenderHandle = null; this._frameRenderCancel = null; - this.frameSrc = this._pendingFrameSrc || ""; + const frameSrc = this._pendingFrameSrc || ""; + const options = this._pendingFrameOptions || {}; this._pendingFrameSrc = ""; + this._pendingFrameOptions = null; + const sequence = this._frameRenderSequence + 1; + this._frameRenderSequence = sequence; + void this.renderDecodedFrame(frameSrc, options, sequence); + }, + + async renderDecodedFrame(frameSrc, options = {}, sequence = 0) { + if (!frameSrc) { + if (sequence === this._frameRenderSequence) { + this.frameSrc = ""; + } + return; + } + const dimensions = await loadFrameDimensions(frameSrc); + if (sequence !== this._frameRenderSequence) return; + const viewport = this.currentViewportSize() || this._lastViewport; + if (!this.frameMatchesViewport(dimensions, viewport)) { + this.requestViewportSyncAfterRejectedFrame(); + return; + } + this.frameSrc = frameSrc; + this._lastFrameDimensions = dimensions; + this._lastFrameAt = Date.now(); + options?.onAccepted?.(); + }, + + frameMatchesViewport(dimensions = null, viewport = null) { + if (!dimensions?.width || !dimensions?.height || !viewport?.width || !viewport?.height) { + return false; + } + return Math.abs(Number(dimensions.width) - Number(viewport.width)) <= VIEWPORT_SYNC_SIZE_TOLERANCE + && Math.abs(Number(dimensions.height) - Number(viewport.height)) <= VIEWPORT_SYNC_SIZE_TOLERANCE; + }, + + requestViewportSyncAfterRejectedFrame() { + const now = Date.now(); + if (now - this._lastFrameRejectSyncAt < FRAME_REJECT_SYNC_COOLDOWN_MS) return; + this._lastFrameRejectSyncAt = now; + this.queueViewportSync(true); + }, + + clearRenderedFrameIfViewportChanged() { + const viewport = this.currentViewportSize(); + if (!this.frameSrc || !this._lastFrameDimensions || !viewport) return; + if (this.frameMatchesViewport(this._lastFrameDimensions, viewport)) return; + this.resetRenderedFrame(); + if (this.activeBrowserId) { + this._surfaceSwitching = true; + this.switchingBrowserId = this.activeBrowserId; + } }, cancelFrameRender() { @@ -681,6 +772,8 @@ const model = { this._frameRenderHandle = null; this._frameRenderCancel = null; this._pendingFrameSrc = ""; + this._pendingFrameOptions = null; + this._frameRenderSequence += 1; }, async command(command, extra = {}) { @@ -865,11 +958,16 @@ const model = { if (snapshot.state) { this.applyActiveFrameState(snapshot.state); } - this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`); - if (this.sameBrowserId(this.switchingBrowserId, snapshotId || this.activeBrowserId)) { - this.switchingBrowserId = null; - } - this._surfaceSwitching = false; + const frameBrowserId = snapshotId || this.activeBrowserId; + this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, { + browserId: frameBrowserId, + onAccepted: () => { + if (this.sameBrowserId(this.switchingBrowserId, frameBrowserId)) { + this.switchingBrowserId = null; + } + this._surfaceSwitching = false; + }, + }); }, isSwitchingBrowser() { @@ -1371,6 +1469,7 @@ const model = { }, queueViewportSync(force = false) { + this.clearRenderedFrameIfViewportChanged(); if (this._viewportSyncTimer) { globalThis.clearTimeout(this._viewportSyncTimer); } diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py index bcbec0f67..f4bc81b17 100644 --- a/tests/test_browser_agent_regressions.py +++ b/tests/test_browser_agent_regressions.py @@ -110,6 +110,21 @@ import plugins._browser.tools.browser as browser_tool_module import plugins._browser.api.ws_browser as ws_browser_module +SMALL_JPEG_10X10 = ( + "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsL" + "DBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/" + "2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy" + "MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAKAAoDASIAAhEBAxEB/8QAFQAB" + "AAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhADE" + "AAAAKf/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAEFAqf/xAAUEQEAAAAAAAA" + "AAAAAAAAAAAAA/9oACAEDAQE/ASP/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECA" + "QE/ASP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAY/Aqf/xAAUEAEAAAAAAA" + "AAAAAAAAAAAAAA/9oACAEBAAE/ISf/2gAMAwEAAgADAAAAEP/EABQRAQAAAAAAAAA" + "AAAAAAAAAAP/aAAgBAwEBPxAk/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAgBAgEB" + "PxAk/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABPxAn/9k=" +) + + def test_browser_url_normalization_matches_address_bar_hosts(): assert normalize_url("localhost:3000") == "http://localhost:3000/" assert normalize_url("127.0.0.1:8000/path") == "http://127.0.0.1:8000/path" @@ -365,7 +380,8 @@ def test_browser_canvas_startup_waits_for_raw_viewport_settle(): encoding="utf-8" ) - assert "const CANVAS_VIEWPORT_SETTLE_MS = 260;" in js + assert "const CANVAS_VIEWPORT_SETTLE_MS = 520;" in js + assert "const SURFACE_VIEWPORT_STABLE_FRAMES = 4;" in js assert "surfaceViewportMeasurement()" in js assert "rawWidth" in js assert "rawHeight" in js @@ -454,6 +470,9 @@ def test_browser_viewer_uses_cdp_screencast_transport(): assert "viewport_width: initialViewport?.width" in browser_store assert "viewport_height: initialViewport?.height" in browser_store assert "this.frameState = data.state || null" not in browser_store + assert "function loadFrameDimensions(src)" in browser_store + assert "frameMatchesViewport(dimensions = null, viewport = null)" in browser_store + assert "requestViewportSyncAfterRejectedFrame()" in browser_store assert "overflow: hidden;" in main_html assert "object-fit: fill;" in main_html assert "image-rendering: auto;" in main_html @@ -504,19 +523,7 @@ def test_browser_runtime_and_content_helper_expose_annotation_target(): @pytest.mark.anyio async def test_browser_screencast_acknowledges_and_drops_stale_frames(): - first_image = ( - "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsL" - "DBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/" - "2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy" - "MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAKAAoDASIAAhEBAxEB/8QAFQAB" - "AAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhADE" - "AAAAKf/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAEFAqf/xAAUEQEAAAAAAAA" - "AAAAAAAAAAAAA/9oACAEDAQE/ASP/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECA" - "QE/ASP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAY/Aqf/xAAUEAEAAAAAAA" - "AAAAAAAAAAAAAA/9oACAEBAAE/ISf/2gAMAwEAAgADAAAAEP/EABQRAQAAAAAAAAA" - "AAAAAAAAAAP/aAAgBAwEBPxAk/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAgBAgEB" - "PxAk/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABPxAn/9k=" - ) + first_image = SMALL_JPEG_10X10 class FakeSession: def __init__(self): @@ -572,6 +579,43 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames(): assert session.detached is True +@pytest.mark.anyio +async def test_browser_screencast_keeps_rejecting_wrong_viewport_frames(): + class FakeSession: + def __init__(self): + self.handlers = {} + self.sent = [] + + def on(self, event, handler): + self.handlers[event] = handler + + async def send(self, method, params=None): + self.sent.append((method, params or {})) + + async def detach(self): + pass + + session = FakeSession() + screencast = _BrowserScreencast( + stream_id="stream", + browser_id=7, + session=session, + mime="image/jpeg", + ) + + await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 1118, "height": 662}) + for session_id in range(1, 14): + session.handlers["Page.screencastFrame"]( + {"data": SMALL_JPEG_10X10, "metadata": {}, "sessionId": session_id} + ) + await asyncio.sleep(0) + + assert await screencast.pop_frame() is None + assert ("Page.screencastFrameAck", {"sessionId": 13}) in session.sent + + await screencast.stop() + + def test_browser_docker_installs_full_chromium_to_persistent_cache(): script = ( PROJECT_ROOT / "docker" / "run" / "fs" / "ins" / "install_playwright.sh"