diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py index db459de91..3111b3068 100644 --- a/plugins/_browser/helpers/runtime.py +++ b/plugins/_browser/helpers/runtime.py @@ -111,7 +111,6 @@ class _BrowserScreencast: self._ack_tasks: set[asyncio.Task] = set() self._expected_width = 0 self._expected_height = 0 - self._dimension_mismatches = 0 async def start( self, @@ -125,7 +124,6 @@ class _BrowserScreencast: height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"]))) self._expected_width = width self._expected_height = height - self._dimension_mismatches = 0 with contextlib.suppress(Exception): await self.session.send("Page.enable") await self.session.send( @@ -229,10 +227,12 @@ class _BrowserScreencast: if not size: return True width, height = size - if abs(width - self._expected_width) <= 2 and abs(height - self._expected_height) <= 2: + if ( + abs(width - self._expected_width) <= VIEWPORT_SIZE_TOLERANCE + and abs(height - self._expected_height) <= VIEWPORT_SIZE_TOLERANCE + ): return True - self._dimension_mismatches += 1 - return self._dimension_mismatches > 10 + return False @staticmethod def _jpeg_size(data: str) -> tuple[int, int] | None: diff --git a/plugins/_browser/webui/browser-panel.html b/plugins/_browser/webui/browser-panel.html index e10baaaf5..3ff50ef5d 100644 --- a/plugins/_browser/webui/browser-panel.html +++ b/plugins/_browser/webui/browser-panel.html @@ -620,7 +620,7 @@ align-items: end; gap: 6px; min-width: 0; - padding-bottom: 1px; + padding-bottom: 7px; } .browser-annotate-toggle { diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.js index b7db50928..bb12b161e 100644 --- a/plugins/_browser/webui/browser-store.js +++ b/plugins/_browser/webui/browser-store.js @@ -13,8 +13,9 @@ const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000; const BROWSER_CONFIG_REFRESH_MS = 15000; const VIEWPORT_SYNC_DEBOUNCE_MS = 220; const VIEWPORT_SYNC_SIZE_TOLERANCE = 4; -const CANVAS_VIEWPORT_SETTLE_MS = 260; -const SURFACE_VIEWPORT_STABLE_FRAMES = 2; +const CANVAS_VIEWPORT_SETTLE_MS = 520; +const SURFACE_VIEWPORT_STABLE_FRAMES = 4; +const FRAME_REJECT_SYNC_COOLDOWN_MS = 600; const ANNOTATION_DRAG_THRESHOLD = 6; const ANNOTATION_MAX_COMMENTS = 24; const ANNOTATION_DOM_LIMIT = 1200; @@ -55,6 +56,34 @@ function nextAnimationFrame() { }); } +function loadFrameDimensions(src) { + return new Promise((resolve) => { + if (!src) { + resolve(null); + return; + } + + const image = new Image(); + let settled = false; + const finish = (dimensions) => { + if (settled) return; + settled = true; + resolve(dimensions); + }; + + image.onload = () => finish({ + width: image.naturalWidth || 0, + height: image.naturalHeight || 0, + }); + image.onerror = () => finish(null); + image.src = src; + + if (image.complete) { + image.onload(); + } + }); +} + const model = { loading: true, error: "", @@ -79,13 +108,17 @@ const model = { _frameOff: null, _stateOff: null, _lastFrameAt: 0, + _lastFrameDimensions: null, _pendingFrameSrc: "", + _pendingFrameOptions: null, _frameRenderHandle: null, _frameRenderCancel: null, + _frameRenderSequence: 0, _floatingCleanup: null, _stageElement: null, _stageResizeObserver: null, _viewportSyncTimer: null, + _lastFrameRejectSyncAt: 0, _lastViewportKey: "", _lastViewport: null, _annotationPointer: null, @@ -466,6 +499,7 @@ const model = { resetRenderedFrame() { this.cancelFrameRender(); this.frameSrc = ""; + this._lastFrameDimensions = null; this._lastFrameAt = 0; }, @@ -599,11 +633,16 @@ const model = { this.address = data.state.currentUrl; } if (data.image) { - this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`); - if (this.sameBrowserId(this.switchingBrowserId, incomingBrowserId || this.activeBrowserId)) { - this.switchingBrowserId = null; - } - this._surfaceSwitching = false; + const frameBrowserId = incomingBrowserId || this.activeBrowserId; + this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`, { + browserId: frameBrowserId, + onAccepted: () => { + if (this.sameBrowserId(this.switchingBrowserId, frameBrowserId)) { + this.switchingBrowserId = null; + } + this._surfaceSwitching = false; + }, + }); } else { this.cancelFrameRender(); if (!data.state) { @@ -654,8 +693,9 @@ const model = { } }, - queueFrameRender(frameSrc) { + queueFrameRender(frameSrc, options = {}) { this._pendingFrameSrc = frameSrc; + this._pendingFrameOptions = options || null; if (this._frameRenderHandle) return; const schedule = globalThis.requestAnimationFrame?.bind(globalThis); if (schedule) { @@ -670,8 +710,59 @@ const model = { flushFrameRender() { this._frameRenderHandle = null; this._frameRenderCancel = null; - this.frameSrc = this._pendingFrameSrc || ""; + const frameSrc = this._pendingFrameSrc || ""; + const options = this._pendingFrameOptions || {}; this._pendingFrameSrc = ""; + this._pendingFrameOptions = null; + const sequence = this._frameRenderSequence + 1; + this._frameRenderSequence = sequence; + void this.renderDecodedFrame(frameSrc, options, sequence); + }, + + async renderDecodedFrame(frameSrc, options = {}, sequence = 0) { + if (!frameSrc) { + if (sequence === this._frameRenderSequence) { + this.frameSrc = ""; + } + return; + } + const dimensions = await loadFrameDimensions(frameSrc); + if (sequence !== this._frameRenderSequence) return; + const viewport = this.currentViewportSize() || this._lastViewport; + if (!this.frameMatchesViewport(dimensions, viewport)) { + this.requestViewportSyncAfterRejectedFrame(); + return; + } + this.frameSrc = frameSrc; + this._lastFrameDimensions = dimensions; + this._lastFrameAt = Date.now(); + options?.onAccepted?.(); + }, + + frameMatchesViewport(dimensions = null, viewport = null) { + if (!dimensions?.width || !dimensions?.height || !viewport?.width || !viewport?.height) { + return false; + } + return Math.abs(Number(dimensions.width) - Number(viewport.width)) <= VIEWPORT_SYNC_SIZE_TOLERANCE + && Math.abs(Number(dimensions.height) - Number(viewport.height)) <= VIEWPORT_SYNC_SIZE_TOLERANCE; + }, + + requestViewportSyncAfterRejectedFrame() { + const now = Date.now(); + if (now - this._lastFrameRejectSyncAt < FRAME_REJECT_SYNC_COOLDOWN_MS) return; + this._lastFrameRejectSyncAt = now; + this.queueViewportSync(true); + }, + + clearRenderedFrameIfViewportChanged() { + const viewport = this.currentViewportSize(); + if (!this.frameSrc || !this._lastFrameDimensions || !viewport) return; + if (this.frameMatchesViewport(this._lastFrameDimensions, viewport)) return; + this.resetRenderedFrame(); + if (this.activeBrowserId) { + this._surfaceSwitching = true; + this.switchingBrowserId = this.activeBrowserId; + } }, cancelFrameRender() { @@ -681,6 +772,8 @@ const model = { this._frameRenderHandle = null; this._frameRenderCancel = null; this._pendingFrameSrc = ""; + this._pendingFrameOptions = null; + this._frameRenderSequence += 1; }, async command(command, extra = {}) { @@ -865,11 +958,16 @@ const model = { if (snapshot.state) { this.applyActiveFrameState(snapshot.state); } - this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`); - if (this.sameBrowserId(this.switchingBrowserId, snapshotId || this.activeBrowserId)) { - this.switchingBrowserId = null; - } - this._surfaceSwitching = false; + const frameBrowserId = snapshotId || this.activeBrowserId; + this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, { + browserId: frameBrowserId, + onAccepted: () => { + if (this.sameBrowserId(this.switchingBrowserId, frameBrowserId)) { + this.switchingBrowserId = null; + } + this._surfaceSwitching = false; + }, + }); }, isSwitchingBrowser() { @@ -1371,6 +1469,7 @@ const model = { }, queueViewportSync(force = false) { + this.clearRenderedFrameIfViewportChanged(); if (this._viewportSyncTimer) { globalThis.clearTimeout(this._viewportSyncTimer); } diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py index bcbec0f67..f4bc81b17 100644 --- a/tests/test_browser_agent_regressions.py +++ b/tests/test_browser_agent_regressions.py @@ -110,6 +110,21 @@ import plugins._browser.tools.browser as browser_tool_module import plugins._browser.api.ws_browser as ws_browser_module +SMALL_JPEG_10X10 = ( + "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsL" + "DBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/" + "2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy" + "MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAKAAoDASIAAhEBAxEB/8QAFQAB" + "AAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhADE" + "AAAAKf/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAEFAqf/xAAUEQEAAAAAAAA" + "AAAAAAAAAAAAA/9oACAEDAQE/ASP/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECA" + "QE/ASP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAY/Aqf/xAAUEAEAAAAAAA" + "AAAAAAAAAAAAAA/9oACAEBAAE/ISf/2gAMAwEAAgADAAAAEP/EABQRAQAAAAAAAAA" + "AAAAAAAAAAP/aAAgBAwEBPxAk/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAgBAgEB" + "PxAk/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABPxAn/9k=" +) + + def test_browser_url_normalization_matches_address_bar_hosts(): assert normalize_url("localhost:3000") == "http://localhost:3000/" assert normalize_url("127.0.0.1:8000/path") == "http://127.0.0.1:8000/path" @@ -365,7 +380,8 @@ def test_browser_canvas_startup_waits_for_raw_viewport_settle(): encoding="utf-8" ) - assert "const CANVAS_VIEWPORT_SETTLE_MS = 260;" in js + assert "const CANVAS_VIEWPORT_SETTLE_MS = 520;" in js + assert "const SURFACE_VIEWPORT_STABLE_FRAMES = 4;" in js assert "surfaceViewportMeasurement()" in js assert "rawWidth" in js assert "rawHeight" in js @@ -454,6 +470,9 @@ def test_browser_viewer_uses_cdp_screencast_transport(): assert "viewport_width: initialViewport?.width" in browser_store assert "viewport_height: initialViewport?.height" in browser_store assert "this.frameState = data.state || null" not in browser_store + assert "function loadFrameDimensions(src)" in browser_store + assert "frameMatchesViewport(dimensions = null, viewport = null)" in browser_store + assert "requestViewportSyncAfterRejectedFrame()" in browser_store assert "overflow: hidden;" in main_html assert "object-fit: fill;" in main_html assert "image-rendering: auto;" in main_html @@ -504,19 +523,7 @@ def test_browser_runtime_and_content_helper_expose_annotation_target(): @pytest.mark.anyio async def test_browser_screencast_acknowledges_and_drops_stale_frames(): - first_image = ( - "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsL" - "DBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/" - "2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy" - "MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAKAAoDASIAAhEBAxEB/8QAFQAB" - "AAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhADE" - "AAAAKf/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAEFAqf/xAAUEQEAAAAAAAA" - "AAAAAAAAAAAAA/9oACAEDAQE/ASP/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECA" - "QE/ASP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAY/Aqf/xAAUEAEAAAAAAA" - "AAAAAAAAAAAAAA/9oACAEBAAE/ISf/2gAMAwEAAgADAAAAEP/EABQRAQAAAAAAAAA" - "AAAAAAAAAAP/aAAgBAwEBPxAk/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAgBAgEB" - "PxAk/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABPxAn/9k=" - ) + first_image = SMALL_JPEG_10X10 class FakeSession: def __init__(self): @@ -572,6 +579,43 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames(): assert session.detached is True +@pytest.mark.anyio +async def test_browser_screencast_keeps_rejecting_wrong_viewport_frames(): + class FakeSession: + def __init__(self): + self.handlers = {} + self.sent = [] + + def on(self, event, handler): + self.handlers[event] = handler + + async def send(self, method, params=None): + self.sent.append((method, params or {})) + + async def detach(self): + pass + + session = FakeSession() + screencast = _BrowserScreencast( + stream_id="stream", + browser_id=7, + session=session, + mime="image/jpeg", + ) + + await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 1118, "height": 662}) + for session_id in range(1, 14): + session.handlers["Page.screencastFrame"]( + {"data": SMALL_JPEG_10X10, "metadata": {}, "sessionId": session_id} + ) + await asyncio.sleep(0) + + assert await screencast.pop_frame() is None + assert ("Page.screencastFrameAck", {"sessionId": 13}) in session.sent + + await screencast.stop() + + def test_browser_docker_installs_full_chromium_to_persistent_cache(): script = ( PROJECT_ROOT / "docker" / "run" / "fs" / "ins" / "install_playwright.sh"