vrr/agent-zero
2
0
Fork 0
mirror of https://github.com/agent0ai/agent-zero.git synced 2026-05-19 07:59:34 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Stabilize browser viewer viewport rendering

Browse source 
Decode browser frames before display and only render frames that match the active viewer viewport, avoiding stretched stale screencast images during startup and resize.

Keep rejecting mismatched CDP screencast frames on the backend, extend canvas viewport settling, and cover the behavior with browser regression tests.

Include small browser panel CSS polish.
This commit is contained in:
Alessandro 2026-04-28 03:50:38 +02:00
parent 67bfd3e350
commit decb05a682
4 changed files with 177 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

10
plugins/_browser/helpers/runtime.py
View file

@ -111,7 +111,6 @@ class _BrowserScreencast:
self._ack_tasks: set[asyncio.Task] = set()
self._expected_width = 0
self._expected_height = 0
self._dimension_mismatches = 0
async def start(
self,
@ -125,7 +124,6 @@ class _BrowserScreencast:
height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"])))
self._expected_width = width
self._expected_height = height
self._dimension_mismatches = 0
with contextlib.suppress(Exception):
await self.session.send("Page.enable")
await self.session.send(
@ -229,10 +227,12 @@ class _BrowserScreencast:
if not size:
return True
width, height = size
if abs(width - self._expected_width) <= 2 and abs(height - self._expected_height) <= 2:
if (
abs(width - self._expected_width) <= VIEWPORT_SIZE_TOLERANCE
and abs(height - self._expected_height) <= VIEWPORT_SIZE_TOLERANCE
):
return True
self._dimension_mismatches += 1
return self._dimension_mismatches > 10
return False
@staticmethod
def _jpeg_size(data: str) -> tuple[int, int] | None:

2
plugins/_browser/webui/browser-panel.html
View file

@ -620,7 +620,7 @@
align-items: end;
gap: 6px;
min-width: 0;
padding-bottom: 1px;
padding-bottom: 7px;
}
.browser-annotate-toggle {

127
plugins/_browser/webui/browser-store.js
View file

@ -13,8 +13,9 @@ const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000;
const BROWSER_CONFIG_REFRESH_MS = 15000;
const VIEWPORT_SYNC_DEBOUNCE_MS = 220;
const VIEWPORT_SYNC_SIZE_TOLERANCE = 4;
const CANVAS_VIEWPORT_SETTLE_MS = 260;
const SURFACE_VIEWPORT_STABLE_FRAMES = 2;
const CANVAS_VIEWPORT_SETTLE_MS = 520;
const SURFACE_VIEWPORT_STABLE_FRAMES = 4;
const FRAME_REJECT_SYNC_COOLDOWN_MS = 600;
const ANNOTATION_DRAG_THRESHOLD = 6;
const ANNOTATION_MAX_COMMENTS = 24;
const ANNOTATION_DOM_LIMIT = 1200;
@ -55,6 +56,34 @@ function nextAnimationFrame() {
});
}
function loadFrameDimensions(src) {
return new Promise((resolve) => {
if (!src) {
resolve(null);
return;
}
const image = new Image();
let settled = false;
const finish = (dimensions) => {
if (settled) return;
settled = true;
resolve(dimensions);
};
image.onload = () => finish({
width: image.naturalWidth || 0,
height: image.naturalHeight || 0,
});
image.onerror = () => finish(null);
image.src = src;
if (image.complete) {
image.onload();
}
});
}
const model = {
loading: true,
error: "",
@ -79,13 +108,17 @@ const model = {
_frameOff: null,
_stateOff: null,
_lastFrameAt: 0,
_lastFrameDimensions: null,
_pendingFrameSrc: "",
_pendingFrameOptions: null,
_frameRenderHandle: null,
_frameRenderCancel: null,
_frameRenderSequence: 0,
_floatingCleanup: null,
_stageElement: null,
_stageResizeObserver: null,
_viewportSyncTimer: null,
_lastFrameRejectSyncAt: 0,
_lastViewportKey: "",
_lastViewport: null,
_annotationPointer: null,
@ -466,6 +499,7 @@ const model = {
resetRenderedFrame() {
this.cancelFrameRender();
this.frameSrc = "";
this._lastFrameDimensions = null;
this._lastFrameAt = 0;
},
@ -599,11 +633,16 @@ const model = {
this.address = data.state.currentUrl;
}
if (data.image) {
this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`);
if (this.sameBrowserId(this.switchingBrowserId, incomingBrowserId || this.activeBrowserId)) {
this.switchingBrowserId = null;
}
this._surfaceSwitching = false;
const frameBrowserId = incomingBrowserId || this.activeBrowserId;
this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`, {
browserId: frameBrowserId,
onAccepted: () => {
if (this.sameBrowserId(this.switchingBrowserId, frameBrowserId)) {
this.switchingBrowserId = null;
}
this._surfaceSwitching = false;
},
});
} else {
this.cancelFrameRender();
if (!data.state) {
@ -654,8 +693,9 @@ const model = {
}
},
queueFrameRender(frameSrc) {
queueFrameRender(frameSrc, options = {}) {
this._pendingFrameSrc = frameSrc;
this._pendingFrameOptions = options || null;
if (this._frameRenderHandle) return;
const schedule = globalThis.requestAnimationFrame?.bind(globalThis);
if (schedule) {
@ -670,8 +710,59 @@ const model = {
flushFrameRender() {
this._frameRenderHandle = null;
this._frameRenderCancel = null;
this.frameSrc = this._pendingFrameSrc || "";
const frameSrc = this._pendingFrameSrc || "";
const options = this._pendingFrameOptions || {};
this._pendingFrameSrc = "";
this._pendingFrameOptions = null;
const sequence = this._frameRenderSequence + 1;
this._frameRenderSequence = sequence;
void this.renderDecodedFrame(frameSrc, options, sequence);
},
async renderDecodedFrame(frameSrc, options = {}, sequence = 0) {
if (!frameSrc) {
if (sequence === this._frameRenderSequence) {
this.frameSrc = "";
}
return;
}
const dimensions = await loadFrameDimensions(frameSrc);
if (sequence !== this._frameRenderSequence) return;
const viewport = this.currentViewportSize() || this._lastViewport;
if (!this.frameMatchesViewport(dimensions, viewport)) {
this.requestViewportSyncAfterRejectedFrame();
return;
}
this.frameSrc = frameSrc;
this._lastFrameDimensions = dimensions;
this._lastFrameAt = Date.now();
options?.onAccepted?.();
},
frameMatchesViewport(dimensions = null, viewport = null) {
if (!dimensions?.width || !dimensions?.height || !viewport?.width || !viewport?.height) {
return false;
}
return Math.abs(Number(dimensions.width) - Number(viewport.width)) <= VIEWPORT_SYNC_SIZE_TOLERANCE
&& Math.abs(Number(dimensions.height) - Number(viewport.height)) <= VIEWPORT_SYNC_SIZE_TOLERANCE;
},
requestViewportSyncAfterRejectedFrame() {
const now = Date.now();
if (now - this._lastFrameRejectSyncAt < FRAME_REJECT_SYNC_COOLDOWN_MS) return;
this._lastFrameRejectSyncAt = now;
this.queueViewportSync(true);
},
clearRenderedFrameIfViewportChanged() {
const viewport = this.currentViewportSize();
if (!this.frameSrc || !this._lastFrameDimensions || !viewport) return;
if (this.frameMatchesViewport(this._lastFrameDimensions, viewport)) return;
this.resetRenderedFrame();
if (this.activeBrowserId) {
this._surfaceSwitching = true;
this.switchingBrowserId = this.activeBrowserId;
}
},
cancelFrameRender() {
@ -681,6 +772,8 @@ const model = {
this._frameRenderHandle = null;
this._frameRenderCancel = null;
this._pendingFrameSrc = "";
this._pendingFrameOptions = null;
this._frameRenderSequence += 1;
},
async command(command, extra = {}) {
@ -865,11 +958,16 @@ const model = {
if (snapshot.state) {
this.applyActiveFrameState(snapshot.state);
}
this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`);
if (this.sameBrowserId(this.switchingBrowserId, snapshotId || this.activeBrowserId)) {
this.switchingBrowserId = null;
}
this._surfaceSwitching = false;
const frameBrowserId = snapshotId || this.activeBrowserId;
this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, {
browserId: frameBrowserId,
onAccepted: () => {
if (this.sameBrowserId(this.switchingBrowserId, frameBrowserId)) {
this.switchingBrowserId = null;
}
this._surfaceSwitching = false;
},
});
},
isSwitchingBrowser() {
@ -1371,6 +1469,7 @@ const model = {
},
queueViewportSync(force = false) {
this.clearRenderedFrameIfViewportChanged();
if (this._viewportSyncTimer) {
globalThis.clearTimeout(this._viewportSyncTimer);
}

72
tests/test_browser_agent_regressions.py
View file

@ -110,6 +110,21 @@ import plugins._browser.tools.browser as browser_tool_module
import plugins._browser.api.ws_browser as ws_browser_module
SMALL_JPEG_10X10 = (
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsL"
"DBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/"
"2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy"
"MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAKAAoDASIAAhEBAxEB/8QAFQAB"
"AAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhADE"
"AAAAKf/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAEFAqf/xAAUEQEAAAAAAAA"
"AAAAAAAAAAAAA/9oACAEDAQE/ASP/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECA"
"QE/ASP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAY/Aqf/xAAUEAEAAAAAAA"
"AAAAAAAAAAAAAA/9oACAEBAAE/ISf/2gAMAwEAAgADAAAAEP/EABQRAQAAAAAAAAA"
"AAAAAAAAAAP/aAAgBAwEBPxAk/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAgBAgEB"
"PxAk/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABPxAn/9k="
)
def test_browser_url_normalization_matches_address_bar_hosts():
assert normalize_url("localhost:3000") == "http://localhost:3000/"
assert normalize_url("127.0.0.1:8000/path") == "http://127.0.0.1:8000/path"
@ -365,7 +380,8 @@ def test_browser_canvas_startup_waits_for_raw_viewport_settle():
encoding="utf-8"
)
assert "const CANVAS_VIEWPORT_SETTLE_MS = 260;" in js
assert "const CANVAS_VIEWPORT_SETTLE_MS = 520;" in js
assert "const SURFACE_VIEWPORT_STABLE_FRAMES = 4;" in js
assert "surfaceViewportMeasurement()" in js
assert "rawWidth" in js
assert "rawHeight" in js
@ -454,6 +470,9 @@ def test_browser_viewer_uses_cdp_screencast_transport():
assert "viewport_width: initialViewport?.width" in browser_store
assert "viewport_height: initialViewport?.height" in browser_store
assert "this.frameState = data.state || null" not in browser_store
assert "function loadFrameDimensions(src)" in browser_store
assert "frameMatchesViewport(dimensions = null, viewport = null)" in browser_store
assert "requestViewportSyncAfterRejectedFrame()" in browser_store
assert "overflow: hidden;" in main_html
assert "object-fit: fill;" in main_html
assert "image-rendering: auto;" in main_html
@ -504,19 +523,7 @@ def test_browser_runtime_and_content_helper_expose_annotation_target():
@pytest.mark.anyio
async def test_browser_screencast_acknowledges_and_drops_stale_frames():
first_image = (
"/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsL"
"DBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/"
"2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy"
"MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAKAAoDASIAAhEBAxEB/8QAFQAB"
"AAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhADE"
"AAAAKf/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAEFAqf/xAAUEQEAAAAAAAA"
"AAAAAAAAAAAAA/9oACAEDAQE/ASP/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECA"
"QE/ASP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAY/Aqf/xAAUEAEAAAAAAA"
"AAAAAAAAAAAAAA/9oACAEBAAE/ISf/2gAMAwEAAgADAAAAEP/EABQRAQAAAAAAAAA"
"AAAAAAAAAAP/aAAgBAwEBPxAk/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAgBAgEB"
"PxAk/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABPxAn/9k="
)
first_image = SMALL_JPEG_10X10
class FakeSession:
def __init__(self):
@ -572,6 +579,43 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():
assert session.detached is True
@pytest.mark.anyio
async def test_browser_screencast_keeps_rejecting_wrong_viewport_frames():
class FakeSession:
def __init__(self):
self.handlers = {}
self.sent = []
def on(self, event, handler):
self.handlers[event] = handler
async def send(self, method, params=None):
self.sent.append((method, params or {}))
async def detach(self):
pass
session = FakeSession()
screencast = _BrowserScreencast(
stream_id="stream",
browser_id=7,
session=session,
mime="image/jpeg",
)
await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 1118, "height": 662})
for session_id in range(1, 14):
session.handlers["Page.screencastFrame"](
{"data": SMALL_JPEG_10X10, "metadata": {}, "sessionId": session_id}
)
await asyncio.sleep(0)
assert await screencast.pop_frame() is None
assert ("Page.screencastFrameAck", {"sessionId": 13}) in session.sent
await screencast.stop()
def test_browser_docker_installs_full_chromium_to_persistent_cache():
script = (
PROJECT_ROOT / "docker" / "run" / "fs" / "ins" / "install_playwright.sh"