Smooth browser viewer rendering and frame push

Render live browser screencast frames through a canvas/ImageBitmap path while keeping the existing image/data-url path for snapshots and fallback rendering.

Attach WebSocket streams as runtime screencast consumers so delivered frames are pushed to the server loop and acknowledged after emit, removing the per-frame read_screencast_frame runtime round trip.

plugins/_browser/AGENTS.md

@@ -19,6 +19,8 @@
 - Preserve Playwright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding.
 - Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation.
 - Default the visible WebUI Browser to live CDP screencast for responsiveness. Keep lightweight CDP/DOM state snapshots as the fallback transport.
+- Paint live screencast frames through the Browser panel canvas/ImageBitmap path when available; keep the `<img>`/data URL path for snapshots and fallback rendering.
+- Push internal screencast frames from the runtime to the WebSocket consumer after subscription; keep `read/pop_screencast_frame` as fallback/tooling APIs, not the WebUI hot path.
 - Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback. Do not let the WebUI advertise binary frames unless its Socket.IO client reconstructs attachments as real `Blob`, `ArrayBuffer`, or typed-array values.
 - Keep narrow WebUI Browser controls usable by grouping navigation with Annotate/settings above a full-width address bar.
 - Browser URL-intent handling must only claim web URL schemes and leave custom Agent Zero schemes to their owning surfaces.
@@ -36,6 +38,7 @@
 ## Verification
 
 - Smoke-test browser launch, navigation, DOM capture, and WebUI viewer after runtime changes.
+- For viewer render-path changes, verify the live Browser panel paints a screencast frame on canvas with `frameSrc` empty and snapshots still falling back to the image path.
 - Run browser prompt/skill regression tests after changing browser prompt or Browser plugin skills.
 
 ## Child DOX Index

plugins/_browser/api/ws_browser.py

@@ -451,7 +451,37 @@ class WsBrowser(WsHandler):
                 last_state_refresh = 0.0
                 last_state_signature = self._state_signature(active_id, browsers)
                 frame_sequence = 0
+                server_loop = asyncio.get_running_loop()
+                stop_event = asyncio.Event()
+
+                async def emit_frame(frame: dict[str, Any]) -> None:
+                    nonlocal frame_sequence
+                    try:
+                        frame_sequence += 1
+                        payload = self._frame_payload(
+                            frame,
+                            context_id=context_id,
+                            viewer_id=viewer_id,
+                            browser_id=active_id,
+                            sequence=frame_sequence,
+                            binary_frames=binary_frames,
+                        )
+                        if not slim_frames:
+                            payload["browsers"] = browsers
+                            payload["state"] = state
+                        await self._emit_to_connected_viewer(sid, "browser_viewer_frame", payload)
+                    except BaseException:
+                        stop_event.set()
+                        raise
+
+                def frame_consumer(frame: dict[str, Any]):
+                    return asyncio.run_coroutine_threadsafe(emit_frame(frame), server_loop)
+
+                await runtime.call("attach_screencast_consumer", stream_id, frame_consumer)
+
                 while True:
+                    if stop_event.is_set():
+                        break
                     now = time.monotonic()
                     if now - last_state_refresh >= FRAME_STATE_REFRESH_SECONDS:
                         listing = await runtime.call("list")
@@ -475,29 +505,10 @@ class WsBrowser(WsHandler):
                         last_state_refresh = now
 
                     try:
-                        frame = await runtime.call(
-                            "read_screencast_frame",
-                            stream_id,
-                            timeout=FRAME_READ_TIMEOUT_SECONDS,
-                        )
-                    except KeyError:
+                        await asyncio.wait_for(stop_event.wait(), timeout=FRAME_READ_TIMEOUT_SECONDS)
                         break
                     except TimeoutError:
                         continue
-
-                    frame_sequence += 1
-                    payload = self._frame_payload(
-                        frame,
-                        context_id=context_id,
-                        viewer_id=viewer_id,
-                        browser_id=active_id,
-                        sequence=frame_sequence,
-                        binary_frames=binary_frames,
-                    )
-                    if not slim_frames:
-                        payload["browsers"] = browsers
-                        payload["state"] = state
-                    await self._emit_to_connected_viewer(sid, "browser_viewer_frame", payload)
             except asyncio.CancelledError:
                 raise
             except Exception:

plugins/_browser/helpers/runtime.py

@@ -317,6 +317,7 @@ class _BrowserScreencast:
         self.browser_id = browser_id
         self.session = session
         self.mime = mime
+        self.frame_consumer: Any | None = None
         self.queue = asyncio.Queue(maxsize=1)
         self.stopped = False
         self._ack_tasks: set[asyncio.Task] = set()
@@ -398,6 +399,12 @@ class _BrowserScreencast:
             raise RuntimeError("Browser screencast stopped.")
         return frame
 
+    async def attach_consumer(self, frame_consumer: Any) -> None:
+        self.frame_consumer = frame_consumer
+        frame = await self.pop_frame()
+        if frame:
+            await self._deliver_frame(frame)
+
     async def stop(self) -> None:
         if self.stopped:
             return
@@ -423,6 +430,7 @@ class _BrowserScreencast:
         task.add_done_callback(self._ack_tasks.discard)
 
     async def _handle_frame(self, params: dict[str, Any]) -> None:
+        stop_after_ack = False
         try:
             data = params.get("data") or ""
             if data:
@@ -432,7 +440,7 @@ class _BrowserScreencast:
                     metadata["jpegWidth"], metadata["jpegHeight"] = size
                 metadata["expectedWidth"] = self._expected_width
                 metadata["expectedHeight"] = self._expected_height
-                self._queue_latest(
+                await self._deliver_frame(
                     {
                         "browser_id": self.browser_id,
                         "mime": self.mime,
@@ -440,6 +448,13 @@ class _BrowserScreencast:
                         "metadata": metadata,
                     }
                 )
+        except asyncio.CancelledError:
+            stop_after_ack = True
+        except Exception:
+            if self.frame_consumer:
+                stop_after_ack = True
+            else:
+                raise
         finally:
             session_id = params.get("sessionId")
             if session_id is not None and not self.stopped:
@@ -448,6 +463,16 @@ class _BrowserScreencast:
                         "Page.screencastFrameAck",
                         {"sessionId": int(session_id)},
                     )
+            if stop_after_ack:
+                self.stopped = True
+
+    async def _deliver_frame(self, frame: dict[str, Any]) -> None:
+        if not self.frame_consumer:
+            self._queue_latest(frame)
+            return
+        future = self.frame_consumer(frame)
+        if future is not None:
+            await asyncio.wrap_future(future)
 
     def _queue_latest(self, frame: dict[str, Any]) -> None:
         self._drop_queued_frames()
@@ -1669,6 +1694,12 @@ class _BrowserRuntimeCore:
             raise KeyError("Browser screencast is not active.")
         return await screencast.pop_frame()
 
+    async def attach_screencast_consumer(self, stream_id: str, frame_consumer: Any) -> None:
+        screencast = self.screencasts.get(str(stream_id or ""))
+        if not screencast:
+            raise KeyError("Browser screencast is not active.")
+        await screencast.attach_consumer(frame_consumer)
+
     async def stop_screencast(self, stream_id: str) -> None:
         screencast = self.screencasts.pop(str(stream_id or ""), None)
         if screencast:

plugins/_browser/webui/browser-panel.html

@@ -187,12 +187,17 @@
 	        <div class="browser-stage" tabindex="0" @click="$el.focus()"
 	          :class="{ 'is-annotating': $store.browserPage.annotating }"
 	          @wheel.prevent="$store.browserPage.handleStageWheel($event)">
+	          <canvas class="browser-frame browser-frame-canvas"
+	            x-show="$store.browserPage.frameCanvasReady"
+	            x-init="$store.browserPage.attachFrameCanvas($el)"
+	            @click="$store.browserPage.sendMouse('click', $event)"
+	            @mousemove.throttle.250ms="$store.browserPage.sendMouse('move', $event)"></canvas>
 	          <template x-if="$store.browserPage.frameSrc">
-            <img class="browser-frame" :src="$store.browserPage.frameSrc"
+            <img class="browser-frame browser-frame-image" :src="$store.browserPage.frameSrc"
               @click="$store.browserPage.sendMouse('click', $event)"
               @mousemove.throttle.250ms="$store.browserPage.sendMouse('move', $event)" draggable="false" />
           </template>
-	          <template x-if="$store.browserPage.annotating && $store.browserPage.frameSrc">
+	          <template x-if="$store.browserPage.annotating && $store.browserPage.hasFrame()">
 	            <div class="browser-annotation-layer"
 	              :class="{ 'is-busy': $store.browserPage.annotationBusy }"
 	              @pointerdown.stop.prevent="$store.browserPage.startAnnotationSelection($event)"
@@ -267,7 +272,7 @@
 	                @click="$store.browserPage.sendAnnotationsToChat()">Send now</button>
 	            </div>
 	          </div>
-	          <template x-if="!$store.browserPage.frameSrc && !$store.browserPage.isBusy()">
+	          <template x-if="!$store.browserPage.hasFrame() && !$store.browserPage.isBusy()">
             <div class="browser-empty">
               <span class="material-symbols-outlined">captive_portal</span>
               <button class="btn btn-field" @click="$store.browserPage.command('open')">Open

plugins/_browser/webui/browser-store.js

@@ -37,6 +37,7 @@ const LOCAL_EDITABLE_SELECTOR = "input, textarea, select, [contenteditable]";
 const BROWSER_BINARY_FRAME_REQUESTS_ENABLED = false;
 const BROWSER_BINARY_PAYLOADS_SUPPORTED = typeof Blob === "function"
   && typeof globalThis.URL?.createObjectURL === "function";
+const BROWSER_CANVAS_FRAMES_SUPPORTED = typeof globalThis.createImageBitmap === "function";
 
 function makeViewerToken() {
   return globalThis.crypto?.randomUUID?.()
@@ -130,6 +131,7 @@ function frameImageSource(data = {}) {
     const src = globalThis.URL.createObjectURL(blob);
     return {
       src,
+      blob,
       objectUrl: src,
       cleanup: () => globalThis.URL.revokeObjectURL(src),
     };
@@ -143,6 +145,16 @@ function frameImageSource(data = {}) {
   };
 }
 
+async function loadFrameBitmap(src, options = {}) {
+  if (!BROWSER_CANVAS_FRAMES_SUPPORTED || !src) return null;
+  try {
+    const blob = options.blob || await fetch(src).then((response) => response.blob());
+    return await globalThis.createImageBitmap(blob);
+  } catch {
+    return null;
+  }
+}
+
 const model = {
   loading: true,
   error: "",
@@ -153,6 +165,7 @@ const model = {
   activeBrowserContextId: "",
   address: "",
   frameSrc: "",
+  frameCanvasReady: false,
   frameState: null,
   viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST,
   liveScreencastEnabled: true,
@@ -179,6 +192,7 @@ const model = {
   _frameRenderHandle: null,
   _frameRenderCancel: null,
   _frameRenderSequence: 0,
+  _frameCanvas: null,
   _floatingCleanup: null,
   _stageElement: null,
   _stageResizeObserver: null,
@@ -473,6 +487,7 @@ const model = {
       this.address = "";
       this.frameState = null;
       this.clearFrameSrc();
+      this.clearFrameCanvas();
       if (this.contextId) {
         await this.connectViewer();
       }
@@ -784,11 +799,13 @@ const model = {
   },
 
   releaseSurfaceBindings() {
+    this.freezeCanvasFrameToImage();
     this._floatingCleanup?.();
     this._floatingCleanup = null;
     this._stageResizeObserver?.disconnect?.();
     this._stageResizeObserver = null;
     this._stageElement = null;
+    this._frameCanvas = null;
   },
 
   isCanvasSurfaceVisible(element = null) {
@@ -843,7 +860,7 @@ const model = {
     this._surfaceMounted = true;
     this._surfaceOpenedAt = Date.now();
     this._lastViewportKey = "";
-    if (this.frameSrc && !targetChanged) {
+    if (this.hasFrame() && !targetChanged) {
       this._surfaceSwitching = false;
       this.switchingBrowserId = null;
       return;
@@ -864,12 +881,13 @@ const model = {
   resetRenderedFrame() {
     this.cancelFrameRender();
     this.clearFrameSrc();
+    this.clearFrameCanvas();
     this._lastFrameDimensions = null;
     this._lastFrameAt = 0;
   },
 
   resetRenderedFrameIfViewportChanged(viewport = null, requestedBrowserId = null, requestedContextId = "") {
-    if (!viewport || !this.frameSrc || !this._lastViewport) return;
+    if (!viewport || !this.hasFrame() || !this._lastViewport) return;
     const targetBrowserId = requestedBrowserId || this.activeBrowserId || this.firstBrowserId();
     const targetContextId = this.normalizeContextId(requestedContextId || this.contextIdForBrowserId(targetBrowserId) || this.activeBrowserContextId);
     if (!this.sameBrowserTab(this._lastViewport.browserId, this._lastViewport.contextId, targetBrowserId, targetContextId)) return;
@@ -1107,7 +1125,9 @@ const model = {
             browserId: frameBrowserId,
             contextId: incomingContextId,
             dimensions: this.frameDimensionsFromData(data),
+            blob: frameImage.blob,
             objectUrl: frameImage.objectUrl,
+            useCanvas: true,
             cleanup: frameImage.cleanup,
             onAccepted: () => {
               if (
@@ -1122,12 +1142,14 @@ const model = {
         } else if (!data.state) {
           this.cancelFrameRender();
           this.clearFrameSrc();
+          this.clearFrameCanvas();
         }
         if (!data.image && !data.state) {
           if (!this.activeBrowserId) {
             this.setActiveBrowserId(null, "");
             this.frameState = null;
             this.clearFrameSrc();
+            this.clearFrameCanvas();
           }
         }
         this._lastFrameAt = Date.now();
@@ -1229,11 +1251,21 @@ const model = {
     if (!frameSrc) {
       if (sequence === this._frameRenderSequence) {
         this.clearFrameSrc();
+        this.clearFrameCanvas();
       }
       return;
     }
-    const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc);
+    let bitmap = null;
+    let dimensions = options?.dimensions || null;
+    if (options?.useCanvas && this.canUseCanvasFrames()) {
+      bitmap = await loadFrameBitmap(frameSrc, options);
+      if (bitmap) {
+        dimensions ||= { width: bitmap.width || 0, height: bitmap.height || 0 };
+      }
+    }
+    dimensions ||= await loadFrameDimensions(frameSrc);
     if (sequence !== this._frameRenderSequence || surfaceSequence !== this._surfaceOpenSequence) {
+      bitmap?.close?.();
       options?.cleanup?.();
       return;
     }
@@ -1241,13 +1273,21 @@ const model = {
     if (!this.frameMatchesViewport(dimensions, viewport)) {
       this.requestViewportSyncAfterRejectedFrame();
       if (!this.shouldAcceptMismatchedFrame(dimensions)) {
+        bitmap?.close?.();
         options?.cleanup?.();
         return;
       }
     }
-    this.releaseRenderedFrameUrl(frameSrc);
-    this.frameSrc = frameSrc;
-    this._frameObjectUrl = options?.objectUrl || "";
+    if (bitmap && this.paintFrameBitmap(bitmap)) {
+      this.clearFrameSrc();
+      options?.cleanup?.();
+    } else {
+      this.clearFrameCanvas();
+      this.releaseRenderedFrameUrl(frameSrc);
+      this.frameSrc = frameSrc;
+      this._frameObjectUrl = options?.objectUrl || "";
+    }
+    bitmap?.close?.();
     this._lastFrameDimensions = dimensions;
     this._lastFrameAt = Date.now();
     options?.onAccepted?.();
@@ -1259,7 +1299,7 @@ const model = {
     return Boolean(
       dimensions?.width
       && dimensions?.height
-      && (!this.frameSrc || this._surfaceSwitching || this.isSwitchingBrowser())
+      && (!this.hasFrame() || this._surfaceSwitching || this.isSwitchingBrowser())
     );
   },
 
@@ -1330,7 +1370,7 @@ const model = {
 
   clearRenderedFrameIfViewportChanged() {
     const viewport = this.currentViewportSize();
-    if (!this.frameSrc || !this._lastFrameDimensions || !viewport) return;
+    if (!this.hasFrame() || !this._lastFrameDimensions || !viewport) return;
     if (this.frameMatchesViewport(this._lastFrameDimensions, viewport)) return;
     this.cancelFrameRender();
     this.resetViewportTracking();
@@ -1368,6 +1408,55 @@ const model = {
     this.frameSrc = "";
   },
 
+  attachFrameCanvas(canvas = null) {
+    this._frameCanvas = canvas || null;
+  },
+
+  canUseCanvasFrames() {
+    return Boolean(BROWSER_CANVAS_FRAMES_SUPPORTED && this._frameCanvas?.getContext);
+  },
+
+  hasFrame() {
+    return Boolean(this.frameSrc || this.frameCanvasReady);
+  },
+
+  paintFrameBitmap(bitmap) {
+    const canvas = this._frameCanvas;
+    if (!canvas || !bitmap?.width || !bitmap?.height) return false;
+    canvas.width = bitmap.width;
+    canvas.height = bitmap.height;
+    const context = canvas.getContext("2d");
+    if (!context) return false;
+    context.drawImage(bitmap, 0, 0);
+    this.frameCanvasReady = true;
+    return true;
+  },
+
+  clearFrameCanvas() {
+    const canvas = this._frameCanvas;
+    if (canvas?.width && canvas?.height) {
+      canvas.getContext("2d")?.clearRect(0, 0, canvas.width, canvas.height);
+    }
+    this.frameCanvasReady = false;
+  },
+
+  freezeCanvasFrameToImage() {
+    if (!this.frameCanvasReady || !this._frameCanvas) return;
+    try {
+      this.frameSrc = this._frameCanvas.toDataURL("image/jpeg", 0.86);
+    } catch {
+      this.frameSrc = "";
+    }
+    this.clearFrameCanvas();
+  },
+
+  frameElement() {
+    if (this.frameCanvasReady && this._frameCanvas) {
+      return this._frameCanvas;
+    }
+    return this._stageElement?.querySelector?.(".browser-frame-image") || null;
+  },
+
   beginCommand() {
     this._commandInFlightCount += 1;
     this.commandInFlight = true;
@@ -1431,6 +1520,7 @@ const model = {
       if (!this.activeBrowserId) {
         this.frameState = null;
         this.clearFrameSrc();
+        this.clearFrameCanvas();
       }
       if (result.state?.currentUrl || result.currentUrl) {
         this.address = result.state?.currentUrl || result.currentUrl;
@@ -1540,6 +1630,7 @@ const model = {
     this.switchingBrowserId = targetId;
     this.cancelFrameRender();
     this.clearFrameSrc();
+    this.clearFrameCanvas();
     this.frameState = browser || null;
     if (!this.addressFocused && browser?.currentUrl) {
       this.address = browser.currentUrl;
@@ -1811,8 +1902,8 @@ const model = {
     const target = element || event?.currentTarget;
     if (!target) return null;
     const rect = target.getBoundingClientRect();
-    const naturalWidth = target.naturalWidth || rect.width;
-    const naturalHeight = target.naturalHeight || rect.height;
+    const naturalWidth = target.naturalWidth || target.width || rect.width;
+    const naturalHeight = target.naturalHeight || target.height || rect.height;
     let contentLeft = rect.left;
     let contentTop = rect.top;
     let contentWidth = rect.width;
@@ -1963,7 +2054,7 @@ const model = {
   },
 
   canAnnotate() {
-    return Boolean(this.activeBrowserId && this.frameSrc && !this.isBusy());
+    return Boolean(this.activeBrowserId && this.hasFrame() && !this.isBusy());
   },
 
   activeAnnotationUrl() {
@@ -2124,8 +2215,7 @@ const model = {
   },
 
   stagePointForEvent(event) {
-    const image = this._stageElement?.querySelector?.(".browser-frame") || null;
-    return this.pointerCoordinatesFor(event, image);
+    return this.pointerCoordinatesFor(event, this.frameElement());
   },
 
   normalizeAnnotationRect(start = {}, end = {}) {
@@ -2538,8 +2628,7 @@ const model = {
   async sendWheel(event) {
     const contextId = this.normalizeContextId(this.activeBrowserContextId || this.contextId);
     if (!contextId || !this.activeBrowserId || !event) return;
-    const image = event.currentTarget?.querySelector?.(".browser-frame") || event.target?.closest?.(".browser-frame");
-    const pointer = this.pointerCoordinatesFor(event, image);
+    const pointer = this.pointerCoordinatesFor(event, this.frameElement());
     if (!pointer) return;
     const payload = {
       context_id: contextId,