From a767824fb63e4117a26802dda1f34e9dbe34c108 Mon Sep 17 00:00:00 2001 From: Alessandro <155005371+3clyp50@users.noreply.github.com> Date: Thu, 13 Aug 2026 01:48:41 +0200 Subject: [PATCH 01/20] Fix Responses tool call completion Recover function calls emitted during streaming when a terminal Responses event omits them. Advertise the final response tool with a strict text-only schema across agent profiles. --- helpers/litellm_transport.py | 8 ++++++- helpers/litellm_transport.py.dox.md | 1 + helpers/responses_tools.py | 33 ++++++++++++++++++---------- helpers/responses_tools.py.dox.md | 1 + tests/test_responses_architecture.py | 15 +++++-------- tests/test_responses_tools.py | 32 ++++++++++++++++++++++----- 6 files changed, 62 insertions(+), 28 deletions(-) diff --git a/helpers/litellm_transport.py b/helpers/litellm_transport.py index f84a12285..6a1498b71 100644 --- a/helpers/litellm_transport.py +++ b/helpers/litellm_transport.py @@ -428,7 +428,13 @@ class LiteLLMTransport: ) -> LLMResult | None: if parser.completed_response is None: return None - return self._llm_result_from_response(parser.completed_response, request) + response = _object_to_dict(parser.completed_response) + output = _as_list(response.get("output")) + if parser.function_calls and not any( + _get_value(item, "type") == "function_call" for item in output + ): + response["output"] = [*output, *parser.function_calls.values()] + return self._llm_result_from_response(response, request) def _stream_result_from_chat_parser( self, parser: "ChatCompletionsStreamParser" diff --git a/helpers/litellm_transport.py.dox.md b/helpers/litellm_transport.py.dox.md index 710765642..1a8e20795 100644 --- a/helpers/litellm_transport.py.dox.md +++ b/helpers/litellm_transport.py.dox.md @@ -33,6 +33,7 @@ - Fall back to Chat Completions when a Responses endpoint fails before output with an endpoint-specific server error, proxy path-unavailable error, or LiteLLM proxy-extra import error. - Fall back to Chat Completions when LiteLLM's Responses mock streaming path tries to JSON-decode a real SSE stream before any output. - Preserve Chat Completions tool calls from both non-streaming responses and streaming deltas as canonical `LLMResult` function-call items. +- Preserve Responses function calls collected from stream events when a terminal completed event omits them. - Preserve provider-state metadata when Responses API calls succeed, and fall back to local replay when provider state is unsupported. - Keep prompt-cache markers only for providers that accept them. diff --git a/helpers/responses_tools.py b/helpers/responses_tools.py index 93778d652..8bfffedca 100644 --- a/helpers/responses_tools.py +++ b/helpers/responses_tools.py @@ -39,20 +39,31 @@ def build_responses_function_tools(agent: Any) -> tuple[list[dict[str, Any]], di continue native_name = _native_tool_name(tool_name) name_map[native_name] = tool_name - tools.append( + parameters = ( { - "type": "function", - "name": native_name, - "description": _truncate( - tool_policy.tool_prompt_description( - prompt, - tool_name, - fallback=tool_name, - ) - ), - "parameters": _schema_from_prompt(prompt), + "type": "object", + "properties": {"text": {"type": "string"}}, + "required": ["text"], + "additionalProperties": False, } + if tool_name == "response" + else _schema_from_prompt(prompt) ) + tool = { + "type": "function", + "name": native_name, + "description": _truncate( + tool_policy.tool_prompt_description( + prompt, + tool_name, + fallback=tool_name, + ) + ), + "parameters": parameters, + } + if tool_name == "response": + tool["strict"] = True + tools.append(tool) for tool_name, tool in _mcp_tools(agent): if not tool_policy.resolve_tool( diff --git a/helpers/responses_tools.py.dox.md b/helpers/responses_tools.py.dox.md index f534eda5e..58a493b11 100644 --- a/helpers/responses_tools.py.dox.md +++ b/helpers/responses_tools.py.dox.md @@ -17,6 +17,7 @@ owns the Responses-specific prompt-name compatibility rules. - Local prompt-derived function names use existing bullet declarations that pair a backticked name with `arg` or `args` for multi-tool prompt files, otherwise prefer explicit `"tool_name"` examples, then the first prompt heading, and finally the prompt filename. - Apply registered tool-prompt render kwargs before deriving native metadata so descriptions never expose unresolved prompt templates. +- Always expose the native `response` tool as strict with one required `text` string, independent of profile prompt wording. - Use an explicitly embedded JSON input schema when present. Infer only an unambiguous single backticked argument on an otherwise empty `args:` line; all other local tools receive an honest permissive object schema instead of prose-guessed types. - Native local-tool descriptions reuse the tool catalog's compact prompt description; Responses retains native-name mapping, schema derivation, and diff --git a/tests/test_responses_architecture.py b/tests/test_responses_architecture.py index 1a195cb62..f344e9ebf 100644 --- a/tests/test_responses_architecture.py +++ b/tests/test_responses_architecture.py @@ -249,7 +249,9 @@ async def test_transport_downgrades_unsupported_builtin_tools(monkeypatch): @pytest.mark.asyncio -async def test_unified_turn_captures_response_id_without_stop_request(monkeypatch): +async def test_unified_turn_keeps_streamed_call_when_completion_omits_output( + monkeypatch, +): stream = _AsyncEventStream( [ { @@ -274,15 +276,7 @@ async def test_unified_turn_captures_response_id_without_stop_request(monkeypatc "type": "response.completed", "response": { "id": "resp_1", - "output": [ - { - "type": "function_call", - "id": "fc_1", - "call_id": "call_1", - "name": "lookup", - "arguments": '{"q":"a0"}', - } - ], + "output": [], }, }, ] @@ -315,6 +309,7 @@ async def test_unified_turn_captures_response_id_without_stop_request(monkeypatc assert stream.closed is False assert result.response_id == "resp_1" assert result.function_calls[0].call_id == "call_1" + assert result.function_calls[0].arguments == {"q": "a0"} @pytest.mark.asyncio diff --git a/tests/test_responses_tools.py b/tests/test_responses_tools.py index aaacaec6c..e1790b8da 100644 --- a/tests/test_responses_tools.py +++ b/tests/test_responses_tools.py @@ -168,20 +168,40 @@ def test_responses_function_tools_add_empty_properties_to_mcp_schemas( ] -def test_response_tool_native_contract_omits_wrapper_and_exposes_text(): - prompt = (PROJECT_ROOT / "prompts" / "agent.system.tool.response.md").read_text( - encoding="utf-8" - ) +def test_response_tool_native_contract_is_strict_and_requires_text(monkeypatch): + prompt_root = PROJECT_ROOT / "agents" / "agent0" / "prompts" + prompt = (prompt_root / "agent.system.tool.response.md").read_text(encoding="utf-8") description = tool_policy.tool_prompt_description( prompt, "response", fallback="response", ) - schema = responses_tools._schema_from_prompt(prompt) + monkeypatch.setattr( + responses_tools.subagents, + "get_paths", + lambda *args, **kwargs: [str(prompt_root)], + ) + monkeypatch.setattr( + responses_tools, + "_include_local_tool_prompt", + lambda agent, tool_name: True, + ) + monkeypatch.setattr(responses_tools, "_vision_tool_prompt", lambda agent: "") + monkeypatch.setattr(responses_tools, "_mcp_tools", lambda agent: []) + tools, _name_map = responses_tools.build_responses_function_tools( + FakeAgent(prompt_root) + ) + response_tool = next(tool for tool in tools if tool["name"] == "response") assert description == "final answer to user" - assert schema["properties"] == {"text": {"type": "string"}} + assert response_tool["parameters"] == { + "type": "object", + "properties": {"text": {"type": "string"}}, + "required": ["text"], + "additionalProperties": False, + } + assert response_tool["strict"] is True def test_complex_prompt_args_are_not_guessed_as_string_schemas(): From e2f43a3fb8df5811f2234c26337436fe5719e618 Mon Sep 17 00:00:00 2001 From: Alessandro <155005371+3clyp50@users.noreply.github.com> Date: Thu, 13 Aug 2026 00:38:01 +0200 Subject: [PATCH 02/20] Harden internal Browser against bot detection Run Chromium headful through Patchright on a private Xvfb display while keeping Agent Zero's page helpers in Patchright's isolated world. Reconcile the pinned Patchright package and matching architecture-specific Chromium from Browser plugin hooks so fresh Docker images and self-updated installations converge on the same runtime. --- docker/run/fs/ins/install_additional.sh | 1 + docker/run/fs/ins/install_playwright.sh | 7 +- docs/guides/browser.md | 2 +- docs/guides/troubleshooting.md | 4 +- docs/setup/dev-setup.md | 4 +- plugins/_browser/AGENTS.md | 3 + .../_20_browser_playwright_cache.py | 2 +- plugins/_browser/helpers/config.py | 7 +- plugins/_browser/helpers/playwright.py | 164 ++++++++++++-- plugins/_browser/helpers/runtime.py | 68 ++++-- plugins/_browser/hooks.py | 65 ++++++ requirements.txt | 2 +- tests/test_browser_agent_regressions.py | 213 ++++++++++++++++-- 13 files changed, 459 insertions(+), 83 deletions(-) diff --git a/docker/run/fs/ins/install_additional.sh b/docker/run/fs/ins/install_additional.sh index e9c844be7..257247d17 100644 --- a/docker/run/fs/ins/install_additional.sh +++ b/docker/run/fs/ins/install_additional.sh @@ -99,6 +99,7 @@ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ x11-xserver-utils \ xdotool \ xauth \ + xvfb \ dbus-x11 \ fonts-dejavu \ fonts-liberation \ diff --git a/docker/run/fs/ins/install_playwright.sh b/docker/run/fs/ins/install_playwright.sh index ca257004a..d5fc31e42 100644 --- a/docker/run/fs/ins/install_playwright.sh +++ b/docker/run/fs/ins/install_playwright.sh @@ -4,13 +4,10 @@ set -e # activate venv . "/ins/setup_venv.sh" "$@" -# install playwright if not installed (should be from requirements.txt) -uv pip install playwright - # set PW installation path to temporary Browser runtime storage export PLAYWRIGHT_BROWSERS_PATH=/a0/tmp/playwright mkdir -p "$PLAYWRIGHT_BROWSERS_PATH" -# install chromium with dependencies +# preinstall Chromium for fresh images; the Browser hook also reconciles self-updated installs apt-get install -y fonts-unifont libnss3 libnspr4 libatk1.0-0 libatspi2.0-0 libxcomposite1 libxdamage1 libatk-bridge2.0-0 libcups2 -playwright install chromium +patchright install chromium --no-shell diff --git a/docs/guides/browser.md b/docs/guides/browser.md index a005c252a..586384228 100644 --- a/docs/guides/browser.md +++ b/docs/guides/browser.md @@ -234,7 +234,7 @@ See [MCP Setup](mcp-setup.md) for MCP setup. ## Troubleshooting -- **Browser says Playwright is missing:** Docker installs already include the browser. In local development, let Agent Zero install it on first use or preinstall it with `PLAYWRIGHT_BROWSERS_PATH=tmp/playwright playwright install chromium`. +- **Browser says Chromium is missing:** Docker installs already include the browser. In local development, let Agent Zero install it on first use or preinstall it with `PLAYWRIGHT_BROWSERS_PATH=tmp/playwright patchright install chromium --no-shell`. - **The Browser surface does not open automatically:** That is expected. Open the Browser surface manually or ask the agent to show it. - **The Canvas does not follow the agent:** Enable **Autofocus active page** in Browser settings. - **Bring Your Own Browser cannot start:** Keep A0 CLI connected, verify Browser location is **Bring Your Own Browser**, and check `/browser status` in A0 CLI. diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index 21f8e979e..05a3cf02f 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -26,7 +26,7 @@ Refer to the [Choosing your LLMs](../setup/installation.md#installing-and-using- **7. How can I make Agent Zero retain memory between sessions?** Use **Settings -> Backup & Restore** and avoid mapping the entire `/a0` directory. See [How to update Agent Zero](../setup/installation.md#how-to-update-agent-zero). -**8. My browser tool fails or says Playwright is missing. What now?** +**8. My browser tool fails or says Chromium is missing. What now?** In normal Docker installs, the Browser already includes what it needs. @@ -35,7 +35,7 @@ browser the first time it is needed. To install it ahead of time, run this from the project root after installing Python requirements: ```bash -PLAYWRIGHT_BROWSERS_PATH=tmp/playwright playwright install chromium +PLAYWRIGHT_BROWSERS_PATH=tmp/playwright patchright install chromium --no-shell ``` If **Bring Your Own Browser** mode fails: diff --git a/docs/setup/dev-setup.md b/docs/setup/dev-setup.md index 5fd35fa17..d1390d0de 100644 --- a/docs/setup/dev-setup.md +++ b/docs/setup/dev-setup.md @@ -68,12 +68,12 @@ Now when you select one of the python files in the project, you should see prope ```bash pip install -r requirements.txt -PLAYWRIGHT_BROWSERS_PATH=./tmp/playwright playwright install chromium +PLAYWRIGHT_BROWSERS_PATH=./tmp/playwright patchright install chromium --no-shell ``` The first command installs Python dependencies. -The second command installs full Playwright Chromium into `./tmp/playwright`, +The second command installs full Patchright Chromium into `./tmp/playwright`, relative to the project root. Docker images use the absolute path `/a0/tmp/playwright` and ship Chromium preinstalled. diff --git a/plugins/_browser/AGENTS.md b/plugins/_browser/AGENTS.md index b790991e4..bfe481621 100644 --- a/plugins/_browser/AGENTS.md +++ b/plugins/_browser/AGENTS.md @@ -32,6 +32,9 @@ - Do not hardcode user-specific browser paths or secrets. - Browser model-preset selection resolves omitted preset fields from `_model_config`'s global `Default` preset, not from an unrelated currently scoped model selection. - Internal-browser proxy settings map directly to Playwright's persistent-context proxy option, never to Bring Your Own Browser, and changes must restart active internal runtimes. +- Run internal Chromium headful through Patchright on the private virtual display; do not add user-agent or header spoofing on top of the patched driver. +- Browser startup and on-demand launch must converge on the Chromium revision declared by Patchright; let its installer select the host architecture rather than hardcoding x64 or ARM downloads. +- `hooks.prepare_playwright_cache()` owns reconciliation of the pinned Patchright package and Chromium binary so repository self-updates and fresh images use the same setup path. ## Work Guidance diff --git a/plugins/_browser/extensions/python/startup_migration/_20_browser_playwright_cache.py b/plugins/_browser/extensions/python/startup_migration/_20_browser_playwright_cache.py index c30aa4b57..e76a19856 100644 --- a/plugins/_browser/extensions/python/startup_migration/_20_browser_playwright_cache.py +++ b/plugins/_browser/extensions/python/startup_migration/_20_browser_playwright_cache.py @@ -33,7 +33,7 @@ def _start_background_cache_migration() -> threading.Thread: def _migrate_cache_safely() -> None: try: - _log_cache_migration_result(hooks.cleanup_playwright_cache()) + _log_cache_migration_result(hooks.prepare_playwright_cache()) except Exception as exc: PrintStyle.warning("Browser Playwright cache migration failed:", exc) diff --git a/plugins/_browser/helpers/config.py b/plugins/_browser/helpers/config.py index e412b652c..589eeeb31 100644 --- a/plugins/_browser/helpers/config.py +++ b/plugins/_browser/helpers/config.py @@ -30,11 +30,6 @@ DEFAULT_MAX_OPEN_TABS = 32 MIN_MAX_OPEN_TABS = 1 HARD_MAX_OPEN_TABS = 50 DEFAULT_HOST_BROWSER_PRIVACY_POLICY = "allow" -BASE_BROWSER_ARGS = [ - "--no-sandbox", - "--disable-dev-shm-usage", - "--disable-gpu", -] def _normalize_extension_paths(value: Any) -> list[str]: @@ -388,7 +383,7 @@ def describe_browser_extensions(settings: dict[str, Any] | None) -> dict[str, An def build_browser_launch_config(settings: dict[str, Any] | None) -> dict[str, Any]: config = normalize_browser_config(settings) extensions = describe_browser_extensions(config) - args = list(BASE_BROWSER_ARGS) + args: list[str] = [] channel: str | None = None browser_mode = "chromium" proxy = None diff --git a/plugins/_browser/helpers/playwright.py b/plugins/_browser/helpers/playwright.py index f1c91014f..f0463e4fa 100644 --- a/plugins/_browser/helpers/playwright.py +++ b/plugins/_browser/helpers/playwright.py @@ -1,12 +1,20 @@ +import atexit +import json import os +import re +import select +import shutil import subprocess +import sys +import threading +from importlib import resources from pathlib import Path from helpers import files FULL_CHROMIUM_PATTERNS = ( - "chromium-*/chrome-linux/chrome", - "chromium-*/chrome-win/chrome.exe", + "chromium-*/chrome-linux*/chrome", + "chromium-*/chrome-win*/chrome.exe", ) PLAYWRIGHT_CACHE_ENV = "A0_BROWSER_PLAYWRIGHT_CACHE_DIR" PLAYWRIGHT_CACHE_DIR = ("tmp", "playwright") @@ -14,6 +22,10 @@ RETIRED_PLAYWRIGHT_CACHE_DIRS = ( ("usr", "plugins", "_browser", "playwright"), ("usr", "browser", "playwright"), ) +_INSTALL_LOCK = threading.Lock() +_DISPLAY_LOCK = threading.Lock() +_DISPLAY_PROCESS: subprocess.Popen | None = None +_DISPLAY_NAME = "" def _primary_cache_dir() -> Path: @@ -52,33 +64,139 @@ def configure_playwright_env() -> str: return cache_dir -def find_playwright_binary(cache_dir: Path) -> Path | None: - for pattern in FULL_CHROMIUM_PATTERNS: - binary = next(cache_dir.glob(pattern), None) - if binary and binary.exists(): - return binary - return None +def ensure_browser_display() -> str: + global _DISPLAY_NAME, _DISPLAY_PROCESS + + with _DISPLAY_LOCK: + if _DISPLAY_PROCESS and _DISPLAY_PROCESS.poll() is None: + return _DISPLAY_NAME + + xvfb = shutil.which("Xvfb") + if not xvfb: + return "" + + read_fd, write_fd = os.pipe() + try: + process = subprocess.Popen( + [ + xvfb, + "-displayfd", + str(write_fd), + "-screen", + "0", + "1365x768x24", + "+extension", + "GLX", + "-nolisten", + "tcp", + "-noreset", + "-ac", + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + pass_fds=(write_fd,), + ) + except OSError: + os.close(read_fd) + return "" + finally: + os.close(write_fd) + + try: + ready, _, _ = select.select([read_fd], [], [], 5) + display_number = os.read(read_fd, 32).decode().strip() if ready else "" + finally: + os.close(read_fd) + + if not display_number.isdigit() or process.poll() is not None: + _terminate_browser_display(process) + return "" + + _DISPLAY_PROCESS = process + _DISPLAY_NAME = f":{display_number}" + return _DISPLAY_NAME + + +def close_browser_display() -> None: + global _DISPLAY_NAME, _DISPLAY_PROCESS + + with _DISPLAY_LOCK: + process = _DISPLAY_PROCESS + _DISPLAY_PROCESS = None + _DISPLAY_NAME = "" + if process: + _terminate_browser_display(process) + + +def _terminate_browser_display(process: subprocess.Popen) -> None: + if process.poll() is not None: + return + process.terminate() + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=2) + + +def find_playwright_binary(cache_dir: Path, revision: str = "") -> Path | None: + prefix = f"chromium-{revision}" if revision.isdigit() else "chromium-*" + binaries = [ + binary + for pattern in FULL_CHROMIUM_PATTERNS + for binary in cache_dir.glob(pattern.replace("chromium-*", prefix)) + if binary.exists() + ] + return max(binaries, key=_chromium_revision) if binaries else None + + +def _chromium_revision(binary: Path) -> int: + match = re.search(r"chromium-(\d+)", binary.as_posix()) + return int(match.group(1)) if match else -1 def get_playwright_binary() -> Path | None: - return find_playwright_binary(_primary_cache_dir()) + cache_dir = _primary_cache_dir() + binary = find_playwright_binary(_primary_cache_dir()) + revision = get_playwright_chromium_revision() + if revision and (not binary or _chromium_revision(binary) != int(revision)): + return find_playwright_binary(cache_dir, revision=revision) + return binary + + +def get_playwright_chromium_revision() -> str: + try: + manifest = resources.files("patchright").joinpath("driver/package/browsers.json") + browsers = json.loads(manifest.read_text(encoding="utf-8"))["browsers"] + revision = next( + str(browser.get("revision", "")) + for browser in browsers + if browser.get("name") == "chromium" + ) + except (ImportError, FileNotFoundError, KeyError, StopIteration, TypeError, ValueError): + return "" + return revision if revision.isdigit() else "" def ensure_playwright_binary() -> Path: - binary = get_playwright_binary() - if binary: + with _INSTALL_LOCK: + binary = get_playwright_binary() + if binary: + return binary + + cache_dir = configure_playwright_env() + env = os.environ.copy() + env["PLAYWRIGHT_BROWSERS_PATH"] = cache_dir + subprocess.check_call( + [sys.executable, "-m", "patchright", "install", "chromium", "--no-shell"], + env=env, + ) + + binary = get_playwright_binary() + if not binary: + raise RuntimeError("Patchright Chromium binary not found after installation") return binary - cache_dir = configure_playwright_env() - env = os.environ.copy() - env["PLAYWRIGHT_BROWSERS_PATH"] = cache_dir - install_command = ["playwright", "install", "chromium"] - subprocess.check_call( - install_command, - env=env, - ) - binary = get_playwright_binary() - if not binary: - raise RuntimeError("Playwright Chromium binary not found after installation") - return binary +atexit.register(close_browser_display) diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py index f5ac52866..040503cba 100644 --- a/plugins/_browser/helpers/runtime.py +++ b/plugins/_browser/helpers/runtime.py @@ -27,7 +27,9 @@ from plugins._browser.helpers.config import ( build_browser_launch_config, get_browser_config, ) -from plugins._browser.helpers.playwright import configure_playwright_env, ensure_playwright_binary +from plugins._browser.helpers.playwright import ( + ensure_browser_display, +) from plugins._browser.helpers.url import normalize_url @@ -591,6 +593,7 @@ class _BrowserRuntimeCore: self._closing = False self._pending_popups: list[asyncio.Future[int]] = [] self._background_popup_pages: set[int] = set() + self._bootstrap_page: Any | None = None def _ensure_registry_lock(self) -> asyncio.Lock: if self._registry_lock is None: @@ -773,6 +776,7 @@ class _BrowserRuntimeCore: waiter.set_exception(RuntimeError("Browser context closed.")) self._pending_popups.clear() self._background_popup_pages.clear() + self._bootstrap_page = None self.pages.clear() self.last_interacted_browser_id = None for screencast in self.screencasts.values(): @@ -797,20 +801,26 @@ class _BrowserRuntimeCore: self.playwright = None async def _start(self) -> None: - from playwright.async_api import async_playwright + from plugins._browser import hooks + + preparation = hooks.prepare_playwright_cache() + if preparation.get("errors") or not preparation.get("binary"): + problem = preparation.get("errors") or "missing binary" + raise RuntimeError(f"Browser setup failed: {problem}") + from patchright.async_api import async_playwright self.profile_dir.mkdir(parents=True, exist_ok=True) self.downloads_dir.mkdir(parents=True, exist_ok=True) self._release_orphaned_profile_singleton() browser_config = get_browser_config() launch_config = build_browser_launch_config(browser_config) - configure_playwright_env() - browser_binary = ensure_playwright_binary() + browser_binary = Path(preparation["binary"]) + browser_display = ensure_browser_display() self.playwright = await async_playwright().start() launch_kwargs: dict[str, Any] = { "user_data_dir": str(self.profile_dir), - "headless": True, + "headless": not bool(browser_display), "accept_downloads": True, "downloads_path": str(self.downloads_dir), "viewport": DEFAULT_VIEWPORT, @@ -818,6 +828,8 @@ class _BrowserRuntimeCore: "no_viewport": False, "args": launch_config["args"], } + if browser_display: + launch_kwargs["env"] = {**os.environ, "DISPLAY": browser_display} if launch_config["channel"]: launch_kwargs["channel"] = launch_config["channel"] else: @@ -840,11 +852,12 @@ class _BrowserRuntimeCore: self.context.set_default_navigation_timeout(30000) self.context.on("close", self._on_context_closed) self.context.on("page", self._on_new_page_sync) - await self.context.add_init_script(path=str(DOM_HELPER_PATH)) - await self.context.add_init_script(path=str(CONTENT_HELPER_PATH)) for page in list(self.context.pages): if page.url == "about:blank": + if browser_display and self._bootstrap_page is None: + self._bootstrap_page = page + continue try: await page.close() except Exception: @@ -915,7 +928,10 @@ class _BrowserRuntimeCore: async def open(self, url: str = "") -> dict[str, Any]: await self.ensure_started() self._ensure_can_open_page() - page = await self.context.new_page() + page = self._bootstrap_page + self._bootstrap_page = None + if not page or page.is_closed(): + page = await self.context.new_page() browser_page = await self._register_page(page) self.last_interacted_browser_id = browser_page.id target_url = self._initial_url(url) @@ -1296,6 +1312,7 @@ class _BrowserRuntimeCore: result = await page.evaluate( "(payload) => globalThis.__spaceBrowserPageContent__.capture(payload || null)", payload or None, + isolated_context=True, ) self._maybe_promote(resolved_id) return result or {} @@ -1308,6 +1325,7 @@ class _BrowserRuntimeCore: result = await page.evaluate( "(ref) => globalThis.__spaceBrowserPageContent__.detail(ref)", reference_id, + isolated_context=True, ) self._maybe_promote(resolved_id) return result or {} @@ -1324,6 +1342,7 @@ class _BrowserRuntimeCore: result = await page.evaluate( "(payload) => globalThis.__spaceBrowserPageContent__.annotate(payload || null)", payload or None, + isolated_context=True, ) self._maybe_promote(resolved_id) return result or {} @@ -1332,7 +1351,7 @@ class _BrowserRuntimeCore: await self.ensure_started() resolved_id = self._resolve_browser_id(browser_id) page = self._page(resolved_id) - result = await page.evaluate(str(script or "undefined")) + result = await page.evaluate(str(script or "undefined"), isolated_context=False) self._maybe_promote(resolved_id) return {"result": result, "state": await self._state(resolved_id)} @@ -1364,6 +1383,7 @@ class _BrowserRuntimeCore: box = await page.evaluate( "(ref) => globalThis.__spaceBrowserPageContent__.boundingBoxFor(ref)", reference_id, + isolated_context=True, ) background = focus_popup is False or ( @@ -1515,6 +1535,7 @@ class _BrowserRuntimeCore: "action": normalized_action, "text": str(text or ""), }, + isolated_context=False, ) or {} except Exception as exc: clipboard_result = { @@ -1779,6 +1800,7 @@ class _BrowserRuntimeCore: "useOffsets": bool(offset_x or offset_y), }, }, + isolated_context=True, ) if not point or not isinstance(point, dict): raise ValueError(f"Could not resolve Browser ref {reference_id!r} to a viewport point") @@ -1995,6 +2017,7 @@ class _BrowserRuntimeCore: "ref": ref, "values": values if values is not None else value, }, + isolated_context=True, ) await self._settle(page, short=True) self._maybe_promote(resolved_id) @@ -2016,6 +2039,7 @@ class _BrowserRuntimeCore: "ref": ref, "checked": bool(checked), }, + isolated_context=True, ) await self._settle(page, short=True) self._maybe_promote(resolved_id) @@ -2036,12 +2060,14 @@ class _BrowserRuntimeCore: metadata = await page.evaluate( "(ref) => globalThis.__spaceBrowserPageContent__.fileInputFor(ref)", ref, + isolated_context=True, ) handle = None try: handle = await page.evaluate_handle( "(ref) => globalThis.__spaceBrowserPageContent__.fileInputElementFor(ref)", ref, + isolated_context=True, ) element = handle.as_element() if handle else None if element: @@ -2198,11 +2224,13 @@ class _BrowserRuntimeCore: action = await page.evaluate( "(args) => globalThis.__spaceBrowserPageContent__[args.method](args.ref)", {"method": helper_method, "ref": reference_id}, + isolated_context=True, ) else: action = await page.evaluate( "(args) => globalThis.__spaceBrowserPageContent__[args.method](args.ref, args.text)", {"method": helper_method, "ref": reference_id, "text": text}, + isolated_context=True, ) await self._settle(page, short=False) self._maybe_promote(resolved_id) @@ -2215,8 +2243,8 @@ class _BrowserRuntimeCore: *, wait_until: str = "domcontentloaded", ) -> None: - from playwright.async_api import Error as PlaywrightError - from playwright.async_api import TimeoutError as PlaywrightTimeoutError + from patchright.async_api import Error as PlaywrightError + from patchright.async_api import TimeoutError as PlaywrightTimeoutError try: await page.goto(url, wait_until=wait_until, timeout=30000) @@ -2227,8 +2255,8 @@ class _BrowserRuntimeCore: await self._settle(page, short=wait_until == "commit") async def _settle(self, page: Any, short: bool = False) -> None: - from playwright.async_api import Error as PlaywrightError - from playwright.async_api import TimeoutError as PlaywrightTimeoutError + from patchright.async_api import Error as PlaywrightError + from patchright.async_api import TimeoutError as PlaywrightTimeoutError try: await page.wait_for_load_state( @@ -2249,7 +2277,10 @@ class _BrowserRuntimeCore: except Exception: title = "" try: - history_length = await page.evaluate("() => globalThis.history?.length || 0") + history_length = await page.evaluate( + "() => globalThis.history?.length || 0", + isolated_context=False, + ) except Exception: history_length = 0 return { @@ -2384,13 +2415,14 @@ class _BrowserRuntimeCore: async def _ensure_content_helper(self, page: Any) -> None: await self._ensure_dom_helper(page) has_helper = await page.evaluate( - "() => Boolean(globalThis.__spaceBrowserPageContent__?.ready?.())" + "() => Boolean(globalThis.__spaceBrowserPageContent__?.ready?.())", + isolated_context=True, ) if has_helper: return if self._content_helper_source is None: self._content_helper_source = CONTENT_HELPER_PATH.read_text(encoding="utf-8") - await page.evaluate(self._content_helper_source) + await page.evaluate(self._content_helper_source, isolated_context=True) async def _ensure_dom_helper(self, page: Any) -> None: if self._dom_helper_source is None: @@ -2408,13 +2440,13 @@ class _BrowserRuntimeCore: targets = frames for target in targets: try: - has_helper = await target.evaluate(ready_script) + has_helper = await target.evaluate(ready_script, isolated_context=True) except Exception: continue if has_helper: continue with contextlib.suppress(Exception): - await target.evaluate(source) + await target.evaluate(source, isolated_context=True) _runtimes: dict[str, BrowserRuntime] = {} _runtime_lock = threading.RLock() diff --git a/plugins/_browser/hooks.py b/plugins/_browser/hooks.py index 257917f2b..8ccd83981 100644 --- a/plugins/_browser/hooks.py +++ b/plugins/_browser/hooks.py @@ -1,6 +1,12 @@ from __future__ import annotations +import importlib +import importlib.metadata +import importlib.util import shutil +import subprocess +import sys +import threading from pathlib import Path from helpers import files, plugins, yaml as yaml_helper @@ -10,6 +16,7 @@ from plugins._browser.helpers.config import ( normalize_browser_config, ) from plugins._browser.helpers.playwright import ( + ensure_playwright_binary, find_playwright_binary, get_playwright_cache_dir, get_retired_playwright_cache_dirs, @@ -17,6 +24,11 @@ from plugins._browser.helpers.playwright import ( from plugins._browser.helpers.runtime import close_all_runtimes_sync +_SETUP_LOCK = threading.Lock() +_PLUGIN_DIR = Path(__file__).resolve().parent +_ROOT_REQUIREMENTS_FILE = _PLUGIN_DIR.parents[1] / "requirements.txt" + + def _load_saved_browser_config(project_name: str = "", agent_profile: str = "") -> dict: entries = plugins.find_plugin_assets( plugins.CONFIG_FILE_NAME, @@ -95,6 +107,59 @@ def cleanup_playwright_cache() -> dict: return result +def prepare_playwright_cache() -> dict: + with _SETUP_LOCK: + _ensure_patchright_dependency() + result = cleanup_playwright_cache() + if result["errors"]: + return result + result["binary"] = str(ensure_playwright_binary()) + return result + + +def install() -> dict: + return prepare_playwright_cache() + + +def _ensure_patchright_dependency() -> None: + requirement = _patchright_requirement() + if _patchright_is_current(requirement): + return + + uv = shutil.which("uv") + if not uv: + raise RuntimeError("Browser plugin requires 'uv' to install Patchright automatically") + + subprocess.check_call( + [uv, "pip", "install", "--python", sys.executable, requirement], + cwd=str(_PLUGIN_DIR), + ) + importlib.invalidate_caches() + if not _patchright_is_current(requirement): + raise RuntimeError( + f"Browser dependency {requirement!r} is unavailable after installation" + ) + + +def _patchright_requirement() -> str: + if _ROOT_REQUIREMENTS_FILE.is_file(): + for line in _ROOT_REQUIREMENTS_FILE.read_text(encoding="utf-8").splitlines(): + requirement = line.strip() + if requirement.startswith("patchright=="): + return requirement + raise RuntimeError(f"Browser Patchright requirement not found in {_ROOT_REQUIREMENTS_FILE}") + + +def _patchright_is_current(requirement: str) -> bool: + expected_version = requirement.partition("==")[2] + if not expected_version or importlib.util.find_spec("patchright") is None: + return False + try: + return importlib.metadata.version("patchright") == expected_version + except importlib.metadata.PackageNotFoundError: + return False + + def _best_playwright_cache(candidates: list[Path]) -> Path | None: valid = [path for path in candidates if path.is_dir() and find_playwright_binary(path)] if not valid: diff --git a/requirements.txt b/requirements.txt index 5c4381828..fb90e1e44 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,7 @@ markdown==3.7 mcp==1.27.0 newspaper3k==0.2.8 paramiko==3.5.0 -playwright==1.52.0 +patchright==1.61.2 pypdf==6.0.0 python-dotenv==1.1.0 pytz==2024.2 diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py index eaa49ef4a..2c546b421 100644 --- a/tests/test_browser_agent_regressions.py +++ b/tests/test_browser_agent_regressions.py @@ -110,6 +110,7 @@ from plugins._browser.helpers.runtime import ( ) import plugins._browser.helpers.runtime as browser_runtime_module from plugins._browser.helpers.playwright import ( + ensure_playwright_binary, get_playwright_binary, get_playwright_cache_dir, ) @@ -331,6 +332,9 @@ def test_browser_launch_config_uses_full_chromium_for_all_sessions(tmp_path): assert default_launch["requires_full_browser"] is True assert default_launch["proxy"] is None assert not any(arg.startswith("--load-extension=") for arg in default_launch["args"]) + assert "--no-sandbox" not in default_launch["args"] + assert "--disable-dev-shm-usage" not in default_launch["args"] + assert "--disable-gpu" not in default_launch["args"] assert "--headless=new" not in default_launch["args"] extension_dir = tmp_path / "extension" @@ -357,6 +361,102 @@ def _patch_playwright_cache_root(monkeypatch, tmp_path): "get_abs_path", lambda *parts: str(tmp_path.joinpath(*parts)), ) + monkeypatch.setattr( + browser_playwright_module, + "get_playwright_chromium_revision", + lambda: "1169", + ) + + +def test_browser_uses_patchright_revision_when_newer_playwright_cache_exists( + monkeypatch, tmp_path +): + _patch_playwright_cache_root(monkeypatch, tmp_path) + monkeypatch.setattr( + browser_playwright_module, + "get_playwright_chromium_revision", + lambda: "1228", + ) + cache_dir = Path(get_playwright_cache_dir()) + expected = cache_dir / "chromium-1228" / "chrome-linux64" / "chrome" + other = cache_dir / "chromium-1234" / "chrome-linux64" / "chrome" + expected.parent.mkdir(parents=True) + other.parent.mkdir(parents=True) + expected.touch() + other.touch() + + assert get_playwright_binary() == expected + + +@pytest.mark.parametrize("platform_dir", ["chrome-linux", "chrome-linux64"]) +def test_browser_installs_current_patchright_chromium_for_host_architecture( + monkeypatch, tmp_path, platform_dir +): + _patch_playwright_cache_root(monkeypatch, tmp_path) + monkeypatch.setattr( + browser_playwright_module, + "get_playwright_chromium_revision", + lambda: "1234", + ) + old_binary = ( + tmp_path / "tmp" / "playwright" / "chromium-1169" / "chrome-linux" / "chrome" + ) + old_binary.parent.mkdir(parents=True) + old_binary.touch() + expected = ( + tmp_path / "tmp" / "playwright" / "chromium-1234" / platform_dir / "chrome" + ) + + def install(command, *, env): + assert command == [ + sys.executable, + "-m", + "patchright", + "install", + "chromium", + "--no-shell", + ] + assert env["PLAYWRIGHT_BROWSERS_PATH"] == str(tmp_path / "tmp" / "playwright") + expected.parent.mkdir(parents=True) + expected.touch() + + monkeypatch.setattr(browser_playwright_module.subprocess, "check_call", install) + + assert ensure_playwright_binary() == expected + + +def test_browser_hook_installs_patchright_for_existing_self_updated_runtime(monkeypatch): + current = False + + def is_current(requirement): + assert requirement == "patchright==1.61.2" + return current + + def install(command, *, cwd): + nonlocal current + assert command == [ + "/usr/local/bin/uv", + "pip", + "install", + "--python", + sys.executable, + "patchright==1.61.2", + ] + assert cwd == str(PROJECT_ROOT / "plugins" / "_browser") + current = True + + monkeypatch.setattr( + browser_hooks_module, + "_patchright_requirement", + lambda: "patchright==1.61.2", + ) + monkeypatch.setattr(browser_hooks_module, "_patchright_is_current", is_current) + monkeypatch.setattr(browser_hooks_module.shutil, "which", lambda name: "/usr/local/bin/uv") + monkeypatch.setattr(browser_hooks_module.subprocess, "check_call", install) + + browser_hooks_module._ensure_patchright_dependency() + + assert current is True def _write_playwright_binary(cache_dir: Path) -> Path: @@ -1705,23 +1805,18 @@ def test_browser_runtime_requires_current_content_helper_for_modifier_clicks(): ).read_text(encoding="utf-8") assert "__spaceBrowserPageContent__?.ready?.()" in runtime + assert "context.add_init_script" not in runtime + assert "isolated_context=False" in runtime @pytest.mark.anyio async def test_browser_dom_helper_clicks_content_ref_inside_iframe(): - pytest.importorskip("playwright.async_api") - from playwright.async_api import async_playwright + pytest.importorskip("patchright.async_api") + from patchright.async_api import async_playwright browser_binary = get_playwright_binary() if not browser_binary: - pytest.skip("Playwright Chromium binary is not installed") - - dom_helper = ( - PROJECT_ROOT / "plugins" / "_browser" / "assets" / "browser-dom-helper.js" - ).read_text(encoding="utf-8") - content_helper = ( - PROJECT_ROOT / "plugins" / "_browser" / "assets" / "browser-page-content.js" - ).read_text(encoding="utf-8") + pytest.skip("Patchright Chromium binary is not installed") async with async_playwright() as playwright: try: @@ -1731,12 +1826,10 @@ async def test_browser_dom_helper_clicks_content_ref_inside_iframe(): args=["--no-sandbox"], ) except Exception as exc: - pytest.skip(f"Playwright Chromium could not launch: {exc}") + pytest.skip(f"Patchright Chromium could not launch: {exc}") try: context = await browser.new_context() - await context.add_init_script(dom_helper) - await context.add_init_script(content_helper) page = await context.new_page() await page.set_content( """ @@ -1755,9 +1848,8 @@ async def test_browser_dom_helper_clicks_content_ref_inside_iframe(): """ ) - await page.wait_for_function( - "() => Boolean(document.querySelector('iframe')?.contentWindow?.__spaceBrowserDomHelper__)" - ) + core = _BrowserRuntimeCore("patchright-helper") + await core._ensure_content_helper(page) captured = await page.evaluate( "(payload) => globalThis.__spaceBrowserPageContent__.capture(payload || null)", @@ -2093,13 +2185,32 @@ def test_browser_docker_installs_full_chromium_to_tmp_cache(): script = ( PROJECT_ROOT / "docker" / "run" / "fs" / "ins" / "install_playwright.sh" ).read_text(encoding="utf-8") + requirements = (PROJECT_ROOT / "requirements.txt").read_text(encoding="utf-8") assert "PLAYWRIGHT_BROWSERS_PATH=/a0/tmp/playwright" in script - assert "playwright install chromium" in script + assert "patchright install chromium --no-shell" in script + assert "playwright install chromium" not in script + assert "uv pip install" not in script + assert "patchright==1.61.2" in requirements assert "--only-shell" not in script + runtime = (PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py").read_text( + encoding="utf-8" + ) + assert "from patchright.async_api import async_playwright" in runtime + assert "from playwright.async_api import async_playwright" not in runtime + assert runtime.index("hooks.prepare_playwright_cache()") < runtime.index( + "from patchright.async_api import async_playwright" + ) + install_additional = ( + PROJECT_ROOT / "docker" / "run" / "fs" / "ins" / "install_additional.sh" + ).read_text(encoding="utf-8") + assert '"headless": not bool(browser_display)' in runtime + assert 'launch_kwargs["env"] = {**os.environ, "DISPLAY": browser_display}' in runtime + assert " xvfb \\" in install_additional -def test_browser_startup_migration_runs_playwright_cache_cleanup(): + +def test_browser_startup_migration_prepares_current_playwright_binary(): extension = ( PROJECT_ROOT / "plugins" @@ -2111,7 +2222,7 @@ def test_browser_startup_migration_runs_playwright_cache_cleanup(): ).read_text(encoding="utf-8") assert "class BrowserPlaywrightCacheMigration(Extension)" in extension - assert "hooks.cleanup_playwright_cache()" in extension + assert "hooks.prepare_playwright_cache()" in extension assert "PrintStyle.warning" in extension @@ -2135,6 +2246,55 @@ def test_browser_runtime_removes_stale_profile_singletons(monkeypatch, tmp_path) ) +@pytest.mark.anyio +async def test_browser_first_open_reuses_headful_bootstrap_page(monkeypatch): + class BootstrapPage: + url = "about:blank" + + @staticmethod + def is_closed(): + return False + + page = BootstrapPage() + + class Context: + pages = [page] + + @staticmethod + async def new_page(): + raise AssertionError("The first headful tab must reuse Chrome's bootstrap page") + + core = _BrowserRuntimeCore("headful") + core.context = Context() + core._bootstrap_page = page + + async def register(registered_page): + assert registered_page is page + browser_page = BrowserPage(id=1, page=registered_page) + core.pages[1] = browser_page + return browser_page + + async def settle(_page, short=False): + return None + + async def state(browser_id): + return {"id": browser_id, "currentUrl": "about:blank"} + + core._register_page = register + core._settle = settle + core._state = state + monkeypatch.setattr( + browser_runtime_module, + "get_browser_config", + lambda: {"default_homepage": "about:blank", "max_open_tabs": 8}, + ) + + result = await core.open() + + assert result == {"id": 1, "state": {"id": 1, "currentUrl": "about:blank"}} + assert core._bootstrap_page is None + + @pytest.mark.anyio async def test_browser_runtime_restarts_when_cached_context_is_stale(): starts = [] @@ -2780,7 +2940,12 @@ async def test_browser_viewer_subscribe_returns_initial_snapshot(monkeypatch): result = await handler.process( "browser_viewer_subscribe", - {"context_id": "ctx", "browser_id": 1, "viewport_width": 900, "viewport_height": 600}, + { + "context_id": "ctx", + "browser_id": 1, + "viewport_width": 900, + "viewport_height": 600, + }, "sid-snapshot", ) @@ -3202,7 +3367,7 @@ async def test_browser_runtime_screenshot_file_defaults_to_chat_scoped_artifact( async def title(self): return "Blank" - async def evaluate(self, script, payload=None): + async def evaluate(self, script, payload=None, **kwargs): return 1 core = _BrowserRuntimeCore("ctx/id") @@ -3321,7 +3486,7 @@ async def test_browser_runtime_ref_point_resolution_applies_offsets(): def __init__(self): self.mouse = FakeMouse() - async def evaluate(self, script, payload=None): + async def evaluate(self, script, payload=None, **kwargs): eval_payloads.append((script, payload)) if payload and "offsets" in payload: return { @@ -3392,7 +3557,7 @@ async def test_browser_runtime_clipboard_paste_uses_dom_bridge(): def __init__(self): self.keyboard = FakeKeyboard() - async def evaluate(self, script, payload=None): + async def evaluate(self, script, payload=None, **kwargs): if payload is not None: eval_payloads.append((script, payload)) return { @@ -3442,7 +3607,7 @@ async def test_browser_runtime_clipboard_paste_falls_back_to_keyboard_insert_tex def __init__(self): self.keyboard = FakeKeyboard() - async def evaluate(self, script, payload=None): + async def evaluate(self, script, payload=None, **kwargs): if payload is not None: return { "action": "paste", From 005b366b5147f35c5fe66c44c864f74e1ec4633d Mon Sep 17 00:00:00 2001 From: Alessandro <155005371+3clyp50@users.noreply.github.com> Date: Sun, 16 Aug 2026 12:58:17 +0200 Subject: [PATCH 03/20] Add interactive internal Browser viewport Render the existing Patchright page through an isolated, authenticated Xpra session with CDP screencast and snapshot fallbacks. Keep Chromium out of fullscreen, synchronize native viewport resizing across canvas and modal handoffs, and remove Xpra decoration and shadow-cursor artifacts. Move loading feedback into each Browser tab while reserving the footer for actual errors. --- helpers/virtual_desktop.py | 55 +-- helpers/virtual_desktop.py.dox.md | 6 +- plugins/_browser/AGENTS.md | 18 +- plugins/_browser/api/status.py | 2 + plugins/_browser/api/ws_browser.py | 91 ++++- .../_20_browser_playwright_cache.py | 2 + plugins/_browser/helpers/interactive_view.py | 293 ++++++++++++++ plugins/_browser/helpers/playwright.py | 85 ----- plugins/_browser/helpers/runtime.py | 184 ++++++++- plugins/_browser/webui/browser-panel.html | 94 +++-- plugins/_browser/webui/browser-store.js | 187 +++++++-- tests/test_browser_agent_regressions.py | 357 +++++++++++++++++- 12 files changed, 1163 insertions(+), 211 deletions(-) create mode 100644 plugins/_browser/helpers/interactive_view.py diff --git a/helpers/virtual_desktop.py b/helpers/virtual_desktop.py index d417cf4ad..8de9ded15 100644 --- a/helpers/virtual_desktop.py +++ b/helpers/virtual_desktop.py @@ -115,29 +115,38 @@ def get_registry() -> VirtualDesktopRegistry: return _registry -def session_url(token: str, *, title: str = "Desktop") -> str: +def session_url( + token: str, + *, + title: str = "Desktop", + encoding: str = "jpeg", + quality: int = 85, + speed: int = 80, + file_transfer: bool = True, + printing: bool = True, +) -> str: quoted_token = quote(str(token), safe="") base_path = f"{SESSION_PATH}/{quoted_token}/" - query = urlencode( - { - "path": base_path, - "title": title, - "encoding": "jpeg", - "quality": "85", - "speed": "80", - "sharing": "true", - "clipboard": "true", - "clipboard_direction": "both", - "clipboard_poll": "true", - "clipboard_preferred_format": "text/plain", - "printing": "true", - "file_transfer": "true", - "sound": "false", - "offscreen": "true", - "floating_menu": "false", - "xpramenu": "false", - }, - ) + options = { + "path": base_path, + "title": title, + "quality": str(max(0, min(100, int(quality)))), + "speed": str(max(0, min(100, int(speed)))), + "sharing": "true", + "clipboard": "true", + "clipboard_direction": "both", + "clipboard_poll": "true", + "clipboard_preferred_format": "text/plain", + "printing": str(bool(printing)).lower(), + "file_transfer": str(bool(file_transfer)).lower(), + "sound": "false", + "offscreen": "true", + "floating_menu": "false", + "xpramenu": "false", + } + if encoding: + options["encoding"] = str(encoding) + query = urlencode(options) return f"{base_path}index.html?{query}" @@ -264,6 +273,7 @@ def resize_display( keys: tuple[str, ...] = (), xauthority: str = "", home: str = "", + settle_seconds: float = 0.15, ) -> dict[str, Any]: target_width, target_height = normalize_size(width, height, max_width=max_width, max_height=max_height) xrandr = shutil.which("xrandr") @@ -296,7 +306,8 @@ def resize_display( timeout=4, env=env, ) - time.sleep(0.15) + if settle_seconds > 0: + time.sleep(settle_seconds) current = current_display_size(display, xauthority=xauthority, home=home) ok = current == (target_width, target_height) if ok: diff --git a/helpers/virtual_desktop.py.dox.md b/helpers/virtual_desktop.py.dox.md index 34e7b2510..2681e2cfb 100644 --- a/helpers/virtual_desktop.py.dox.md +++ b/helpers/virtual_desktop.py.dox.md @@ -23,13 +23,13 @@ - `proxy_for_token(token: str) -> VirtualDesktopEndpoint | None` - `resize_session(token: str, width: int, height: int) -> dict[str, Any]` - `get_registry() -> VirtualDesktopRegistry` -- `session_url(token: str, title: str=...) -> str` +- `session_url(token: str, title: str=..., encoding: str=..., quality: int=..., speed: int=..., file_transfer: bool=..., printing: bool=...) -> str` - `collect_status() -> dict[str, Any]` - `find_xpra_html_root() -> Path | None` - `_package_installed(package: str) -> bool` - `normalize_size(width: int | float | str, height: int | float | str, max_width: int=..., max_height: int=..., min_width: int=..., min_height: int=...) -> tuple[int, int]` - `normalize_desktop_display_size(width: int | float | str, height: int | float | str, max_width: int=..., max_height: int=..., min_width: int=..., min_height: int=..., min_aspect_ratio: float=...) -> tuple[int, int]` -- `resize_display(display: int, width: int, height: int, max_width: int=..., max_height: int=..., window_class: str=..., keys: tuple[str, ...]=..., xauthority: str=..., home: str=...) -> dict[str, Any]` +- `resize_display(display: int, width: int, height: int, max_width: int=..., max_height: int=..., window_class: str=..., keys: tuple[str, ...]=..., xauthority: str=..., home: str=..., settle_seconds: float=...) -> dict[str, Any]` - `_ensure_xrandr_mode(env: dict[str, str], width: int, height: int) -> None` - `_select_xrandr_mode(env: dict[str, str], width: int, height: int) -> subprocess.CompletedProcess[str]` - `_xrandr_output_modes(env: dict[str, str]) -> tuple[str, set[str]]` @@ -54,6 +54,8 @@ - Important called helpers/classes observed in the source: `Path`, `files.get_abs_path`, `get_registry.register`, `get_registry.unregister`, `get_registry.proxy_for_token`, `get_registry.resize`, `quote`, `urlencode`, `find_xpra_html_root`, `subprocess.run`, `normalize_size`, `shutil.which`, `_display_env`, `current_display_size`, `_ensure_xrandr_mode`, `_select_xrandr_mode`, `time.sleep`, `strip`, `_xrandr_output_modes`, `result.stdout.splitlines`. - Keep request/response, tool, or helper semantics documented here at the same time as source changes. +- Session URLs keep Desktop's JPEG, printing, and file-transfer defaults while allowing restricted viewers such as Browser to negotiate encoding and disable unrelated capabilities. +- Display resizing keeps the Desktop settle delay by default; latency-sensitive callers may skip it when they immediately verify the XRandR size. ## Work Guidance diff --git a/plugins/_browser/AGENTS.md b/plugins/_browser/AGENTS.md index bfe481621..973ec15ca 100644 --- a/plugins/_browser/AGENTS.md +++ b/plugins/_browser/AGENTS.md @@ -2,23 +2,30 @@ ## Purpose -- Own the built-in Playwright browser tool and WebUI browser viewer. +- Own the built-in Patchright browser tool and WebUI browser viewer. - Bridge browser automation, page inspection helpers, and browser panel UI. ## Ownership - `plugin.yaml` and `default_config.yaml` own metadata and browser settings defaults. - `tools/browser.py` owns the agent-facing browser tool. -- `helpers/` owns Playwright runtime, selectors, URL helpers, extension management, and connector runtime logic. +- `helpers/` owns the Patchright runtime, private interactive display, selectors, URL helpers, extension management, and connector runtime logic. - `api/` owns status, extension, and browser WebSocket handlers. - `assets/`, `prompts/`, `skills/`, `extensions/`, and `webui/` own browser scripts, prompts, skill guidance, hook contributions, and UI. ## Local Contracts - Keep browser actions safe around external pages, credentials, and user data. -- Preserve Playwright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding. +- Preserve Patchright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding. - Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation. -- Default the visible WebUI Browser to live CDP screencast for responsiveness. Keep lightweight CDP/DOM state snapshots as the fallback transport. +- Default the visible WebUI Browser to the authenticated Xpra HTML5 viewer for its existing Patchright page. Keep live CDP screencast and lightweight snapshots as automatic fallbacks. +- Keep headful Chromium in a normal window with its own toolbar clipped above the private display; do not use browser fullscreen, which shows Chromium's exit warning. +- Throttle interactive resize updates throughout a drag and let the native-sized Chromium viewport follow the private display; do not defer all layout updates until resizing stops. +- Keep exactly one interactive viewer iframe connected during canvas/modal handoff so hidden surfaces cannot compete to resize the same display. +- Notify the active Xpra client of its new frame geometry before resizing the backing display; after an interactive canvas/modal handoff, reconcile once after Xpra's deferred resize so Chromium cannot retain the previous surface size. +- Present the Xpra shadow window as the raw browser canvas: remove its HTML decoration and shadow pointer while preserving exact viewport geometry. +- Give every internal Browser runtime its own Xvfb display and unguessable Xpra gateway token; never expose another chat context's display through a shared viewer. +- Bind Browser Xpra endpoints to loopback, route them through the authenticated virtual-desktop gateway, and keep file transfer, URL opening, printing, and audio disabled. - Paint live screencast frames through the Browser panel canvas/ImageBitmap path when available; keep the ``/data URL path for snapshots and fallback rendering. - Push internal screencast frames from the runtime to the WebSocket consumer after subscription; keep `read/pop_screencast_frame` as fallback/tooling APIs, not the WebUI hot path. - Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback. Do not let the WebUI advertise binary frames unless its Socket.IO client reconstructs attachments as real `Blob`, `ArrayBuffer`, or typed-array values. @@ -35,6 +42,7 @@ - Run internal Chromium headful through Patchright on the private virtual display; do not add user-agent or header spoofing on top of the patched driver. - Browser startup and on-demand launch must converge on the Chromium revision declared by Patchright; let its installer select the host architecture rather than hardcoding x64 or ARM downloads. - `hooks.prepare_playwright_cache()` owns reconciliation of the pinned Patchright package and Chromium binary so repository self-updates and fresh images use the same setup path. +- Browser startup must install the shared virtual-desktop route hook itself; do not make Browser depend on the Desktop plugin being enabled. ## Work Guidance @@ -47,7 +55,7 @@ ## Verification - Smoke-test browser launch, navigation, DOM capture, and WebUI viewer after runtime changes. -- For viewer render-path changes, verify the live Browser panel paints a screencast frame on canvas with `frameSrc` empty and snapshots still falling back to the image path. +- For viewer render-path changes, verify direct iframe interaction reaches the same page controlled by Patchright, separate contexts use separate displays, and an unavailable Xpra runtime falls back to CDP screencast/snapshot rendering. - Run browser prompt/skill regression tests after changing browser prompt or Browser plugin skills. ## Child DOX Index diff --git a/plugins/_browser/api/status.py b/plugins/_browser/api/status.py index 2669fe677..8b9cefd80 100644 --- a/plugins/_browser/api/status.py +++ b/plugins/_browser/api/status.py @@ -1,5 +1,6 @@ from helpers.api import ApiHandler, Request from plugins._browser.helpers.config import build_browser_launch_config, get_browser_config +from plugins._browser.helpers.interactive_view import collect_status as collect_interactive_status from plugins._browser.helpers.playwright import ( get_playwright_binary, get_playwright_cache_dir, @@ -37,5 +38,6 @@ class Status(ApiHandler): "requires_full_browser": launch_config["requires_full_browser"], }, "host_browser": host_browser, + "interactive_view": collect_interactive_status(), "contexts": known_context_ids(), } diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.py index 6c810ebc9..4c5b734e5 100644 --- a/plugins/_browser/api/ws_browser.py +++ b/plugins/_browser/api/ws_browser.py @@ -25,7 +25,12 @@ SCREENCAST_STREAM_QUALITY = 80 SCREENSHOT_QUALITY = 92 VIEWER_TRANSPORT_SCREENCAST = "screencast" VIEWER_TRANSPORT_SNAPSHOT = "snapshot" -VIEWER_TRANSPORTS = {VIEWER_TRANSPORT_SCREENCAST, VIEWER_TRANSPORT_SNAPSHOT} +VIEWER_TRANSPORT_INTERACTIVE = "interactive" +VIEWER_TRANSPORTS = { + VIEWER_TRANSPORT_INTERACTIVE, + VIEWER_TRANSPORT_SCREENCAST, + VIEWER_TRANSPORT_SNAPSHOT, +} class WsBrowser(WsHandler): @@ -87,8 +92,19 @@ class WsBrowser(WsHandler): if opened.get("id"): listing["last_interacted_browser_id"] = opened.get("id") active_id = self._active_browser_id(listing, data.get("browser_id")) + requested_transport = self._viewer_transport(data) + viewer_transport, interactive_view = await self._effective_viewer( + runtime, + active_id, + data, + ) initial_viewport = self._viewport_from_data(data) - if runtime and active_id and initial_viewport: + if ( + runtime + and active_id + and initial_viewport + and viewer_transport != VIEWER_TRANSPORT_INTERACTIVE + ): await runtime.call( "set_viewport", active_id, @@ -103,7 +119,6 @@ class WsBrowser(WsHandler): if existing: existing.cancel() viewer_id = str(data.get("viewer_id") or "") - viewer_transport = self._viewer_transport(data) binary_frames = self._bool(data.get("binary_frames", data.get("binaryFrames"))) slim_frames = self._bool(data.get("slim_frames", data.get("slimFrames", binary_frames))) capture_scale = self._capture_scale_from_data(data) @@ -120,7 +135,13 @@ class WsBrowser(WsHandler): capture_scale=capture_scale, ) else: - stream_task = self._stream_state(sid, context_id, active_id, viewer_id) + stream_task = self._stream_state( + sid, + context_id, + active_id, + viewer_id, + viewer_transport=viewer_transport, + ) self._streams[stream_key] = asyncio.create_task(stream_task) snapshot = await self._snapshot_for_browser(runtime, active_id) @@ -136,6 +157,14 @@ class WsBrowser(WsHandler): "tab_scope": tab_scope, "viewer_id": viewer_id, "viewer_transport": viewer_transport, + "interactive_view": interactive_view, + "viewer_fallback_reason": ( + str(interactive_view.get("error") or "") + if requested_transport == VIEWER_TRANSPORT_INTERACTIVE + and interactive_view + and not interactive_view.get("available") + else "" + ), "binary_frames": binary_frames, "slim_frames": slim_frames, } @@ -253,6 +282,15 @@ class WsBrowser(WsHandler): listing = await runtime.call("list") last_interacted_browser_id = listing.get("last_interacted_browser_id") + active_id = self._active_browser_id( + listing, + self._result_browser_id(result) or browser_id, + ) + viewer_transport, interactive_view = await self._effective_viewer( + runtime, + active_id, + data, + ) snapshot = await self._snapshot_for_result(runtime, result) browsers, all_browsers, tab_scope = await self._tabs_for_scope( context_id, @@ -273,7 +311,8 @@ class WsBrowser(WsHandler): "all_browsers": all_browsers, "tab_scope": tab_scope, "last_interacted_browser_id": last_interacted_browser_id, - "viewer_transport": self._viewer_transport(data), + "viewer_transport": viewer_transport, + "interactive_view": interactive_view, }, correlation_id=data.get("correlationId"), ) @@ -288,7 +327,8 @@ class WsBrowser(WsHandler): "command": command, "browser_id": browser_id, "viewer_id": viewer_id, - "viewer_transport": self._viewer_transport(data), + "viewer_transport": viewer_transport, + "interactive_view": interactive_view, } async def _input(self, data: dict[str, Any], sid: str) -> dict[str, Any] | WsResult: @@ -326,12 +366,15 @@ class WsBrowser(WsHandler): text=str(data.get("text") or ""), ) elif input_type == "viewport": + viewer_transport = self._viewer_transport(data) result = await runtime.call( "set_viewport", browser_id, int(data.get("width") or 0), int(data.get("height") or 0), restart_screencast=bool(data.get("restart_stream")), + resize_interactive=viewer_transport == VIEWER_TRANSPORT_INTERACTIVE, + include_state=viewer_transport != VIEWER_TRANSPORT_INTERACTIVE, ) elif input_type == "wheel": result = await runtime.call( @@ -582,6 +625,8 @@ class WsBrowser(WsHandler): context_id: str, browser_id: int | str | None, viewer_id: str = "", + *, + viewer_transport: str = VIEWER_TRANSPORT_SNAPSHOT, ) -> None: last_signature = None while True: @@ -594,7 +639,7 @@ class WsBrowser(WsHandler): sid, context_id, viewer_id=viewer_id, - frame_source=VIEWER_TRANSPORT_SNAPSHOT, + frame_source=viewer_transport, ) last_signature = signature await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS) @@ -625,7 +670,7 @@ class WsBrowser(WsHandler): browsers=browsers, viewer_id=viewer_id, state=state, - viewer_transport=VIEWER_TRANSPORT_SNAPSHOT, + viewer_transport=viewer_transport, ) last_signature = signature await asyncio.sleep(SNAPSHOT_STATE_POLL_SECONDS) @@ -653,6 +698,36 @@ class WsBrowser(WsHandler): active_id = browsers[0].get("id") return active_id + async def _effective_viewer( + self, + runtime: Any, + browser_id: int | str | None, + data: dict[str, Any], + ) -> tuple[str, dict[str, Any] | None]: + requested = self._viewer_transport(data) + if requested != VIEWER_TRANSPORT_INTERACTIVE or not runtime or not browser_id: + return requested, None + viewport = self._viewport_from_data(data) or {} + try: + viewer = await runtime.call( + "interactive_viewer", + browser_id, + width=int(viewport.get("width") or 0), + height=int(viewport.get("height") or 0), + ) + except Exception as exc: + viewer = {"available": False, "error": str(exc)} + if viewer.get("available"): + return VIEWER_TRANSPORT_INTERACTIVE, viewer + return VIEWER_TRANSPORT_SCREENCAST, viewer + + @staticmethod + def _result_browser_id(result: Any) -> int | str | None: + if not isinstance(result, dict): + return None + state = result.get("state") if isinstance(result.get("state"), dict) else result + return state.get("id") if isinstance(state, dict) else None + @staticmethod def _state_for_browser( browsers: list[dict[str, Any]], diff --git a/plugins/_browser/extensions/python/startup_migration/_20_browser_playwright_cache.py b/plugins/_browser/extensions/python/startup_migration/_20_browser_playwright_cache.py index e76a19856..8839dc55c 100644 --- a/plugins/_browser/extensions/python/startup_migration/_20_browser_playwright_cache.py +++ b/plugins/_browser/extensions/python/startup_migration/_20_browser_playwright_cache.py @@ -3,6 +3,7 @@ from __future__ import annotations import threading from typing import Any +from helpers import virtual_desktop_routes from helpers.extension import Extension from helpers.print_style import PrintStyle from plugins._browser import hooks @@ -13,6 +14,7 @@ _startup_migration_thread: threading.Thread | None = None class BrowserPlaywrightCacheMigration(Extension): def execute(self, **kwargs): + virtual_desktop_routes.install_route_hooks() _start_background_cache_migration() diff --git a/plugins/_browser/helpers/interactive_view.py b/plugins/_browser/helpers/interactive_view.py new file mode 100644 index 000000000..a603633d0 --- /dev/null +++ b/plugins/_browser/helpers/interactive_view.py @@ -0,0 +1,293 @@ +from __future__ import annotations + +import os +import select +import shutil +import socket +import subprocess +import threading +import time +import uuid +from pathlib import Path +from typing import Any + +from helpers import files, virtual_desktop +from helpers.print_style import PrintStyle + + +DEFAULT_WIDTH = 1024 +DEFAULT_HEIGHT = 768 +START_TIMEOUT_SECONDS = 15.0 + + +def collect_status() -> dict[str, Any]: + binaries = { + name: shutil.which(name) or "" + for name in ("Xvfb", "xpra", "xrandr") + } + html_root = virtual_desktop.find_xpra_html_root() + missing = [name for name, path in binaries.items() if not path] + if not html_root: + missing.append("xpra-html5") + return { + "available": not missing, + "missing": missing, + "binaries": binaries, + "xpra_html_root": str(html_root or ""), + } + + +class BrowserInteractiveView: + """Own one private X display and its optional Xpra viewer.""" + + def __init__(self, context_id: str) -> None: + self.context_id = str(context_id) + self.token = f"browser-{uuid.uuid4().hex}" + self.state_dir = Path(files.get_abs_path("tmp", "browser", "displays", self.token)) + self.display: int | None = None + self.port = 0 + self.width = DEFAULT_WIDTH + self.height = DEFAULT_HEIGHT + self._xvfb: subprocess.Popen[Any] | None = None + self._xpra: subprocess.Popen[Any] | None = None + self._lock = threading.RLock() + + @property + def display_name(self) -> str: + return f":{self.display}" if self.display is not None else "" + + def ensure_display(self) -> str: + with self._lock: + if self._running(self._xvfb) and self.display is not None: + return self.display_name + + self._stop_locked() + xvfb = shutil.which("Xvfb") + if not xvfb: + return "" + + self.state_dir.mkdir(parents=True, exist_ok=True) + self.state_dir.chmod(0o700) + read_fd, write_fd = os.pipe() + try: + process = subprocess.Popen( + [ + xvfb, + "-displayfd", + str(write_fd), + "-screen", + "0", + f"{virtual_desktop.MAX_WIDTH}x{virtual_desktop.MAX_HEIGHT}x24", + "+extension", + "GLX", + "+extension", + "RANDR", + "+extension", + "RENDER", + "+extension", + "Composite", + "-nolisten", + "tcp", + "-noreset", + "-ac", + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + pass_fds=(write_fd,), + ) + except OSError: + os.close(read_fd) + return "" + finally: + os.close(write_fd) + + try: + ready, _, _ = select.select([read_fd], [], [], 5) + display_number = os.read(read_fd, 32).decode().strip() if ready else "" + finally: + os.close(read_fd) + + if not display_number.isdigit() or process.poll() is not None: + self._terminate(process) + return "" + + self._xvfb = process + self.display = int(display_number) + self.resize(self.width, self.height) + return self.display_name + + def ensure_viewer(self, width: int = 0, height: int = 0) -> dict[str, Any]: + with self._lock: + display_name = self.ensure_display() + if not display_name: + return self._unavailable("Xvfb is unavailable.") + + status = collect_status() + if not status["available"]: + return self._unavailable( + f"Interactive Browser runtime needs: {', '.join(status['missing'])}." + ) + + self.resize(width or self.width, height or self.height) + if not self._running(self._xpra): + try: + self._start_xpra(str(status["binaries"]["xpra"])) + except Exception as exc: + PrintStyle.warning(f"Interactive Browser viewer failed to start: {exc}") + self._terminate(self._xpra) + self._xpra = None + self.port = 0 + virtual_desktop.unregister_session(self.token) + return self._unavailable(str(exc)) + + virtual_desktop.register_session( + token=self.token, + host="127.0.0.1", + port=self.port, + owner="browser", + title="Browser", + resize=self.resize, + ) + return { + "available": True, + "token": self.token, + "url": virtual_desktop.session_url( + self.token, + title="Browser", + encoding="", + quality=90, + speed=90, + file_transfer=False, + printing=False, + ), + "width": self.width, + "height": self.height, + } + + def resize(self, width: int, height: int) -> dict[str, Any]: + with self._lock: + target_width, target_height = virtual_desktop.normalize_size(width, height) + self.width = target_width + self.height = target_height + if self.display is None or not self._running(self._xvfb): + return { + "ok": False, + "error": "Browser display is unavailable.", + "width": target_width, + "height": target_height, + } + result = virtual_desktop.resize_display( + display=self.display, + width=target_width, + height=target_height, + settle_seconds=0, + ) + return result + + def close(self) -> None: + with self._lock: + self._stop_locked() + shutil.rmtree(self.state_dir, ignore_errors=True) + + def _start_xpra(self, xpra: str) -> None: + self.port = self._free_port() + runtime_dir = self.state_dir / "runtime" + socket_dir = self.state_dir / "sockets" + runtime_dir.mkdir(parents=True, exist_ok=True) + socket_dir.mkdir(parents=True, exist_ok=True) + runtime_dir.chmod(0o700) + env = { + **os.environ, + "DISPLAY": self.display_name, + "XDG_RUNTIME_DIR": str(runtime_dir), + } + self._xpra = subprocess.Popen( + [ + xpra, + "shadow", + self.display_name, + "--daemon=no", + "--mdns=no", + "--html=on", + "--tray=no", + "--system-tray=no", + "--notifications=no", + "--clipboard=yes", + "--clipboard-direction=both", + "--file-transfer=no", + "--open-files=no", + "--open-url=no", + "--printing=no", + "--audio=no", + "--speaker=off", + "--microphone=off", + "--sharing=yes", + "--resize-display=yes", + "--encoding=auto", + "--quality=90", + "--speed=90", + f"--bind-tcp=127.0.0.1:{self.port}", + f"--socket-dir={socket_dir}", + f"--log-dir={self.state_dir}", + "--log-file=xpra.log", + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + env=env, + ) + self._wait_for_port(self._xpra, self.port) + + def _stop_locked(self) -> None: + virtual_desktop.unregister_session(self.token) + self._terminate(self._xpra) + self._terminate(self._xvfb) + self._xpra = None + self._xvfb = None + self.port = 0 + self.display = None + + def _unavailable(self, error: str) -> dict[str, Any]: + return { + "available": False, + "error": str(error or "Interactive Browser viewer is unavailable."), + } + + @staticmethod + def _running(process: subprocess.Popen[Any] | None) -> bool: + return bool(process and process.poll() is None) + + @staticmethod + def _terminate(process: subprocess.Popen[Any] | None) -> None: + if not process or process.poll() is not None: + return + process.terminate() + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=2) + + @staticmethod + def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.bind(("127.0.0.1", 0)) + return int(probe.getsockname()[1]) + + @staticmethod + def _wait_for_port( + process: subprocess.Popen[Any], + port: int, + timeout: float = START_TIMEOUT_SECONDS, + ) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError("Xpra exited before its Browser endpoint was ready.") + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.2): + return + except OSError: + time.sleep(0.1) + raise TimeoutError("Timed out waiting for the interactive Browser endpoint.") diff --git a/plugins/_browser/helpers/playwright.py b/plugins/_browser/helpers/playwright.py index f0463e4fa..bd0a2d639 100644 --- a/plugins/_browser/helpers/playwright.py +++ b/plugins/_browser/helpers/playwright.py @@ -1,9 +1,6 @@ -import atexit import json import os import re -import select -import shutil import subprocess import sys import threading @@ -23,9 +20,6 @@ RETIRED_PLAYWRIGHT_CACHE_DIRS = ( ("usr", "browser", "playwright"), ) _INSTALL_LOCK = threading.Lock() -_DISPLAY_LOCK = threading.Lock() -_DISPLAY_PROCESS: subprocess.Popen | None = None -_DISPLAY_NAME = "" def _primary_cache_dir() -> Path: @@ -64,82 +58,6 @@ def configure_playwright_env() -> str: return cache_dir -def ensure_browser_display() -> str: - global _DISPLAY_NAME, _DISPLAY_PROCESS - - with _DISPLAY_LOCK: - if _DISPLAY_PROCESS and _DISPLAY_PROCESS.poll() is None: - return _DISPLAY_NAME - - xvfb = shutil.which("Xvfb") - if not xvfb: - return "" - - read_fd, write_fd = os.pipe() - try: - process = subprocess.Popen( - [ - xvfb, - "-displayfd", - str(write_fd), - "-screen", - "0", - "1365x768x24", - "+extension", - "GLX", - "-nolisten", - "tcp", - "-noreset", - "-ac", - ], - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - pass_fds=(write_fd,), - ) - except OSError: - os.close(read_fd) - return "" - finally: - os.close(write_fd) - - try: - ready, _, _ = select.select([read_fd], [], [], 5) - display_number = os.read(read_fd, 32).decode().strip() if ready else "" - finally: - os.close(read_fd) - - if not display_number.isdigit() or process.poll() is not None: - _terminate_browser_display(process) - return "" - - _DISPLAY_PROCESS = process - _DISPLAY_NAME = f":{display_number}" - return _DISPLAY_NAME - - -def close_browser_display() -> None: - global _DISPLAY_NAME, _DISPLAY_PROCESS - - with _DISPLAY_LOCK: - process = _DISPLAY_PROCESS - _DISPLAY_PROCESS = None - _DISPLAY_NAME = "" - if process: - _terminate_browser_display(process) - - -def _terminate_browser_display(process: subprocess.Popen) -> None: - if process.poll() is not None: - return - process.terminate() - try: - process.wait(timeout=2) - except subprocess.TimeoutExpired: - process.kill() - process.wait(timeout=2) - - def find_playwright_binary(cache_dir: Path, revision: str = "") -> Path | None: prefix = f"chromium-{revision}" if revision.isdigit() else "chromium-*" binaries = [ @@ -197,6 +115,3 @@ def ensure_playwright_binary() -> Path: if not binary: raise RuntimeError("Patchright Chromium binary not found after installation") return binary - - -atexit.register(close_browser_display) diff --git a/plugins/_browser/helpers/runtime.py b/plugins/_browser/helpers/runtime.py index 040503cba..1514d6b40 100644 --- a/plugins/_browser/helpers/runtime.py +++ b/plugins/_browser/helpers/runtime.py @@ -27,9 +27,7 @@ from plugins._browser.helpers.config import ( build_browser_launch_config, get_browser_config, ) -from plugins._browser.helpers.playwright import ( - ensure_browser_display, -) +from plugins._browser.helpers.interactive_view import BrowserInteractiveView from plugins._browser.helpers.url import normalize_url @@ -558,7 +556,10 @@ class BrowserRuntime: await self.call("close", delete_profile=delete_profile) finally: self._closed = True - self._worker.kill(terminate_thread=True) + try: + self._worker.kill(terminate_thread=True) + finally: + self._core.interactive_view.close() class _BrowserRuntimeCore: @@ -594,6 +595,11 @@ class _BrowserRuntimeCore: self._pending_popups: list[asyncio.Future[int]] = [] self._background_popup_pages: set[int] = set() self._bootstrap_page: Any | None = None + self._browser_chrome_height: int | None = None + self._browser_window_page: Any | None = None + self._browser_window_session: Any | None = None + self._browser_window_id: int | None = None + self.interactive_view = BrowserInteractiveView(context_id) def _ensure_registry_lock(self) -> asyncio.Lock: if self._registry_lock is None: @@ -777,6 +783,10 @@ class _BrowserRuntimeCore: self._pending_popups.clear() self._background_popup_pages.clear() self._bootstrap_page = None + self._browser_chrome_height = None + self._browser_window_page = None + self._browser_window_session = None + self._browser_window_id = None self.pages.clear() self.last_interacted_browser_id = None for screencast in self.screencasts.values(): @@ -815,7 +825,15 @@ class _BrowserRuntimeCore: browser_config = get_browser_config() launch_config = build_browser_launch_config(browser_config) browser_binary = Path(preparation["binary"]) - browser_display = ensure_browser_display() + browser_display = self.interactive_view.ensure_display() + launch_args = list(launch_config["args"]) + if browser_display: + launch_args.extend( + [ + "--window-position=0,0", + f"--window-size={self.interactive_view.width},{self.interactive_view.height}", + ] + ) self.playwright = await async_playwright().start() launch_kwargs: dict[str, Any] = { @@ -823,13 +841,17 @@ class _BrowserRuntimeCore: "headless": not bool(browser_display), "accept_downloads": True, "downloads_path": str(self.downloads_dir), - "viewport": DEFAULT_VIEWPORT, - "screen": DEFAULT_VIEWPORT, - "no_viewport": False, - "args": launch_config["args"], + "args": launch_args, } if browser_display: launch_kwargs["env"] = {**os.environ, "DISPLAY": browser_display} + launch_kwargs["no_viewport"] = True + else: + launch_kwargs.update( + viewport=DEFAULT_VIEWPORT, + screen=DEFAULT_VIEWPORT, + no_viewport=False, + ) if launch_config["channel"]: launch_kwargs["channel"] = launch_config["channel"] else: @@ -857,6 +879,7 @@ class _BrowserRuntimeCore: if page.url == "about:blank": if browser_display and self._bootstrap_page is None: self._bootstrap_page = page + await self._fit_browser_window(page) continue try: await page.close() @@ -1236,8 +1259,12 @@ class _BrowserRuntimeCore: async def set_active(self, browser_id: int | str | None) -> dict[str, Any]: await self.ensure_started() resolved_id = self._resolve_browser_id(browser_id) + page = self._page(resolved_id) # Explicit focus change — bypass _maybe_promote. self.last_interacted_browser_id = int(resolved_id) + with contextlib.suppress(Exception): + await page.bring_to_front() + await self._fit_browser_window(page) return await self._state(resolved_id) async def state(self, browser_id: int | str | None = None) -> dict[str, Any]: @@ -1703,7 +1730,7 @@ class _BrowserRuntimeCore: await screencast.start( quality=quality, every_nth_frame=every_nth_frame, - viewport=page.viewport_size or DEFAULT_VIEWPORT, + viewport=await self._page_viewport(page), capture_scale=capture_scale, ) except Exception: @@ -1717,6 +1744,61 @@ class _BrowserRuntimeCore: "state": await self._state(resolved_id), } + @staticmethod + async def _page_viewport(page: Any) -> dict[str, int]: + viewport = getattr(page, "viewport_size", None) + if viewport: + return { + "width": int(viewport.get("width") or DEFAULT_VIEWPORT["width"]), + "height": int(viewport.get("height") or DEFAULT_VIEWPORT["height"]), + } + try: + measured = await page.evaluate( + "() => ({ width: globalThis.innerWidth, height: globalThis.innerHeight })", + isolated_context=False, + ) + return { + "width": int(measured.get("width") or DEFAULT_VIEWPORT["width"]), + "height": int(measured.get("height") or DEFAULT_VIEWPORT["height"]), + } + except Exception: + return dict(DEFAULT_VIEWPORT) + + async def interactive_viewer( + self, + browser_id: int | str | None = None, + *, + width: int = 0, + height: int = 0, + ) -> dict[str, Any]: + await self.ensure_started() + resolved_id = self._resolve_browser_id(browser_id) + page = self._page(resolved_id) + current_viewport = await self._page_viewport(page) + viewer = self.interactive_view.ensure_viewer( + width or int(current_viewport.get("width") or DEFAULT_VIEWPORT["width"]), + height or int(current_viewport.get("height") or DEFAULT_VIEWPORT["height"]), + ) + if not viewer.get("available"): + return viewer + + await self._stop_screencasts_for_browser(resolved_id) + with contextlib.suppress(Exception): + await page.bring_to_front() + viewport_result = await self.set_viewport( + resolved_id, + int(viewer.get("width") or width or DEFAULT_VIEWPORT["width"]), + int(viewer.get("height") or height or DEFAULT_VIEWPORT["height"]), + resize_interactive=True, + ) + self.last_interacted_browser_id = int(resolved_id) + return { + **viewer, + "browser_id": resolved_id, + "state": viewport_result["state"], + "viewport": viewport_result["viewport"], + } + async def read_screencast_frame( self, stream_id: str, @@ -1756,15 +1838,30 @@ class _BrowserRuntimeCore: width: int, height: int, restart_screencast: bool = False, + resize_interactive: bool = False, + include_state: bool = True, ) -> dict[str, Any]: await self.ensure_started() resolved_id = self._resolve_browser_id(browser_id) page = self._page(resolved_id) + if resize_interactive: + resized = self.interactive_view.resize(width, height) + viewport = { + "width": int(resized.get("width") or self.interactive_view.width), + "height": int(resized.get("height") or self.interactive_view.height), + } + await self._fit_browser_window(page) + self._maybe_promote(resolved_id) + return { + "state": await self._state(resolved_id) if include_state else None, + "viewport": viewport, + } + viewport = { "width": max(320, min(4096, int(width or DEFAULT_VIEWPORT["width"]))), "height": max(200, min(4096, int(height or DEFAULT_VIEWPORT["height"]))), } - current_viewport = page.viewport_size or {} + current_viewport = await self._page_viewport(page) changed = ( abs(int(current_viewport.get("width") or 0) - viewport["width"]) > VIEWPORT_SIZE_TOLERANCE @@ -2182,6 +2279,7 @@ class _BrowserRuntimeCore: self._pending_popups.clear() self._background_popup_pages.clear() await self._stop_all_screencasts() + await self._reset_browser_window_session() for browser_id in list(self.pages): try: await self.pages[browser_id].page.close() @@ -2315,7 +2413,67 @@ class _BrowserRuntimeCore: async def _register_page(self, page: Any) -> BrowserPage: lock = self._ensure_registry_lock() async with lock: - return self._register_page_locked(page) + browser_page = self._register_page_locked(page) + await self._fit_browser_window(page) + return browser_page + + async def _fit_browser_window(self, page: Any) -> None: + if getattr(self.interactive_view, "display", None) is None or not self.context: + return + try: + if self._browser_window_session is None or self._browser_window_page is not page: + await self._reset_browser_window_session() + self._browser_window_page = page + self._browser_window_session = await self.context.new_cdp_session(page) + target = await self._browser_window_session.send("Browser.getWindowForTarget") + self._browser_window_id = target.get("windowId") + if self._browser_window_id is None: + await self._reset_browser_window_session() + return + current = await self._browser_window_session.send( + "Browser.getWindowBounds", + {"windowId": self._browser_window_id}, + ) + if current.get("bounds", {}).get("windowState") != "normal": + await self._browser_window_session.send( + "Browser.setWindowBounds", + { + "windowId": self._browser_window_id, + "bounds": {"windowState": "normal"}, + }, + ) + if self._browser_chrome_height is None: + chrome_height = await page.evaluate( + "() => Math.max(0, globalThis.outerHeight - globalThis.innerHeight)", + isolated_context=False, + ) + self._browser_chrome_height = max(0, min(256, int(chrome_height or 0))) + chrome_height = self._browser_chrome_height + await self._browser_window_session.send( + "Browser.setWindowBounds", + { + "windowId": self._browser_window_id, + "bounds": { + "windowState": "normal", + "left": 0, + "top": -chrome_height, + "width": self.interactive_view.width, + "height": self.interactive_view.height + chrome_height, + }, + }, + ) + except Exception as exc: + await self._reset_browser_window_session() + PrintStyle.warning(f"Interactive Browser window fit failed: {exc}") + + async def _reset_browser_window_session(self) -> None: + session = self._browser_window_session + self._browser_window_page = None + self._browser_window_session = None + self._browser_window_id = None + if session: + with contextlib.suppress(Exception): + await session.detach() async def _unregister_page_async(self, browser_id: int) -> None: try: @@ -2372,6 +2530,8 @@ class _BrowserRuntimeCore: if close_over_limit: with contextlib.suppress(Exception): await page.close() + else: + await self._fit_browser_window(page) except Exception as exc: PrintStyle.warning(f"Popup registration failed: {exc}") diff --git a/plugins/_browser/webui/browser-panel.html b/plugins/_browser/webui/browser-panel.html index 05d0f43b7..96e8bdfe7 100644 --- a/plugins/_browser/webui/browser-panel.html +++ b/plugins/_browser/webui/browser-panel.html @@ -20,10 +20,13 @@
-
- -
@@ -603,6 +606,13 @@ font-weight: 600; } + .browser-tab-loading { + flex: 0 0 auto; + color: color-mix(in srgb, var(--color-text) 66%, var(--color-primary) 34%); + font-size: 0.96rem; + line-height: 1; + } + .browser-tab-close { display: inline-flex; align-items: center; @@ -997,19 +1007,31 @@ cursor: crosshair; } - .browser-frame { - display: block; - position: absolute; - inset: 0; - width: 100%; - height: 100%; - min-width: 0; - min-height: 0; - object-fit: contain; - image-rendering: auto; - user-select: none; - background: #fff; - } + .browser-frame { + display: block; + position: absolute; + inset: 0; + width: 100%; + height: 100%; + min-width: 0; + min-height: 0; + object-fit: contain; + image-rendering: auto; + user-select: none; + background: #fff; + } + + .browser-interactive-frame { + display: block; + position: absolute; + inset: 0; + z-index: 1; + box-sizing: border-box; + width: 100%; + height: 100%; + border: 0; + background: #fff; + } .browser-annotation-layer { position: absolute; @@ -1239,7 +1261,6 @@ background: color-mix(in srgb, #7f1d1d 22%, var(--color-background)); } - .browser-status, .browser-error { display: inline-flex; align-items: center; @@ -1250,17 +1271,12 @@ white-space: nowrap; } - .browser-status span:not(.material-symbols-outlined), .browser-error { min-width: 0; overflow: hidden; text-overflow: ellipsis; } - .browser-status .material-symbols-outlined { - font-size: 15px; - } - .browser-modal .spinning, .browser-panel .spinning { display: inline-block; diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.js index cd8d85884..2260f87f0 100644 --- a/plugins/_browser/webui/browser-store.js +++ b/plugins/_browser/webui/browser-store.js @@ -21,11 +21,13 @@ const BROWSER_SUBSCRIBE_TIMEOUT_MS = 60000; const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000; const BROWSER_COMMAND_TIMEOUT_MS = 45000; const BROWSER_CONFIG_REFRESH_MS = 15000; +const BROWSER_VIEWER_TRANSPORT_INTERACTIVE = "interactive"; const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot"; const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast"; -const VIEWPORT_SYNC_DEBOUNCE_MS = 220; +const VIEWPORT_SYNC_INTERVAL_MS = 50; const VIEWPORT_SYNC_SIZE_TOLERANCE = 4; const CANVAS_VIEWPORT_SETTLE_MS = 520; +const INTERACTIVE_VIEWPORT_SETTLE_MS = 320; const SURFACE_VIEWPORT_STABLE_FRAMES = 4; const SURFACE_VIEWPORT_MAX_WAIT_MS = 1200; const FRAME_REJECT_SYNC_COOLDOWN_MS = 600; @@ -168,9 +170,10 @@ const model = { frameSrc: "", frameCanvasReady: false, frameState: null, - viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST, + viewerTransport: BROWSER_VIEWER_TRANSPORT_INTERACTIVE, + interactiveViewUrl: "", + viewerFallbackReason: "", tabScope: "per_context", - liveScreencastEnabled: true, annotating: false, annotationComments: [], annotationDraft: null, @@ -216,6 +219,7 @@ const model = { _openSignature: "", _connectSequence: 0, _viewerToken: "", + _subscribedViewerTransport: BROWSER_VIEWER_TRANSPORT_INTERACTIVE, _contextCreatePromise: null, _lastSelectedContextId: "", _sessionRefreshPromise: null, @@ -879,6 +883,7 @@ const model = { resetRenderedFrame() { this.cancelFrameRender(); + this.interactiveViewUrl = ""; this.clearFrameSrc(); this.clearFrameCanvas(); this._lastFrameDimensions = null; @@ -932,6 +937,7 @@ const model = { async syncViewportAfterSurfaceOpen(sequence = this._surfaceOpenSequence) { if (!this.connected || !this.activeBrowserId) return; + const surfaceMode = this._mode; await this.waitForSurfaceViewport({ sequence }); if (!this.isCurrentSurfaceOpen(sequence)) { return; @@ -939,19 +945,24 @@ const model = { await this.syncViewport(true, { restartStream: this._mode === "canvas" && this.usesScreencastTransport(), }); - if (this._mode !== "canvas") return; - this.scheduleViewportSyncForSurface(sequence, 240); - this.scheduleViewportSyncForSurface(sequence, 520); + if (surfaceMode === "modal" && this.usesInteractiveTransport()) { + this.scheduleViewportSyncForSurface(sequence, INTERACTIVE_VIEWPORT_SETTLE_MS, surfaceMode); + return; + } + if (surfaceMode !== "canvas") return; + this.scheduleViewportSyncForSurface(sequence, 240, surfaceMode); + this.scheduleViewportSyncForSurface(sequence, 520, surfaceMode); }, requestedViewerTransport() { - return this.liveScreencastEnabled - ? BROWSER_VIEWER_TRANSPORT_SCREENCAST - : BROWSER_VIEWER_TRANSPORT_SNAPSHOT; + return BROWSER_VIEWER_TRANSPORT_INTERACTIVE; }, normalizeViewerTransport(value = "") { const normalized = String(value || "").trim().toLowerCase().replace("-", "_"); + if (normalized === BROWSER_VIEWER_TRANSPORT_INTERACTIVE) { + return BROWSER_VIEWER_TRANSPORT_INTERACTIVE; + } if (normalized === BROWSER_VIEWER_TRANSPORT_SCREENCAST) { return BROWSER_VIEWER_TRANSPORT_SCREENCAST; } @@ -974,6 +985,101 @@ const model = { return this.viewerTransport === BROWSER_VIEWER_TRANSPORT_SCREENCAST; }, + usesInteractiveTransport() { + return this.viewerTransport === BROWSER_VIEWER_TRANSPORT_INTERACTIVE + && Boolean(this.interactiveViewUrl); + }, + + isInteractiveSurface(stage = null) { + return this.usesInteractiveTransport() && stage === this._stageElement; + }, + + prepareInteractiveViewFrame(frame = null) { + const target = frame || this._stageElement?.querySelector?.(".browser-interactive-frame"); + const remoteWindow = target?.contentWindow; + if (!remoteWindow) return false; + try { + const remoteDocument = target.contentDocument || remoteWindow.document; + if (!remoteDocument) return false; + if (!remoteDocument.getElementById("a0-xpra-browser-frame-css")) { + const style = remoteDocument.createElement("style"); + style.id = "a0-xpra-browser-frame-css"; + style.textContent = ` + #shadow_pointer { + display: none !important; + visibility: hidden !important; + opacity: 0 !important; + } + .window canvas, + .undecorated canvas { + display: block !important; + margin: 0 !important; + } + `; + remoteDocument.head?.appendChild(style); + } + + const normalizeWindows = () => { + const windows = Object.values(remoteWindow.client?.id_to_window || {}); + for (const xpraWindow of windows) { + xpraWindow.resizable = false; + xpraWindow.decorations = false; + xpraWindow.decorated = false; + xpraWindow.metadata = { ...(xpraWindow.metadata || {}), decorations: false }; + xpraWindow._set_decorated?.(false); + xpraWindow.configure_border_class?.(); + xpraWindow.leftoffset = 0; + xpraWindow.rightoffset = 0; + xpraWindow.topoffset = 0; + xpraWindow.bottomoffset = 0; + xpraWindow.updateCSSGeometry?.(); + } + return windows.length > 0; + }; + + const screen = remoteDocument.querySelector?.("#screen"); + if (screen && !remoteWindow.__a0BrowserFrameObserver && remoteWindow.MutationObserver) { + const observer = new remoteWindow.MutationObserver(normalizeWindows); + observer.observe(screen, { childList: true }); + remoteWindow.__a0BrowserFrameObserver = observer; + } + return normalizeWindows(); + } catch { + return false; + } + }, + + syncInteractiveViewSize() { + if (!this.usesInteractiveTransport()) return; + const frame = this._stageElement?.querySelector?.(".browser-interactive-frame"); + try { + this.prepareInteractiveViewFrame(frame); + frame?.contentWindow?.client?._screen_resized?.(); + } catch {} + }, + + applyViewer(data = {}) { + if (data?.viewer_transport) { + this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport); + } + if (Object.prototype.hasOwnProperty.call(data || {}, "interactive_view")) { + const viewer = data.interactive_view; + this.interactiveViewUrl = viewer?.available && viewer?.url ? String(viewer.url) : ""; + this.viewerFallbackReason = String(data.viewer_fallback_reason || viewer?.error || ""); + } + if (this.viewerTransport !== BROWSER_VIEWER_TRANSPORT_INTERACTIVE) { + this.interactiveViewUrl = ""; + } + }, + + onInteractiveViewLoad() { + if (!this.usesInteractiveTransport()) return; + this.prepareInteractiveViewFrame(); + this.switchingBrowserId = null; + this._surfaceSwitching = false; + this.queueViewportSync(true); + }, + supportsBinaryFrames() { return BROWSER_BINARY_FRAME_REQUESTS_ENABLED && BROWSER_BINARY_PAYLOADS_SUPPORTED; }, @@ -1003,9 +1109,9 @@ const model = { return { width, height }; }, - scheduleViewportSyncForSurface(sequence, delayMs = 0) { + scheduleViewportSyncForSurface(sequence, delayMs = 0, mode = this._mode) { globalThis.setTimeout?.(() => { - if (!this.isCurrentSurfaceOpen(sequence) || this._mode !== "canvas") { + if (!this.isCurrentSurfaceOpen(sequence) || this._mode !== mode) { return; } this.queueViewportSync(true); @@ -1093,7 +1199,8 @@ const model = { replaceAll: Boolean(data.all_browsers), replaceContext: !data.all_browsers, }); - this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport); + this.applyViewer(data); + this._subscribedViewerTransport = this.viewerTransport; this.setActiveBrowserId( data.active_browser_id || requestedBrowserId || this.activeBrowserId || null, data.active_browser_context_id || contextId, @@ -1108,9 +1215,7 @@ const model = { const frameHandler = ({ data }) => { if (data?.context_id !== this.contextId) return; if (data?.viewer_id && data.viewer_id !== this._viewerToken) return; - if (data?.viewer_transport) { - this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport); - } + this.applyViewer(data); this.applyTabScope(data); const incomingContextId = this.normalizeContextId(data.context_id || this.contextId); const incomingBrowserId = this.normalizeBrowserId(data.browser_id || data.state?.id); @@ -1180,9 +1285,7 @@ const model = { const stateHandler = ({ data }) => { if (data?.context_id !== this.contextId) return; if (data?.viewer_id && data.viewer_id !== this._viewerToken) return; - if (data?.viewer_transport) { - this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport); - } + this.applyViewer(data); this.applyTabScope(data); const commandContextId = this.normalizeContextId(data.active_browser_context_id || data.context_id || this.contextId); if (Array.isArray(data.browsers)) { @@ -1384,7 +1487,7 @@ const model = { }, hasFrame() { - return Boolean(this.frameSrc || this.frameCanvasReady); + return Boolean(this.interactiveViewUrl || this.frameSrc || this.frameCanvasReady); }, paintFrameBitmap(bitmap) { @@ -1419,6 +1522,10 @@ const model = { }, frameElement() { + if (this.usesInteractiveTransport()) { + const iframe = this._stageElement?.querySelector?.(".browser-interactive-frame"); + if (iframe) return iframe; + } if (this.frameCanvasReady) { const canvas = this.currentFrameCanvas(); if (canvas) return canvas; @@ -1467,7 +1574,7 @@ const model = { replaceAll: Boolean(data.all_browsers), replaceContext: !data.all_browsers, }); - this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport); + this.applyViewer(data); const result = data.result || {}; const resultContextId = this.normalizeContextId( result.context_id @@ -1510,7 +1617,12 @@ const model = { previousActiveBrowserId, previousActiveContextId, ); - if ((commandName === "open" || commandName === "close" || activeChanged) && this.contextId && this.activeBrowserId) { + const viewerTransportChanged = this._subscribedViewerTransport !== this.viewerTransport; + if ( + (commandName === "open" || commandName === "close" || activeChanged || viewerTransportChanged) + && this.contextId + && this.activeBrowserId + ) { await this.connectViewer({ browserId: this.activeBrowserId, contextId: this.activeBrowserContextId, @@ -1670,6 +1782,10 @@ const model = { return this.sameBrowserTab(browser?.id, browser?.context_id, this.activeBrowserId, this.activeBrowserContextId); }, + isBrowserLoading(browser) { + return Boolean(browser?.loading || (this.isActiveBrowser(browser) && this.isBusy())); + }, + browserTabTitle(browser) { const title = String(browser?.title || "").trim(); const url = String(browser?.currentUrl || "").trim(); @@ -1831,6 +1947,7 @@ const model = { if (snapshot.state) { this.applyActiveFrameState(snapshot.state); } + if (this.usesInteractiveTransport()) return; const frameBrowserId = snapshotId || this.activeBrowserId; this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, { browserId: frameBrowserId, @@ -2521,13 +2638,21 @@ const model = { queueViewportSync(force = false) { this.clearRenderedFrameIfViewportChanged(); + if (force) { + if (this._viewportSyncTimer) { + globalThis.clearTimeout(this._viewportSyncTimer); + this._viewportSyncTimer = null; + } + void this.syncViewport(true); + return; + } if (this._viewportSyncTimer) { - globalThis.clearTimeout(this._viewportSyncTimer); + return; } this._viewportSyncTimer = globalThis.setTimeout(() => { this._viewportSyncTimer = null; - void this.syncViewport(force); - }, force ? 0 : VIEWPORT_SYNC_DEBOUNCE_MS); + void this.syncViewport(false); + }, VIEWPORT_SYNC_INTERVAL_MS); }, async syncViewport(force = false, options = {}) { @@ -2542,7 +2667,7 @@ const model = { } const key = `${contextId}:${this.activeBrowserId}:${viewport.width}x${viewport.height}`; if ( - (!restartStream && this._lastViewportKey === key) + (!force && !restartStream && this._lastViewportKey === key) || ( !force && !restartStream @@ -2555,11 +2680,13 @@ const model = { return; } try { + this.syncInteractiveViewSize(); await websocket.emit("browser_viewer_input", { context_id: contextId, browser_id: this.activeBrowserId, viewer_id: this._viewerToken, input_type: "viewport", + viewer_transport: this.viewerTransport, width: viewport.width, height: viewport.height, restart_stream: restartStream && this.usesScreencastTransport(), @@ -2739,6 +2866,9 @@ const model = { this._viewerToken = ""; this.switchingBrowserId = null; this.viewerTransport = this.requestedViewerTransport(); + this._subscribedViewerTransport = this.viewerTransport; + this.interactiveViewUrl = ""; + this.viewerFallbackReason = ""; this.tabScope = "per_context"; this._surfaceMounted = false; this._surfaceSwitching = false; @@ -3010,13 +3140,6 @@ const model = { return this.frameState?.currentUrl || this.address || "about:blank"; }, - loadingMessage() { - if (this.browserInstallExpected) { - const cacheDir = this.status?.playwright?.cache_dir || "/a0/tmp/playwright"; - return `Installing Chromium for the first Browser run. This can take a few minutes; future starts reuse ${cacheDir}.`; - } - return "Loading"; - }, }; export const store = createStore("browserPage", model); diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py index 2c546b421..edf4cd374 100644 --- a/tests/test_browser_agent_regressions.py +++ b/tests/test_browser_agent_regressions.py @@ -82,7 +82,7 @@ sys.modules.setdefault("plugins._model_config.helpers.model_config", _model_conf def anyio_backend(): return "asyncio" -from helpers import ephemeral_images +from helpers import ephemeral_images, virtual_desktop from helpers.errors import RepairableException from plugins._browser.helpers.config import ( build_browser_launch_config, @@ -109,6 +109,8 @@ from plugins._browser.helpers.runtime import ( normalize_url, ) import plugins._browser.helpers.runtime as browser_runtime_module +from plugins._browser.helpers.interactive_view import BrowserInteractiveView +import plugins._browser.helpers.interactive_view as browser_interactive_view_module from plugins._browser.helpers.playwright import ( ensure_playwright_binary, get_playwright_binary, @@ -833,7 +835,8 @@ def test_browser_viewer_allows_slow_extension_startup(): assert "const BROWSER_COMMAND_TIMEOUT_MS = 45000;" in js assert "? BROWSER_FIRST_INSTALL_TIMEOUT_MS" in js assert ": BROWSER_SUBSCRIBE_TIMEOUT_MS" in js - assert "Installing Chromium for the first Browser run" in js + assert "browserInstallExpected" in js + assert "Installing Chromium for the first Browser run" not in js def test_browser_viewer_creates_chat_when_no_context_is_selected(): @@ -880,6 +883,20 @@ def test_browser_canvas_startup_waits_for_raw_viewport_settle(): assert "this.resetRenderedFrame();" in js +def test_browser_interactive_modal_handoff_reconciles_after_xpra_resize(): + js = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js").read_text( + encoding="utf-8" + ) + + assert "const INTERACTIVE_VIEWPORT_SETTLE_MS = 320;" in js + assert "const surfaceMode = this._mode;" in js + assert 'surfaceMode === "modal" && this.usesInteractiveTransport()' in js + assert "this.scheduleViewportSyncForSurface(sequence, INTERACTIVE_VIEWPORT_SETTLE_MS, surfaceMode);" in js + assert "scheduleViewportSyncForSurface(sequence, delayMs = 0, mode = this._mode)" in js + assert "this._mode !== mode" in js + assert "(!force && !restartStream && this._lastViewportKey === key)" in js + + def test_browser_surface_handoffs_keep_existing_frame_until_replacement_arrives(): js = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js").read_text( encoding="utf-8" @@ -1393,6 +1410,12 @@ def test_browser_viewer_uses_tabs_for_session_switching(): assert ':key="$store.browserPage.browserTabKey(browser)"' in main_html assert "browser.context_id" in main_html assert ':title="$store.browserPage.browserTabTooltip(browser)"' in main_html + assert ':aria-busy="$store.browserPage.isBrowserLoading(browser).toString()"' in main_html + assert 'class="browser-tab-loading spinning"' in main_html + assert 'x-show="$store.browserPage.isBrowserLoading(browser)"' in main_html + assert "Loading" not in main_html + assert "isBrowserLoading(browser)" in browser_store + assert "loadingMessage()" not in browser_store assert "browser-tab-context" not in main_html assert 'handleSelectedContextChange($store.chats?.selected)' in main_html assert "activeBrowserContextId" in browser_store @@ -1445,7 +1468,7 @@ def test_browser_tabs_close_without_confirmation_or_busy_lock(): assert "_commandInFlightCount" in browser_store -def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback(): +def test_browser_viewer_defaults_to_interactive_with_screencast_and_snapshot_fallbacks(): ws_browser = (PROJECT_ROOT / "plugins" / "_browser" / "api" / "ws_browser.py").read_text( encoding="utf-8" ) @@ -1465,6 +1488,9 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback(): assert 'runtime.call("screenshot"' in ws_browser assert 'VIEWER_TRANSPORT_SNAPSHOT = "snapshot"' in ws_browser assert 'VIEWER_TRANSPORT_SCREENCAST = "screencast"' in ws_browser + assert 'VIEWER_TRANSPORT_INTERACTIVE = "interactive"' in ws_browser + assert "async def _effective_viewer(" in ws_browser + assert "return VIEWER_TRANSPORT_SCREENCAST, viewer" in ws_browser assert "def _viewer_transport(data: dict[str, Any])" in ws_browser assert "return VIEWER_TRANSPORT_SNAPSHOT" in ws_browser assert "self._stream_state" in ws_browser @@ -1510,8 +1536,20 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback(): assert "return await globalThis.createImageBitmap(blob);" in browser_store assert 'const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";' in browser_store assert 'const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";' in browser_store - assert "viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST" in browser_store - assert "liveScreencastEnabled: true" in browser_store + assert 'const BROWSER_VIEWER_TRANSPORT_INTERACTIVE = "interactive";' in browser_store + assert "const VIEWPORT_SYNC_INTERVAL_MS = 50;" in browser_store + assert "viewerTransport: BROWSER_VIEWER_TRANSPORT_INTERACTIVE" in browser_store + assert "return BROWSER_VIEWER_TRANSPORT_INTERACTIVE;" in browser_store + assert "usesInteractiveTransport()" in browser_store + assert "isInteractiveSurface(stage = null)" in browser_store + assert "prepareInteractiveViewFrame(frame = null)" in browser_store + assert 'style.id = "a0-xpra-browser-frame-css";' in browser_store + assert "#shadow_pointer" in browser_store + assert "xpraWindow._set_decorated?.(false);" in browser_store + assert "xpraWindow.updateCSSGeometry?.();" in browser_store + assert "syncInteractiveViewSize()" in browser_store + assert "frame?.contentWindow?.client?._screen_resized?.();" in browser_store + assert "applyViewer(data = {})" in browser_store assert "requestedViewerTransport()" in browser_store assert "normalizeViewerTransport(value = \"\")" in browser_store assert "usesScreencastTransport()" in browser_store @@ -1583,6 +1621,11 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback(): assert "canvas_wheel_screenshot" not in ws_browser assert "surface_mode: this._mode" not in browser_store assert ' Math.max(0, globalThis.outerHeight - globalThis.innerHeight)", + ), + ( + "Browser.setWindowBounds", + { + "windowId": 7, + "bounds": { + "windowState": "normal", + "left": 0, + "top": -87, + "width": 1200, + "height": 787, + }, + }, + ), + ( + "Browser.setWindowBounds", + { + "windowId": 7, + "bounds": { + "windowState": "normal", + "left": 0, + "top": -87, + "width": 1280, + "height": 807, + }, + }, + ), + ("detach", None), + ] + + +@pytest.mark.anyio +async def test_browser_viewer_falls_back_when_interactive_runtime_is_unavailable(): + calls = [] + + class FakeRuntime: + async def call(self, method, *args, **kwargs): + calls.append((method, args, kwargs)) + return {"available": False, "error": "xpra unavailable"} + + handler = ws_browser_module.WsBrowser(SimpleNamespace(), threading.RLock(), manager=None) + transport, viewer = await handler._effective_viewer( + FakeRuntime(), + 7, + { + "viewer_transport": "interactive", + "viewport_width": 1200, + "viewport_height": 700, + }, + ) + + assert transport == ws_browser_module.VIEWER_TRANSPORT_SCREENCAST + assert viewer == {"available": False, "error": "xpra unavailable"} + assert calls == [("interactive_viewer", (7,), {"width": 1200, "height": 700})] + + def test_browser_runtime_removes_stale_profile_singletons(monkeypatch, tmp_path): monkeypatch.setattr( browser_runtime_module.files, @@ -3245,6 +3531,7 @@ async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch): "width": 1280, "height": 720, "restart_stream": True, + "viewer_transport": "interactive", }, "sid-1", ) @@ -3254,10 +3541,68 @@ async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch): "snapshot": None, } assert calls == [ - ("set_viewport", (7, 1280, 720), {"restart_screencast": True}) + ( + "set_viewport", + (7, 1280, 720), + { + "restart_screencast": True, + "resize_interactive": True, + "include_state": False, + }, + ) ] +@pytest.mark.anyio +async def test_browser_interactive_resize_uses_native_window_viewport(): + fitted = [] + + class FakePage: + viewport_size = None + + async def set_viewport_size(self, viewport): + raise AssertionError("Interactive native viewport must not enable emulation") + + class FakeInteractiveView: + display = 0 + width = 1024 + height = 768 + + def resize(self, width, height): + self.width = width + self.height = height + return {"ok": True, "width": width, "height": height} + + page = FakePage() + core = _BrowserRuntimeCore("ctx") + core.context = object() + core.pages[7] = browser_runtime_module.BrowserPage(id=7, page=page) + core.interactive_view = FakeInteractiveView() + + async def fake_fit(fitted_page): + fitted.append(fitted_page) + + async def fake_state(browser_id): + return {"id": browser_id} + + core._fit_browser_window = fake_fit + core._state = fake_state + + result = await core.set_viewport( + 7, + 1280, + 720, + resize_interactive=True, + include_state=False, + ) + + assert result == { + "state": None, + "viewport": {"width": 1280, "height": 720}, + } + assert fitted == [page] + + @pytest.mark.anyio async def test_browser_runtime_restarts_screencast_without_resizing_same_viewport(): viewport_calls = [] From a0eefe19bd81e180dd6c72bbb4206b265722f827 Mon Sep 17 00:00:00 2001 From: Alessandro <155005371+3clyp50@users.noreply.github.com> Date: Sun, 16 Aug 2026 15:21:52 +0200 Subject: [PATCH 04/20] Honor configured Browser model preset Activate the selected Browser preset after the first Browser tool call so follow-up model turns use it. Clear the temporary override at monologue end and preserve the main model as the safe fallback. --- .../get_chat_model/start/_20_browser_model.py | 14 ++++ .../python/monologue_end/_20_browser_model.py | 8 +++ plugins/_browser/helpers/config.py | 29 ++++++++- plugins/_browser/tools/browser.py | 5 ++ tests/test_browser_agent_regressions.py | 64 ++++++++++++++++++- 5 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 plugins/_browser/extensions/python/_functions/agent/Agent/get_chat_model/start/_20_browser_model.py create mode 100644 plugins/_browser/extensions/python/monologue_end/_20_browser_model.py diff --git a/plugins/_browser/extensions/python/_functions/agent/Agent/get_chat_model/start/_20_browser_model.py b/plugins/_browser/extensions/python/_functions/agent/Agent/get_chat_model/start/_20_browser_model.py new file mode 100644 index 000000000..de17c4680 --- /dev/null +++ b/plugins/_browser/extensions/python/_functions/agent/Agent/get_chat_model/start/_20_browser_model.py @@ -0,0 +1,14 @@ +from helpers.extension import Extension +from plugins._browser.helpers.config import ( + browser_model_is_active, + resolve_browser_model, +) + + +class BrowserModelProvider(Extension): + def execute(self, data: dict = {}, **kwargs): + if self.agent and browser_model_is_active(self.agent): + data["result"] = resolve_browser_model( + self.agent, + fallback=data.get("result"), + ) diff --git a/plugins/_browser/extensions/python/monologue_end/_20_browser_model.py b/plugins/_browser/extensions/python/monologue_end/_20_browser_model.py new file mode 100644 index 000000000..3a0b42b18 --- /dev/null +++ b/plugins/_browser/extensions/python/monologue_end/_20_browser_model.py @@ -0,0 +1,8 @@ +from helpers.extension import Extension +from plugins._browser.helpers.config import clear_browser_model + + +class BrowserModelCleanup(Extension): + def execute(self, **kwargs): + if self.agent: + clear_browser_model(self.agent) diff --git a/plugins/_browser/helpers/config.py b/plugins/_browser/helpers/config.py index 589eeeb31..ea4de4e4b 100644 --- a/plugins/_browser/helpers/config.py +++ b/plugins/_browser/helpers/config.py @@ -9,6 +9,7 @@ if TYPE_CHECKING: PLUGIN_NAME = "_browser" MODEL_PRESET_KEY = "model_preset" +BROWSER_MODEL_ACTIVE_KEY = "_browser_model_active" DEFAULT_HOMEPAGE_KEY = "default_homepage" AUTOFOCUS_ACTIVE_PAGE_KEY = "autofocus_active_page" TAB_SCOPE_KEY = "browser_tab_scope" @@ -319,10 +320,34 @@ def resolve_browser_model_selection( } -def resolve_browser_model(agent: "Agent", settings: dict[str, Any] | None = None): +def activate_browser_model(agent: "Agent") -> dict[str, Any]: + selection = resolve_browser_model_selection(agent=agent) + agent.set_data( + BROWSER_MODEL_ACTIVE_KEY, + selection["selected_preset_name"] + if selection["source_kind"] == "preset" + else "", + ) + return selection + + +def clear_browser_model(agent: "Agent") -> None: + agent.set_data(BROWSER_MODEL_ACTIVE_KEY, "") + + +def browser_model_is_active(agent: "Agent") -> bool: + return bool(agent.get_data(BROWSER_MODEL_ACTIVE_KEY)) + + +def resolve_browser_model( + agent: "Agent", + settings: dict[str, Any] | None = None, + fallback: Any = None, +): selection = resolve_browser_model_selection(agent=agent, settings=settings) if selection["source_kind"] == "main": - return agent.get_chat_model() + clear_browser_model(agent) + return fallback if fallback is not None else agent.get_chat_model() import models from plugins._model_config.helpers import model_config diff --git a/plugins/_browser/tools/browser.py b/plugins/_browser/tools/browser.py index 10220f028..94773a7a2 100644 --- a/plugins/_browser/tools/browser.py +++ b/plugins/_browser/tools/browser.py @@ -10,6 +10,7 @@ from typing import Any from helpers import files from helpers.print_style import PrintStyle from helpers.tool import Response, Tool +from plugins._browser.helpers.config import activate_browser_model from plugins._browser.helpers.selector import get_tool_runtime @@ -74,6 +75,10 @@ class Browser(Tool): action = "clipboard" else: action = str(action or self.method or "state").strip().lower().replace("-", "_") + try: + activate_browser_model(self.agent) + except Exception as exc: + PrintStyle.warning(f"Browser model preset could not be activated: {exc}") try: runtime = await get_runtime(self.agent.context.id, agent=self.agent) except Exception as exc: diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.py index edf4cd374..c2f1b844a 100644 --- a/tests/test_browser_agent_regressions.py +++ b/tests/test_browser_agent_regressions.py @@ -22,6 +22,19 @@ class _TestAgentContext: return None +class _TestAgent: + pass + + +class _TestAgentConfig: + pass + + +class _TestAgentContextType: + BACKGROUND = "background" + USER = SimpleNamespace(value="user") + + class _TestResponse(SimpleNamespace): def __init__(self, message="", break_loop=False, **kwargs): super().__init__(message=message, break_loop=break_loop, **kwargs) @@ -67,7 +80,15 @@ class _TestWsResult(dict): ) -sys.modules.setdefault("agent", SimpleNamespace(AgentContext=_TestAgentContext)) +sys.modules.setdefault( + "agent", + SimpleNamespace( + Agent=_TestAgent, + AgentConfig=_TestAgentConfig, + AgentContext=_TestAgentContext, + AgentContextType=_TestAgentContextType, + ), +) sys.modules.setdefault("helpers.tool", SimpleNamespace(Response=_TestResponse, Tool=_TestTool)) sys.modules.setdefault("helpers.ws", SimpleNamespace(WsHandler=_TestWsHandler)) sys.modules.setdefault("helpers.ws_manager", SimpleNamespace(WsResult=_TestWsResult)) @@ -295,6 +316,47 @@ def test_browser_model_selection_falls_back_to_main_for_missing_preset(monkeypat assert selection["config"] == {"provider": "openrouter", "name": "main/model"} +def test_browser_model_preset_controls_followup_turn_and_clears(monkeypatch): + import importlib + import plugins._browser.helpers.config as browser_config_module + + state = {} + agent = SimpleNamespace( + set_data=lambda key, value: state.__setitem__(key, value), + get_data=lambda key: state.get(key), + ) + monkeypatch.setattr( + browser_config_module, + "resolve_browser_model_selection", + lambda agent=None, settings=None: { + "source_kind": "preset", + "selected_preset_name": "Browser", + }, + ) + + browser_config_module.activate_browser_model(agent) + assert browser_config_module.browser_model_is_active(agent) + + provider_module = importlib.import_module( + "plugins._browser.extensions.python._functions.agent.Agent.get_chat_model.start._20_browser_model" + ) + cleanup_module = importlib.import_module( + "plugins._browser.extensions.python.monologue_end._20_browser_model" + ) + selected_model = object() + monkeypatch.setattr( + provider_module, + "resolve_browser_model", + lambda agent, fallback=None: selected_model, + ) + model_data = {"result": object()} + provider_module.BrowserModelProvider(agent).execute(data=model_data) + assert model_data["result"] is selected_model + + cleanup_module.BrowserModelCleanup(agent).execute() + assert not browser_config_module.browser_model_is_active(agent) + + def test_browser_model_preset_options_include_missing_selected(monkeypatch): from plugins._model_config.helpers import model_config From 4bf92a607265e8a235a8763bb421c6e528cc11e0 Mon Sep 17 00:00:00 2001 From: Alessandro <155005371+3clyp50@users.noreply.github.com> Date: Sun, 16 Aug 2026 15:22:10 +0200 Subject: [PATCH 05/20] Improve Browser startup and annotations Keep tab loading feedback stable and visible while the interactive Browser starts without waiting for a redundant screenshot. Add DOM hover feedback, multi-page annotation batches, compact composer-style controls, and shared Whisper draft or send behavior. --- plugins/_browser/AGENTS.md | 7 +- plugins/_browser/api/ws_browser.py | 9 +- plugins/_browser/webui/browser-panel.html | 208 ++++++++++++++-- plugins/_browser/webui/browser-store.js | 227 ++++++++++++++---- plugins/_browser/webui/config.html | 4 +- plugins/_whisper_stt/AGENTS.md | 2 + .../chat-input-box-end/microphone-button.html | 1 + .../_whisper_stt/webui/whisper-stt-store.js | 53 ++-- tests/test_browser_agent_regressions.py | 87 ++++++- tests/test_speech_plugin_split.py | 3 +- 10 files changed, 505 insertions(+), 96 deletions(-) diff --git a/plugins/_browser/AGENTS.md b/plugins/_browser/AGENTS.md index 973ec15ca..71400365c 100644 --- a/plugins/_browser/AGENTS.md +++ b/plugins/_browser/AGENTS.md @@ -19,6 +19,7 @@ - Preserve Patchright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding. - Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation. - Default the visible WebUI Browser to the authenticated Xpra HTML5 viewer for its existing Patchright page. Keep live CDP screencast and lightweight snapshots as automatic fallbacks. +- Do not block an available interactive viewer on a redundant Chromium screenshot; capture initial snapshots only for fallback transports. - Keep headful Chromium in a normal window with its own toolbar clipped above the private display; do not use browser fullscreen, which shows Chromium's exit warning. - Throttle interactive resize updates throughout a drag and let the native-sized Chromium viewport follow the private display; do not defer all layout updates until resizing stops. - Keep exactly one interactive viewer iframe connected during canvas/modal handoff so hidden surfaces cannot compete to resize the same display. @@ -30,6 +31,8 @@ - Push internal screencast frames from the runtime to the WebSocket consumer after subscription; keep `read/pop_screencast_frame` as fallback/tooling APIs, not the WebUI hot path. - Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback. Do not let the WebUI advertise binary frames unless its Socket.IO client reconstructs attachments as real `Blob`, `ArrayBuffer`, or typed-array values. - Keep WebUI Browser tabs scoped to the active chat context by default; aggregate tabs from other AgentContext runtimes only when the Browser settings tab scope is `shared`. +- Keep Chromium processes and persistent sign-in profiles isolated per chat even when the tab strip is shared; reset/removal may delete only that chat's profile. +- Show an accessible in-panel startup state while an on-demand Browser runtime is cold-starting; do not create an idle Chromium/Xpra pair for every chat at Agent Zero startup. - Keep narrow WebUI Browser controls usable by grouping navigation with Annotate/settings above a full-width address bar. - For Bring Your Own Browser with an existing host profile, `host_browser_selection` may target automatic CLI selection, a browser family/id, an HTTP CDP discovery address, or a full DevTools WebSocket endpoint and must be forwarded to the connector runtime as `browser_selection`. - Browser Settings must refresh connected A0 CLI host-browser inventory while the settings view is open so newly authorized endpoints appear without saving or reopening. @@ -37,7 +40,9 @@ - Browser URL-intent handling must only claim web URL schemes and leave custom Agent Zero schemes to their owning surfaces. - Prefer DOM/CDP browser actions with refs, selectors, frame-chain refs, and screenshots over viewport coordinate input. Coordinates remain a visual fallback. - Do not hardcode user-specific browser paths or secrets. -- Browser model-preset selection resolves omitted preset fields from `_model_config`'s global `Default` preset, not from an unrelated currently scoped model selection. +- Browser model-preset selection resolves omitted preset fields from `_model_config`'s global `Default` preset, not from an unrelated currently scoped model selection. After the first Browser tool call, use the selected preset for subsequent model turns in that monologue and clear it at monologue end. +- Annotation mode highlights the DOM element under the pointer, keeps saved overlays page-local, and may batch annotated pages only within the active chat context. +- Annotation voice input reuses Whisper STT's configured draft/send delivery mode and shared microphone state. - Internal-browser proxy settings map directly to Playwright's persistent-context proxy option, never to Bring Your Own Browser, and changes must restart active internal runtimes. - Run internal Chromium headful through Patchright on the private virtual display; do not add user-agent or header spoofing on top of the patched driver. - Browser startup and on-demand launch must converge on the Chromium revision declared by Patchright; let its installer select the host architecture rather than hardcoding x64 or ARM downloads. diff --git a/plugins/_browser/api/ws_browser.py b/plugins/_browser/api/ws_browser.py index 4c5b734e5..837bba129 100644 --- a/plugins/_browser/api/ws_browser.py +++ b/plugins/_browser/api/ws_browser.py @@ -143,7 +143,8 @@ class WsBrowser(WsHandler): viewer_transport=viewer_transport, ) self._streams[stream_key] = asyncio.create_task(stream_task) - snapshot = await self._snapshot_for_browser(runtime, active_id) + if viewer_transport != VIEWER_TRANSPORT_INTERACTIVE: + snapshot = await self._snapshot_for_browser(runtime, active_id) browsers, all_browsers, tab_scope = await self._tabs_for_scope(context_id, browsers) @@ -291,7 +292,11 @@ class WsBrowser(WsHandler): active_id, data, ) - snapshot = await self._snapshot_for_result(runtime, result) + snapshot = ( + None + if viewer_transport == VIEWER_TRANSPORT_INTERACTIVE + else await self._snapshot_for_result(runtime, result) + ) browsers, all_browsers, tab_scope = await self._tabs_for_scope( context_id, listing.get("browsers") or [], diff --git a/plugins/_browser/webui/browser-panel.html b/plugins/_browser/webui/browser-panel.html index 96e8bdfe7..673d57ec5 100644 --- a/plugins/_browser/webui/browser-panel.html +++ b/plugins/_browser/webui/browser-panel.html @@ -25,9 +25,10 @@ @click="$store.browserPage.selectBrowser(browser.id, browser.context_id)"> - +
-