diff --git a/bun.lock b/bun.lock index bf45febf740..49247d47cca 100644 --- a/bun.lock +++ b/bun.lock @@ -61,6 +61,7 @@ "@dnd-kit/solid": "0.5.0", "@ibm/plex": "6.4.1", "@kobalte/core": "catalog:", + "@opencode-ai/browser": "workspace:*", "@opencode-ai/client": "workspace:*", "@opencode-ai/schema": "workspace:*", "@opencode-ai/session-ui": "workspace:*", @@ -423,7 +424,9 @@ "@effect/platform-node": "catalog:", "@lydell/node-pty": "catalog:", "@opencode-ai/app": "workspace:*", + "@opencode-ai/browser": "workspace:*", "@opencode-ai/client": "workspace:*", + "@opencode-ai/schema": "workspace:*", "@opencode-ai/ui": "workspace:*", "@sentry/solid": "catalog:", "@sentry/vite-plugin": "catalog:", diff --git a/packages/app/package.json b/packages/app/package.json index 1f729ea50eb..3cdb7e69358 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -56,6 +56,7 @@ "vite-plugin-solid": "2.11.14" }, "dependencies": { + "@opencode-ai/browser": "workspace:*", "@corvu/drawer": "catalog:", "@dnd-kit/abstract": "0.5.0", "@dnd-kit/dom": "0.5.0", diff --git a/packages/app/src/desktop.ts b/packages/app/src/desktop.ts index 180e075bc19..e88e923b993 100644 --- a/packages/app/src/desktop.ts +++ b/packages/app/src/desktop.ts @@ -4,6 +4,16 @@ export { useCommand } from "./shell/commands/command" export { currentRoute, type LayoutRoute, useCurrentRoute } from "./shell/state/layout" export { loadLocaleDict, normalizeLocale, type Locale, useLanguage } from "./runtime/i18n/language" export { type FatalRendererErrorLog, type Platform, PlatformProvider } from "./runtime/platform/platform" +export type { + BrowserPaneCommand, + BrowserPaneEndpoint, + BrowserPaneEvent, + BrowserPaneLayout, + BrowserPanePlatform, + BrowserPaneRegistration, + BrowserPaneState, + BrowserPaneTarget, +} from "./runtime/platform/browser-pane" export { ServerConnection, useServers } from "./runtime/server/registry" export { useTabs } from "./shell/tabs/tabs" export { createDraftStore } from "./runtime/persistence/drafts" diff --git a/packages/app/src/runtime/i18n/en.ts b/packages/app/src/runtime/i18n/en.ts index 42c823fc382..e7d18aa8d38 100644 --- a/packages/app/src/runtime/i18n/en.ts +++ b/packages/app/src/runtime/i18n/en.ts @@ -66,6 +66,7 @@ export const dict = { "command.terminal.toggle": "Toggle terminal", "command.fileTree.toggle": "Toggle file tree", "command.review.toggle": "Toggle review", + "command.browser.toggle": "Toggle browser", "command.terminal.new": "New terminal", "command.terminal.new.description": "Create a new terminal tab", "command.steps.toggle": "Toggle steps", @@ -810,6 +811,10 @@ export const dict = { "PTY connect ticket rejected by origin or CSRF checks. Check the server CORS config.", "terminal.connectTicket.statusError": "PTY connect ticket failed with {{status}}", + "session.browser.address": "Browser address", + "session.browser.address.placeholder": "Enter a URL", + "session.browser.close": "Close browser", + "titlebar.update": "Update", "titlebar.tabs": "Tabs", "titlebar.channel.local": "Local", @@ -1037,6 +1042,8 @@ export const dict = { "settings.general.row.uiFont.description": "Customise the font used throughout the interface", "settings.general.row.showFileTree.title": "File tree", "settings.general.row.showFileTree.description": "Show the file tree panel in sessions", + "settings.general.row.browserPane.title": "Browser pane", + "settings.general.row.browserPane.description": "Allow agents to open and control an in-app development browser.", "settings.general.row.showNavigation.title": "Navigation controls", "settings.general.row.showNavigation.description": "Show the back and forward buttons in the desktop title bar", "settings.general.row.showSearch.title": "Command palette", @@ -1226,6 +1233,9 @@ export const dict = { "settings.permissions.tool.webfetch.description": "Fetch content from a URL", "settings.permissions.tool.websearch.title": "Web Search", "settings.permissions.tool.websearch.description": "Search the web", + "settings.permissions.tool.browser_read.description": "Read pages and capture screenshots in the browser", + "settings.permissions.tool.browser_navigate.description": "Navigate the browser to a URL", + "settings.permissions.tool.browser_interact.description": "Click, type, and interact with pages in the browser", "settings.permissions.tool.external_directory.title": "External Directory", "settings.permissions.tool.external_directory.description": "Access files outside the project directory", "settings.permissions.tool.doom_loop.title": "Doom Loop", diff --git a/packages/app/src/runtime/platform/browser-pane.ts b/packages/app/src/runtime/platform/browser-pane.ts new file mode 100644 index 00000000000..073ce6fb20a --- /dev/null +++ b/packages/app/src/runtime/platform/browser-pane.ts @@ -0,0 +1,19 @@ +import type { Browser } from "@opencode-ai/browser/rpc" + +export type BrowserPaneEndpoint = Readonly<{ url: string; username?: string; password?: string }> +export type BrowserPaneTarget = Readonly<{ sessionID: string; endpoint: BrowserPaneEndpoint }> +export type BrowserPaneLayout = { visible: boolean; bounds?: { x: number; y: number; width: number; height: number } } + +export type BrowserPaneCommand = Browser.Action +export type BrowserPaneState = Browser.State | null +export type BrowserPaneEvent = { type: "open" } | { type: "state"; state: BrowserPaneState; error?: string } + +export type BrowserPaneRegistration = { + setLayout(layout?: BrowserPaneLayout): void + command(command: BrowserPaneCommand): Promise + close(): void +} + +export type BrowserPanePlatform = { + register(target: BrowserPaneTarget, listener: (event: BrowserPaneEvent) => void): BrowserPaneRegistration +} diff --git a/packages/app/src/runtime/platform/platform.tsx b/packages/app/src/runtime/platform/platform.tsx index 4c3572c12f4..2dadd469cf2 100644 --- a/packages/app/src/runtime/platform/platform.tsx +++ b/packages/app/src/runtime/platform/platform.tsx @@ -6,6 +6,7 @@ import { ServerConnection } from "@/runtime/server/registry" import type { WslServersPlatform } from "@/servers/wsl/types" import type { UpdaterPlatform } from "@/shell/updates/types" import type { DraftStore } from "@/runtime/persistence/drafts" +import type { BrowserPanePlatform } from "./browser-pane" type PickerPaths = string | string[] | null type OpenDirectoryPickerOptions = { title?: string; multiple?: boolean } @@ -115,6 +116,9 @@ type PlatformBase = { /** Record a fatal renderer error in platform logs (desktop only) */ recordFatalRendererError?(error: FatalRendererErrorLog): Promise + + /** Native browser pane hosted by the platform (desktop only). */ + browserPane?: BrowserPanePlatform } export type Platform = PlatformBase & diff --git a/packages/app/src/session/browser/model.ts b/packages/app/src/session/browser/model.ts new file mode 100644 index 00000000000..1e26ee1a8ae --- /dev/null +++ b/packages/app/src/session/browser/model.ts @@ -0,0 +1,87 @@ +import { createEffect, createMemo, onCleanup } from "solid-js" +import { createStore } from "solid-js/store" +import { useLanguage } from "@/runtime/i18n/language" +import type { BrowserPaneCommand, BrowserPaneRegistration, BrowserPaneState } from "@/runtime/platform/browser-pane" +import { usePlatform } from "@/runtime/platform/platform" +import { useServer } from "@/runtime/server/current" +import { useSettings } from "@/settings/model" +import { useLayout } from "@/shell/state/layout" +import type { SessionModel } from "../model" + +export function createSessionBrowser(session: SessionModel) { + const platform = usePlatform() + const settings = useSettings() + const language = useLanguage() + const server = useServer() + const layout = useLayout() + const [state, setState] = createStore({ + opened: false, + registration: undefined as BrowserPaneRegistration | undefined, + browser: null as BrowserPaneState, + error: undefined as string | undefined, + }) + const available = createMemo( + () => + !!platform.browserPane && + settings.ready() && + settings.general.experimentalBrowser() && + session.isDesktop() && + !!session.identity.sessionID() && + !server.health?.incompatible, + ) + const open = () => { + session.layout.view().reviewPanel.close() + layout.fileTree.close() + setState("opened", true) + } + + createEffect(() => { + const sessionID = session.identity.sessionID() + const pane = platform.browserPane + setState({ opened: false, registration: undefined, browser: null, error: undefined }) + if (!available() || !sessionID || !pane) return + const owner = session.ownership.capture() + const target = { sessionID, endpoint: server.conn.http } + let registration: BrowserPaneRegistration | undefined + const register = () => { + if (registration) return + registration = pane.register(target, (event) => + owner.run(() => (event.type === "open" ? open() : setState({ browser: event.state, error: event.error }))), + ) + setState({ registration, browser: null, error: undefined }) + } + // A new session appears in the UI before its server-side creation finishes. + const unsubscribe = session.shared.data.on("session.created", (event) => { + if (event.data.sessionID === sessionID) register() + }) + if (!session.shared.data.session.creating(sessionID)) register() + onCleanup(() => { + unsubscribe() + registration?.close() + }) + }) + + createEffect(() => { + if (state.opened && (session.layout.view().reviewPanel.opened() || layout.fileTree.opened())) { + setState("opened", false) + } + }) + + return { + available, + opened: () => state.opened, + state: () => state.browser, + error: () => state.error, + registration: () => (state.opened ? state.registration : undefined), + close: () => setState("opened", false), + toggle: () => (state.opened ? setState("opened", false) : open()), + command(command: BrowserPaneCommand) { + setState("error", undefined) + const owner = session.ownership.capture() + void state.registration?.command(command).catch((error: unknown) => { + if (!owner.current()) return + setState("error", error instanceof Error ? error.message : language.t("common.requestFailed")) + }) + }, + } +} diff --git a/packages/app/src/session/browser/pane.tsx b/packages/app/src/session/browser/pane.tsx new file mode 100644 index 00000000000..db19a4832f4 --- /dev/null +++ b/packages/app/src/session/browser/pane.tsx @@ -0,0 +1,129 @@ +import { Icon } from "@opencode-ai/ui/icon" +import { IconButton } from "@opencode-ai/ui/icon-button" +import { Loader } from "@opencode-ai/ui/loader" +import { useDialog } from "@opencode-ai/ui/context/dialog" +import { createEventListener } from "@solid-primitives/event-listener" +import { createResizeObserver } from "@solid-primitives/resize-observer" +import { createEffect, For, on, onCleanup, Show } from "solid-js" +import { createStore } from "solid-js/store" +import { useLanguage } from "@/runtime/i18n/language" +import type { BrowserPaneRegistration } from "@/runtime/platform/browser-pane" +import { usePlatform } from "@/runtime/platform/platform" +import type { createSessionBrowser } from "./model" + +export function SessionBrowserPane(props: { + registration: BrowserPaneRegistration + browser: ReturnType +}) { + const platform = usePlatform() + const language = useLanguage() + const dialog = useDialog() + const state = props.browser.state + const button = { variant: "ghost", size: "large" } as const + const [store, setStore] = createStore({ + address: "", + editing: false, + visible: typeof document === "undefined" || document.visibilityState === "visible", + }) + let surface: HTMLDivElement | undefined + let frame: number | undefined + let layout: string | undefined + let until = 0 + + const measure = () => { + frame = undefined + if (!surface) return + const rect = surface.getBoundingClientRect() + const zoom = platform.webviewZoom?.() ?? 1 + const left = Math.round(rect.left * zoom) + const top = Math.round(rect.top * zoom) + const right = Math.round(rect.right * zoom) + const bottom = Math.round(rect.bottom * zoom) + const visible = store.visible && !dialog.active + const next = `${visible}:${left}:${top}:${right}:${bottom}` + if (next !== layout) { + layout = next + props.registration.setLayout({ + visible, + bounds: { x: left, y: top, width: Math.max(0, right - left), height: Math.max(0, bottom - top) }, + }) + } + if (performance.now() < until) frame = requestAnimationFrame(measure) + } + const schedule = (duration = 0) => { + until = Math.max(until, performance.now() + duration) + if (frame === undefined) frame = requestAnimationFrame(measure) + } + + createEffect(() => !store.editing && setStore("address", state()?.url ?? "")) + createEffect(on([() => platform.webviewZoom?.(), () => dialog.active, () => store.visible], () => schedule(300))) + createResizeObserver(() => surface, schedule.bind(null, 0)) + createEventListener(window, "resize", () => schedule(300)) + createEventListener(document, "visibilitychange", () => setStore("visible", document.visibilityState === "visible")) + onCleanup(() => { + if (frame !== undefined) cancelAnimationFrame(frame) + props.registration.setLayout() + }) + + return ( + + ) +} diff --git a/packages/app/src/session/header/session-header-actions.tsx b/packages/app/src/session/header/session-header-actions.tsx index 9572f1a3fb3..90a6c8d781c 100644 --- a/packages/app/src/session/header/session-header-actions.tsx +++ b/packages/app/src/session/header/session-header-actions.tsx @@ -32,6 +32,7 @@ export type SessionHeaderActionsState = { reviewVisible: boolean reviewOpened: boolean onReviewToggle: () => void + browser?: { label: string; opened: boolean; onToggle: () => void } } export function SessionHeaderActions(props: { state: SessionHeaderActionsState }) { @@ -64,6 +65,24 @@ export function SessionHeaderActions(props: { state: SessionHeaderActionsState } /> + + {(browser) => ( + + } + /> + + )} + ) } diff --git a/packages/app/src/session/screen-layout.ts b/packages/app/src/session/screen-layout.ts index 570c0baffe9..0a6e27903d9 100644 --- a/packages/app/src/session/screen-layout.ts +++ b/packages/app/src/session/screen-layout.ts @@ -8,7 +8,7 @@ import type { SessionModel } from "./model" import { sessionPanelLayout } from "./session-panel-layout" import { clampSessionPanelWidth, sessionPanelWidthMax } from "./session-panel-width" -export function createSessionScreenLayout(session: SessionModel) { +export function createSessionScreenLayout(session: SessionModel, browserOpen: () => boolean) { const layout = useLayout() const settings = useSettings() const size = createSizing() @@ -27,7 +27,7 @@ export function createSessionScreenLayout(session: SessionModel) { opened: layout.fileTree.opened(), }), ) - const resizable = createMemo(() => reviewPanelOpen() || sideTerminalOpen()) + const resizable = createMemo(() => reviewPanelOpen() || browserOpen() || sideTerminalOpen()) const sidePanelOpen = createMemo(() => resizable() || fileTreeOpen()) const [rowSize, setRowSize] = createStore<{ width?: number; height?: number }>({}) let row: HTMLDivElement | undefined @@ -61,6 +61,7 @@ export function createSessionScreenLayout(session: SessionModel) { const panelLayout = createMemo(() => sessionPanelLayout({ review: reviewPanelOpen(), + browser: browserOpen(), terminal: sideTerminalOpen(), files: fileTreeOpen(), }), @@ -71,7 +72,7 @@ export function createSessionScreenLayout(session: SessionModel) { if (previous !== stacked) setMotion({ gap: stacked, closing: !stacked }) return stacked }, panelLayout().stacked) - const sideRegionOpen = createMemo(() => reviewPanelOpen() || fileTreeOpen()) + const sideRegionOpen = createMemo(() => reviewPanelOpen() || browserOpen() || fileTreeOpen()) const terminalPane = createMemo(() => Math.min(view().terminal.height(), typeof window === "undefined" ? 600 : window.innerHeight * 0.6), ) diff --git a/packages/app/src/session/screen.tsx b/packages/app/src/session/screen.tsx index 77991338ccb..81378825d32 100644 --- a/packages/app/src/session/screen.tsx +++ b/packages/app/src/session/screen.tsx @@ -30,6 +30,8 @@ import { ActiveSessionComposerRegion, createActiveSessionRegion } from "./compos import { SessionIdentityHeader } from "./session-identity-header" import { SessionReviewToggle } from "./header/session-header-actions" import { createAnimatedPresence } from "@/runtime/animated-presence" +import { createSessionBrowser } from "./browser/model" +import { SessionBrowserPane } from "./browser/pane" const SessionMobileFiles = lazy(async () => { const { SessionMobileFiles } = await import("./files/session-mobile-files") @@ -44,7 +46,8 @@ export function SessionScreen(props: { session: SessionModel }) { return info ? projectForSession(info, server.ctx.sync.data.project) : undefined }) const isDesktop = session.isDesktop - const screen = createSessionScreenLayout(session) + const browser = createSessionBrowser(session) + const screen = createSessionScreenLayout(session, browser.opened) const timeline = createSessionTimelineInteraction(session) const messagesReady = timeline.ready const [store, setStore] = createStore({ @@ -364,7 +367,13 @@ export function SessionScreen(props: { session: SessionModel }) { setStore("sideReviewPresent", false) }} > - + } + > + {(registration) => } + diff --git a/packages/app/src/session/session-panel-layout.test.ts b/packages/app/src/session/session-panel-layout.test.ts index 73bd2d8d8c0..643bccdaf90 100644 --- a/packages/app/src/session/session-panel-layout.test.ts +++ b/packages/app/src/session/session-panel-layout.test.ts @@ -3,15 +3,23 @@ import { sessionPanelLayout } from "./session-panel-layout" describe("sessionPanelLayout", () => { test("keeps one owner while changing panel geometry", () => { - expect(sessionPanelLayout({ review: false, terminal: false, files: false })).toEqual({ + expect(sessionPanelLayout({ review: false, browser: false, terminal: false, files: false })).toEqual({ visible: false, stacked: false, }) - expect(sessionPanelLayout({ review: false, terminal: true, files: false })).toEqual({ + expect(sessionPanelLayout({ review: false, browser: false, terminal: true, files: false })).toEqual({ visible: true, stacked: false, }) - expect(sessionPanelLayout({ review: true, terminal: true, files: false })).toEqual({ + expect(sessionPanelLayout({ review: true, browser: false, terminal: true, files: false })).toEqual({ + visible: true, + stacked: true, + }) + expect(sessionPanelLayout({ review: false, browser: true, terminal: false, files: false })).toEqual({ + visible: true, + stacked: false, + }) + expect(sessionPanelLayout({ review: false, browser: true, terminal: true, files: false })).toEqual({ visible: true, stacked: true, }) diff --git a/packages/app/src/session/session-panel-layout.ts b/packages/app/src/session/session-panel-layout.ts index 49feaebedee..7c5e22f8ff8 100644 --- a/packages/app/src/session/session-panel-layout.ts +++ b/packages/app/src/session/session-panel-layout.ts @@ -1,6 +1,6 @@ -export function sessionPanelLayout(input: { review: boolean; terminal: boolean; files: boolean }) { +export function sessionPanelLayout(input: { review: boolean; browser: boolean; terminal: boolean; files: boolean }) { return { - visible: input.review || input.terminal || input.files, - stacked: input.review && input.terminal, + visible: input.review || input.browser || input.terminal || input.files, + stacked: (input.review || input.browser) && input.terminal, } } diff --git a/packages/app/src/settings/general/general.tsx b/packages/app/src/settings/general/general.tsx index e291e65c839..fd11dbbafa6 100644 --- a/packages/app/src/settings/general/general.tsx +++ b/packages/app/src/settings/general/general.tsx @@ -384,6 +384,20 @@ export const SettingsGeneral: Component<{

{language.t("settings.general.section.advanced")}

+ + +
+ settings.general.setExperimentalBrowser(checked)} + /> +
+
+
+ store.general?.experimentalBrowser, + defaultSettings.general.experimentalBrowser, + ), + setExperimentalBrowser(value: boolean) { + setStore("general", "experimentalBrowser", value) + }, }, visibility: { fileTree: showFileTree, diff --git a/packages/desktop/package.json b/packages/desktop/package.json index fbf29f906a2..d25bf9b3f3b 100644 --- a/packages/desktop/package.json +++ b/packages/desktop/package.json @@ -35,7 +35,9 @@ "@effect/platform-node": "catalog:", "@lydell/node-pty": "catalog:", "@opencode-ai/app": "workspace:*", + "@opencode-ai/browser": "workspace:*", "@opencode-ai/client": "workspace:*", + "@opencode-ai/schema": "workspace:*", "@opencode-ai/ui": "workspace:*", "@sentry/solid": "catalog:", "@sentry/vite-plugin": "catalog:", diff --git a/packages/desktop/src/main/browser-chromium.ts b/packages/desktop/src/main/browser-chromium.ts new file mode 100644 index 00000000000..0fb6d14979d --- /dev/null +++ b/packages/desktop/src/main/browser-chromium.ts @@ -0,0 +1,284 @@ +import type { Browser } from "@opencode-ai/browser/rpc" +import electron, { type BrowserWindow } from "electron" + +type AXNode = { + nodeId: string + backendDOMNodeId?: number + childIds?: string[] + ignored?: boolean + role?: { value?: string } + name?: { value?: unknown } + properties?: Array<{ name: string; value?: { value?: unknown } }> +} + +export type BrowserPage = ReturnType + +export function createBrowserPage(win: BrowserWindow, publish: (error?: string) => void, fail: () => void) { + const view = new electron.WebContentsView({ + webPreferences: { + partition: `opencode-browser-${crypto.randomUUID()}`, + nodeIntegration: false, + contextIsolation: true, + sandbox: true, + webSecurity: true, + webviewTag: false, + devTools: false, + disableDialogs: true, + }, + }) + const contents = view.webContents + const refs = new Map() + let generation = 0 + let nextRef = 0 + let closed = false + const state = (): Browser.State => ({ + url: contents.getURL().slice(0, 16_384), + title: contents.getTitle().slice(0, 1_024), + loading: contents.isLoading(), + canGoBack: contents.navigationHistory.canGoBack(), + canGoForward: contents.navigationHistory.canGoForward(), + generation, + }) + const update = () => { + if (!closed) publish() + } + contents.on("before-input-event", (event, input) => { + if (input.type !== "keyDown" || input.alt || !(process.platform === "darwin" ? input.meta : input.control)) return + const step = + input.key === "=" || input.key === "+" || input.code === "NumpadAdd" + ? 0.5 + : input.key === "-" || input.code === "NumpadSubtract" + ? -0.5 + : 0 + if (!step && input.key !== "0") return + event.preventDefault() + contents.setZoomLevel(input.key === "0" ? 0 : contents.getZoomLevel() + step) + }) + const session = contents.session + session.setPermissionRequestHandler((_contents, _permission, callback) => callback(false)) + session.setPermissionCheckHandler(() => false) + session.setDevicePermissionHandler(() => false) + session.setDisplayMediaRequestHandler((_request, callback) => callback({})) + session.on("will-download", (event) => event.preventDefault()) + contents.setWindowOpenHandler(() => ({ action: "deny" })) + contents.on("content-bounds-updated", (event) => event.preventDefault()) + const guard = (event: Electron.Event<{ url: string }>) => { + if (event.url === "about:blank" || destinationOrigin(event.url)) return + event.preventDefault() + publish("ERR_BLOCKED_BY_CLIENT") + } + contents.on("will-frame-navigate", guard) + contents.on("will-redirect", guard) + contents.on("did-stop-loading", update) + contents.on("did-navigate-in-page", update) + contents.on("page-title-updated", update) + contents.on("did-start-navigation", (event) => { + if (!event.isMainFrame) return + generation++ + refs.clear() + update() + }) + contents.on("did-fail-load", (_event, code, error, _url, mainFrame) => { + if (!closed && mainFrame && code !== -3) publish(error) + }) + contents.on("render-process-gone", () => { + if (!closed) fail() + }) + contents.debugger.on("detach", () => { + if (!closed) fail() + }) + view.setVisible(false) + win.contentView.addChildView(view) + return { + view, + state, + execute, + ready: Promise.resolve().then(() => contents.loadURL("about:blank")), + dispose() { + if (closed) return + closed = true + refs.clear() + if (!win.isDestroyed()) win.contentView.removeChildView(view) + if (!contents.isDestroyed()) contents.close({ waitForBeforeUnload: false }) + }, + } + + async function execute(command: Browser.Command, signal: AbortSignal): Promise { + const action = command.action + check() + if (action.type === "navigate") { + await navigate(action.url, signal) + return { type: "state", state: state() } + } + if (["open", "back", "forward", "reload", "stop"].includes(action.type)) { + if (action.type === "stop") contents.stop() + if (action.type === "reload") contents.reload() + if (action.type === "back" && contents.navigationHistory.canGoBack()) contents.navigationHistory.goBack() + if (action.type === "forward" && contents.navigationHistory.canGoForward()) contents.navigationHistory.goForward() + return { type: "state", state: state() } + } + if (action.type === "evaluate") { + const value: unknown = await contents.executeJavaScript(action.script) + check() + return { type: "evaluate", state: state(), content: (JSON.stringify(value) ?? "null").slice(0, 100_000) } + } + if (action.type === "snapshot") { + const tree = (await send("Accessibility.getFullAXTree", { depth: 6 })) as { nodes: AXNode[] } + refs.clear() + const nodes = new Map(tree.nodes.map((node) => [node.nodeId, node])) + const lines = [`Page: ${clean(state().title)}`, `URL: ${state().url}`, ""] + if (tree.nodes[0]) walk(tree.nodes[0], 0) + return { + type: "snapshot", + state: state(), + content: lines.join("\n").slice(0, 40_960), + } + + function walk(node: AXNode, depth: number) { + if (depth > 6 || lines.length >= 503) return + const role = (node.role?.value ?? "node").replace(/[^a-zA-Z0-9_-]/g, "").slice(0, 40) + const properties = new Map((node.properties ?? []).map((item) => [item.name, item.value?.value])) + const editable = + ["textbox", "searchbox", "combobox", "spinbutton"].includes(role) || !!properties.get("editable") + if (!node.ignored) { + const actionable = properties.get("focusable") || /^(button|link|textbox|combobox)$/.test(role) + const ref = actionable && node.backendDOMNodeId ? `e${++nextRef}` : "" + if (ref && node.backendDOMNodeId) + refs.set(ref, { + id: node.backendDOMNodeId, + editable: editable && !properties.get("disabled") && !properties.get("readonly"), + }) + const flags = ["checked", "disabled", "expanded", "selected"].flatMap((flag) => + properties.has(flag) ? [`${flag}=${properties.get(flag)}`] : [], + ) + lines.push( + `${" ".repeat(depth)}${ref ? `@${ref}` : ""} [${role}] ${JSON.stringify(clean(node.name?.value))} ${flags.join(" ")}`, + ) + } + // Editable descendants can repeat the field's value as static text. + if (!editable) + node.childIds?.forEach((id) => { + const child = nodes.get(id) + if (child) walk(child, depth + 1) + }) + } + } + if (action.type === "screenshot") { + const source = await contents.capturePage() + check() + const size = source.getSize() + if (!size.width || !size.height) throw new Error("internal") + const scale = Math.min(1, 2_000 / Math.max(size.width, size.height)) + const image = source.resize({ + width: Math.max(1, Math.round(size.width * scale)), + height: Math.max(1, Math.round(size.height * scale)), + }) + const data = new Uint8Array(image.toPNG()) + if (data.byteLength > 5 * 1_024 * 1_024) throw new Error("result_too_large") + return { type: "screenshot", state: state(), data } + } + if (action.type === "click" || action.type === "fill") { + const target = refs.get(action.ref.replace(/^@/, "")) + if (!target || (action.type === "fill" && !target.editable)) throw new Error("stale_ref") + if (action.type === "fill") { + await send("DOM.focus", { backendNodeId: target.id }) + await key({ key: "a", code: "KeyA", modifiers: process.platform === "darwin" ? 4 : 2 }) + await key({ key: "Backspace", code: "Backspace", windowsVirtualKeyCode: 8 }) + await send("Input.insertText", { text: action.text }) + } + if (action.type === "click") { + await send("DOM.scrollIntoViewIfNeeded", { backendNodeId: target.id }) + const result = (await send("DOM.getBoxModel", { backendNodeId: target.id })) as { + model: { content: number[] } + } + const box = result.model.content + const point = { x: (box[0] + box[4]) / 2, y: (box[1] + box[5]) / 2 } + for (const type of ["mouseMoved", "mousePressed", "mouseReleased"]) { + await send("Input.dispatchMouseEvent", { type, ...point, button: "left", clickCount: 1 }) + } + } + } + if (action.type === "press") { + const codes: Record = { + Enter: 13, + Tab: 9, + Escape: 27, + Backspace: 8, + Delete: 46, + ArrowUp: 38, + ArrowDown: 40, + ArrowLeft: 37, + ArrowRight: 39, + PageUp: 33, + PageDown: 34, + Home: 36, + End: 35, + Space: 32, + } + await key({ + key: action.key === "Space" ? " " : action.key, + code: action.key, + windowsVirtualKeyCode: codes[action.key], + }) + } + if (action.type === "scroll") { + const bounds = view.getBounds() + await send("Input.dispatchMouseEvent", { + type: "mouseWheel", + x: bounds.width / 2, + y: bounds.height / 2, + deltaX: action.direction === "left" ? -action.pixels : action.direction === "right" ? action.pixels : 0, + deltaY: action.direction === "up" ? -action.pixels : action.direction === "down" ? action.pixels : 0, + }) + } + check() + return { type: "state", state: state() } + + function check() { + if (closed) throw new Error("not_attached") + if (signal.aborted) throw new Error("aborted") + // Opening may create a new document before the command runs. + if (action.type !== "open" && generation !== command.generation) throw new Error("stale_ref") + } + + function key(params: Record) { + return send("Input.dispatchKeyEvent", { type: "keyDown", ...params }).finally(() => + contents.debugger.sendCommand("Input.dispatchKeyEvent", { type: "keyUp", ...params }), + ) + } + + async function send(method: string, params: Record): Promise { + check() + if (!contents.debugger.isAttached()) contents.debugger.attach("1.3") + const result: unknown = await contents.debugger.sendCommand(method, params) + check() + return result + } + } + + async function navigate(input: string, signal: AbortSignal) { + const value = input.trim() || "about:blank" + const local = /^(?:localhost|127(?:\.\d{1,3}){3}|\[::1\])(?::\d+)?(?:[/?#]|$)/i.test(value) + const url = + value === "about:blank" || /^[a-z][a-z\d+.-]*:\/\//i.test(value) + ? value + : `${local ? "http" : "https"}://${value}` + if (url.length > 16_384 || (url !== "about:blank" && !destinationOrigin(url))) throw new Error("invalid_url") + const cancel = () => { + if (!closed) contents.stop() + } + signal.addEventListener("abort", cancel, { once: true }) + await contents.loadURL(url).finally(() => signal.removeEventListener("abort", cancel)) + if (signal.aborted) throw new Error("aborted") + } +} + +function clean(value: unknown) { + return typeof value === "string" ? value.replaceAll(/\s+/g, " ").trim().slice(0, 300) : "" +} + +export function destinationOrigin(input: string) { + if (!URL.canParse(input)) return + const url = new URL(input) + return /^https?:$/.test(url.protocol) && !url.username && !url.password ? url.origin : undefined +} diff --git a/packages/desktop/src/main/browser-pane-policy.test.ts b/packages/desktop/src/main/browser-pane-policy.test.ts new file mode 100644 index 00000000000..7c30fac86ff --- /dev/null +++ b/packages/desktop/src/main/browser-pane-policy.test.ts @@ -0,0 +1,15 @@ +import { expect, test } from "bun:test" +import { destinationOrigin } from "./browser-chromium" + +test("allows cross-origin HTTP navigation but rejects unsafe destinations and embedded credentials", () => { + expect(destinationOrigin("https://other.example/path")).toBe("https://other.example") + expect(destinationOrigin("http://localhost:3000/")).toBe("http://localhost:3000") + for (const url of [ + "file:///etc/passwd", + "javascript:alert(1)", + "data:text/html,test", + "https://user:pass@example.com", + ]) { + expect(destinationOrigin(url)).toBeUndefined() + } +}) diff --git a/packages/desktop/src/main/browser-pane.ts b/packages/desktop/src/main/browser-pane.ts new file mode 100644 index 00000000000..3c4f95de86b --- /dev/null +++ b/packages/desktop/src/main/browser-pane.ts @@ -0,0 +1,233 @@ +import type { BrowserPaneCommand, BrowserPaneLayout, BrowserPaneTarget } from "@opencode-ai/app/desktop" +import { NodeHttpClient } from "@effect/platform-node" +import { Browser } from "@opencode-ai/browser/rpc" +import { OpenCode } from "@opencode-ai/client/effect" +import { SessionID } from "@opencode-ai/schema/session-id" +import type { BrowserWindow } from "electron" +import { Deferred, Effect, ManagedRuntime, Queue, Schema, Stream } from "effect" +import { HttpClient, HttpClientRequest } from "effect/unstable/http" +import { BrowserPaneEvent } from "../shared/ipc-rpc/events" +import { createBrowserPage, destinationOrigin, type BrowserPage } from "./browser-chromium" +import { emitIpcEvent } from "./ipc-events" + +type Entry = { + bindingID: string + win: BrowserWindow + abort: AbortController + registered: PromiseWithResolvers + requests: Map + report?: (state: Browser.State | null) => void + cleanup?: () => void + page?: BrowserPage +} + +export function createBrowserPane() { + const entries = new Map() + // Keep long-lived RPC requests off Chromium's shared HTTP connection pool. + const runtime = ManagedRuntime.make(NodeHttpClient.layerNodeHttp) + let disposed = false + return { + async register(win: BrowserWindow, bindingID: string, target: BrowserPaneTarget) { + if (disposed || !destinationOrigin(target.endpoint.url)) throw new Error("browser.pane.registration.invalid") + if (target.endpoint.username && !target.endpoint.password) throw new Error("browser.pane.endpoint.invalid") + if (entries.has(bindingID)) throw new Error("browser.pane.owner.invalid") + if (win.isDestroyed() || win.webContents.isDestroyed()) throw new Error("browser.pane.owner.unavailable") + const sessionID = SessionID.make(target.sessionID) + const entry: Entry = { + bindingID, + win, + abort: new AbortController(), + registered: Promise.withResolvers(), + requests: new Map(), + } + const stop = () => close(entry) + const navigate = (event: Electron.Event<{ isMainFrame: boolean; isSameDocument: boolean }>) => { + if (event.isMainFrame && !event.isSameDocument) stop() + } + win.webContents.once("destroyed", stop) + win.webContents.on("did-start-navigation", navigate) + entry.cleanup = () => { + win.webContents.off("destroyed", stop) + win.webContents.off("did-start-navigation", navigate) + } + entries.set(bindingID, entry) + void runtime + .runPromise( + Effect.gen(function* () { + const http = yield* HttpClient.HttpClient + const client = yield* OpenCode.make({ baseUrl: target.endpoint.url }).pipe( + Effect.provideService( + HttpClient.HttpClient, + target.endpoint.password + ? HttpClient.mapRequest( + http, + HttpClientRequest.basicAuth(target.endpoint.username ?? "opencode", target.endpoint.password), + ) + : http, + ), + ) + const session = yield* client.session.get({ sessionID }) + const options = { + location: { directory: session.location.directory, workspace: session.location.workspaceID }, + } + const attachment = { sessionID, connectionID: crypto.randomUUID() } + const rpc = client.rpc(Browser.Definition) + const connected = yield* Deferred.make() + const outbound = yield* Queue.unbounded>() + // Preserve the state-before-result ordering of the original control channel. + entry.report = (state) => { + Queue.offerUnsafe(outbound, rpc.state({ ...attachment, state }, options)) + } + const receive = client.event.subscribe().pipe( + Stream.runForEach((event) => + Effect.gen(function* () { + if (event.type === "server.connected") { + yield* Deferred.succeed(connected, undefined) + return + } + if (event.type !== "rpc.experimental.browser.control") return + const message = yield* Schema.decodeUnknownEffect(Browser.Control)(event.data) + if (message.connectionID !== attachment.connectionID) return + if (message.type === "attached") return entry.registered.resolve() + if (message.type === "cancel") return entry.requests.get(message.requestID)?.abort() + const abort = new AbortController() + entry.requests.set(message.requestID, abort) + yield* Effect.promise(async () => { + const outcome: Browser.Outcome = await execute(entry, message.command, abort.signal).then( + (result) => ({ type: "success" as const, result }), + (error: unknown) => ({ + type: "failure" as const, + message: (error instanceof Error ? error.message : String(error)).slice(0, 1_024), + }), + ) + Queue.offerUnsafe( + outbound, + rpc.result( + { + ...attachment, + requestID: message.requestID, + outcome: Schema.encodeSync(Browser.Outcome)(outcome), + }, + options, + ), + ) + }).pipe( + Effect.ensuring( + Effect.sync(() => { + abort.abort() + entry.requests.delete(message.requestID) + }), + ), + Effect.catchCause(() => Effect.sync(stop)), + Effect.forkScoped, + ) + }), + ), + ) + yield* Effect.raceAllFirst([ + receive, + Stream.fromQueue(outbound).pipe(Stream.runForEach((send) => send)), + Deferred.await(connected).pipe(Effect.andThen(rpc.attach(attachment, options))), + ]) + }).pipe(Effect.scoped, Effect.ensuring(Effect.sync(stop))), + { signal: entry.abort.signal }, + ) + .catch(stop) + await entry.registered.promise + if (entries.get(bindingID) !== entry) throw new Error("browser.pane.registration.closed") + publishState(entry) + }, + layout(win: BrowserWindow, bindingID: string, value?: BrowserPaneLayout) { + const entry = owned(win, bindingID) + if (!value) return closePage(entry) + const bounds = value.bounds + if (!value.visible || !bounds || bounds.width <= 0 || bounds.height <= 0) { + entry.page?.view.setVisible(false) + return + } + const page = create(entry) + page.view.setBounds(bounds) + page.view.setVisible(true) + }, + async command(win: BrowserWindow, bindingID: string, command: BrowserPaneCommand) { + const entry = owned(win, bindingID) + await execute( + entry, + { action: command, generation: entry.page?.state().generation ?? 0 }, + new AbortController().signal, + ) + }, + async close(win: BrowserWindow, bindingID: string) { + close(owned(win, bindingID)) + }, + async dispose() { + disposed = true + entries.forEach(close) + await runtime.dispose() + }, + } + + function owned(win: BrowserWindow, bindingID: string) { + const entry = entries.get(bindingID) + if (!entry || entry.win !== win) throw new Error("browser.pane.unavailable") + return entry + } + + function publish(entry: Entry, event: BrowserPaneEvent["event"]) { + if (!entries.has(entry.bindingID) || entry.win.isDestroyed() || entry.win.webContents.isDestroyed()) return + emitIpcEvent(entry.win.webContents, new BrowserPaneEvent({ bindingID: entry.bindingID, event })) + } + + function close(entry: Entry) { + if (entries.get(entry.bindingID) !== entry) return + closePage(entry, "browser.pane.registration.closed") + entries.delete(entry.bindingID) + entry.registered.reject(new Error("browser.pane.registration.closed")) + entry.cleanup?.() + entry.abort.abort() + entry.report = undefined + } + + function closePage(entry: Entry, error?: string) { + entry.requests.forEach((request) => request.abort()) + entry.requests.clear() + entry.page?.dispose() + entry.page = undefined + publishState(entry, error) + } + + function publishState(entry: Entry, error?: string) { + const state = entry.page?.state() ?? null + entry.report?.(state) + publish(entry, { type: "state", state, ...(error === undefined ? {} : { error }) }) + } + + function create(entry: Entry) { + if (entry.page) return entry.page + const fail = () => { + if (entry.page === page) closePage(entry, "page_crashed") + } + const page = createBrowserPage( + entry.win, + (error) => { + if (entry.page === page) publishState(entry, error) + }, + fail, + ) + entry.page = page + void page.ready + .then(() => { + if (entry.page === page) publishState(entry) + }) + .catch(fail) + return page + } + + async function execute(entry: Entry, command: Browser.Command, signal: AbortSignal) { + if (command.action.type === "open") publish(entry, { type: "open" }) + const page = command.action.type === "open" ? create(entry) : entry.page + if (!page) throw new Error("not_attached") + await page.ready + return page.execute(command, signal) + } +} diff --git a/packages/desktop/src/main/ipc-handlers/events.ts b/packages/desktop/src/main/ipc-handlers/events.ts index 018bbb1d8d9..1f9ee2643b3 100644 --- a/packages/desktop/src/main/ipc-handlers/events.ts +++ b/packages/desktop/src/main/ipc-handlers/events.ts @@ -1,14 +1,35 @@ +import { BrowserWindow } from "electron" import { Effect } from "effect" import { EventRpcs } from "../../shared/ipc-rpc" +import { createBrowserPane } from "../browser-pane" import { ipcEventStream } from "../ipc-events" import { IpcPortHandoff } from "../ipc-transport" +import { Shutdown } from "../lifecycle/shutdown" +import { isRendererUrl } from "../windows/protocol" import { sender } from "./context" export const eventHandlers = EventRpcs.toLayer( Effect.gen(function* () { const handoff = yield* IpcPortHandoff + const shutdown = yield* Shutdown.Service + const browser = createBrowserPane() + const stop = Effect.promise(() => browser.dispose()) + const remove = yield* shutdown.add(stop) + yield* Effect.addFinalizer(() => Effect.sync(remove).pipe(Effect.andThen(stop))) return EventRpcs.of({ DesktopEvents: (_request, context) => ipcEventStream(sender(handoff, context).id), + BrowserPane: ({ request }, context) => + Effect.tryPromise(async () => { + const contents = sender(handoff, context) + const win = BrowserWindow.fromWebContents(contents) + if (!win || win.isDestroyed() || win.webContents !== contents || !isRendererUrl(contents.getURL())) { + throw new Error("browser.pane.owner.invalid") + } + if (request.type === "register") return browser.register(win, request.bindingID, request.target) + if (request.type === "layout") return browser.layout(win, request.bindingID, request.layout) + if (request.type === "command") return browser.command(win, request.bindingID, request.command) + return browser.close(win, request.bindingID) + }).pipe(Effect.orDie), }) }), ) diff --git a/packages/desktop/src/renderer/api-types.ts b/packages/desktop/src/renderer/api-types.ts index a13b7c0ce22..5d4d47abe92 100644 --- a/packages/desktop/src/renderer/api-types.ts +++ b/packages/desktop/src/renderer/api-types.ts @@ -1,7 +1,9 @@ +import type { BrowserPaneEvent } from "@opencode-ai/app/desktop" import type { DesktopMenuAction } from "@opencode-ai/app/desktop-menu" import type { DesktopNativeBundle } from "@opencode-ai/app/i18n/desktop-native" import type { UpdaterState } from "@opencode-ai/app/updater" import type { WslServersPlatform } from "@opencode-ai/app/wsl/types" +import type { BrowserPaneRequest } from "../shared/ipc-rpc/browser" import type { ClipboardImage, DirectoryPickerOptions, @@ -23,6 +25,11 @@ export type UpdaterAPI = { export type ElectronAPI = { awaitInitialization(): Promise reconnectService(): Promise + browserPane: { + request(request: BrowserPaneRequest): Promise + send(request: BrowserPaneRequest): void + onEvent(callback: (value: { readonly bindingID: string; readonly event: BrowserPaneEvent }) => void): () => void + } wslServers: WslServersAPI updater: UpdaterAPI consumeInitialDeepLinks(): Promise diff --git a/packages/desktop/src/renderer/api.ts b/packages/desktop/src/renderer/api.ts index 1395d1356a8..44fe3ab9676 100644 --- a/packages/desktop/src/renderer/api.ts +++ b/packages/desktop/src/renderer/api.ts @@ -25,6 +25,11 @@ const updaterHandler = (state: UpdaterState) => { export const api: ElectronAPI = { awaitInitialization: () => invoke("AppAwaitInitialization"), reconnectService: () => invoke("AppReconnectService"), + browserPane: { + request: (request) => invoke("BrowserPane", { request }), + send: (request) => send("BrowserPane", { request }), + onEvent: (callback) => listen("BrowserPaneEvent", (value) => callback(mutable(value))), + }, wslServers: { getState: () => invoke("WslGetState").then(mutable), subscribe: (cb) => { diff --git a/packages/desktop/src/renderer/platform/index.ts b/packages/desktop/src/renderer/platform/index.ts index 4a127477cf1..6a2141bd9a6 100644 --- a/packages/desktop/src/renderer/platform/index.ts +++ b/packages/desktop/src/renderer/platform/index.ts @@ -30,6 +30,33 @@ export function createDesktopPlatform( windowID: windowState.id, ...createDesktopFiles(api, os, ACCEPTED_FILE_EXTENSIONS), ...createDesktopStorage(api), + browserPane: { + register(target, onEvent) { + const bindingID = crypto.randomUUID() + let closed = false + const dispose = api.browserPane.onEvent((value) => { + if (!closed && value.bindingID === bindingID) onEvent(value.event) + }) + const ready = api.browserPane.request({ type: "register", bindingID, target }) + return { + setLayout(layout) { + if (!closed) + void ready + .then(() => + api.browserPane.send({ type: "layout", bindingID, ...(layout === undefined ? {} : { layout }) }), + ) + .catch(() => undefined) + }, + command: (command) => ready.then(() => api.browserPane.request({ type: "command", bindingID, command })), + close() { + if (closed) return + closed = true + dispose() + void ready.then(() => api.browserPane.request({ type: "close", bindingID })).catch(() => undefined) + }, + } + }, + }, updater, exportDebugLogs: () => api.exportDebugLogs(), setForceFocus: (enabled) => api.setForceFocus(enabled), diff --git a/packages/desktop/src/shared/ipc-rpc/browser.ts b/packages/desktop/src/shared/ipc-rpc/browser.ts new file mode 100644 index 00000000000..49ed750f835 --- /dev/null +++ b/packages/desktop/src/shared/ipc-rpc/browser.ts @@ -0,0 +1,31 @@ +import { Browser } from "@opencode-ai/browser/rpc" +import { Schema } from "effect" +import { Rpc } from "effect/unstable/rpc" + +const text = (maximum: number) => Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(maximum)) +const bindingID = text(128) +const endpoint = Schema.Struct({ + url: text(16_384), + username: Schema.optionalKey(text(1_024)), + password: Schema.optionalKey(text(4_096)), +}) +const target = Schema.Struct({ sessionID: text(256).check(Schema.isStartsWith("ses")), endpoint }) +const bounds = Schema.Struct({ x: Schema.Finite, y: Schema.Finite, width: Schema.Finite, height: Schema.Finite }) +const layout = Schema.Struct({ visible: Schema.Boolean, bounds: Schema.optionalKey(bounds) }) +export const BrowserPaneRequestSchema = Schema.Union([ + Schema.Struct({ type: Schema.Literal("register"), bindingID, target }), + Schema.Struct({ type: Schema.Literal("layout"), bindingID, layout: Schema.optionalKey(layout) }), + Schema.Struct({ type: Schema.Literal("command"), bindingID, command: Browser.Action }), + Schema.Struct({ type: Schema.Literal("close"), bindingID }), +]) +export type BrowserPaneRequest = Schema.Schema.Type + +export const BrowserPaneEventSchema = Schema.Union([ + Schema.Struct({ type: Schema.Literal("open") }), + Schema.Struct({ + type: Schema.Literal("state"), + state: Schema.NullOr(Browser.State), + error: Schema.optionalKey(Schema.String), + }), +]) +export const BrowserPaneRpc = Rpc.make("BrowserPane", { payload: { request: BrowserPaneRequestSchema } }) diff --git a/packages/desktop/src/shared/ipc-rpc/events.ts b/packages/desktop/src/shared/ipc-rpc/events.ts index ef3f74d0162..d5aa2679e42 100644 --- a/packages/desktop/src/shared/ipc-rpc/events.ts +++ b/packages/desktop/src/shared/ipc-rpc/events.ts @@ -1,8 +1,14 @@ import { Schema } from "effect" import { Rpc, RpcGroup } from "effect/unstable/rpc" +import { BrowserPaneEventSchema, BrowserPaneRpc } from "./browser" import { UpdaterStateSchema } from "./updater" import { WslServersEventSchema } from "./wsl" +export class BrowserPaneEvent extends Schema.TaggedClass()("BrowserPaneEvent", { + bindingID: Schema.String, + event: BrowserPaneEventSchema, +}) {} + export class DeepLinksOpened extends Schema.TaggedClass()("DeepLinksOpened", { urls: Schema.Array(Schema.String), }) {} @@ -32,6 +38,7 @@ export class WindowZoomChanged extends Schema.TaggedClass()(" }) {} export const DesktopEvent = Schema.Union([ + BrowserPaneEvent, DeepLinksOpened, MenuCommandTriggered, UpdaterStateChanged, @@ -43,4 +50,4 @@ export const DesktopEvent = Schema.Union([ export type DesktopEvent = Schema.Schema.Type export const DesktopEvents = Rpc.make("DesktopEvents", { success: DesktopEvent, stream: true }) -export const EventRpcs = RpcGroup.make(DesktopEvents) +export const EventRpcs = RpcGroup.make(DesktopEvents, BrowserPaneRpc)