mirror of
https://github.com/NeuralNomadsAI/CodeNomad.git
synced 2026-11-17 05:49:28 +00:00
## Scope Native browser previews, agent browser automation parity, and preview capabilities across **Electron, Tauri Windows, Tauri macOS and Tauri Linux**. - Shared native embedding, geometry, overlay masking, session-owned history and profile-scoped preview storage, isolated from the trusted application renderer. - Public `codenomad.browser`: open, navigate, snapshot, click, type, select, press, scroll, hover, drag, wait and screenshot through the authenticated backend and fenced native-parent channel. - Six native viewport presets, WebKit HTTP/JavaScript mobile identity, reset/reload and per-registration isolation. Native WebKit density/input differences remain explicit. - OAuth pop-ups with opener/postMessage and shared preview storage; downloads with sanitized unique names; native dialogs and owner/opener/renderer lifetime cleanup. - Camera, microphone, location and notifications remain denied. Other classified permissions retain native consent/selection and fail-closed attribution. Guests receive no application privileges, app IPC, page Node access or general evaluation bridge. - Current-state developer documentation and user help in ten languages. **Tauri macOS requires macOS 14 or newer; Electron keeps its own platform baseline.** ## Host matrix | Host | Native preview/history | Agent automation | Mobile profiles | Pop-ups/downloads/dialogs | | --- | --- | --- | --- | --- | | Electron | Chromium | Supported | Chromium | Supported | | Tauri Windows | WebView2 | Supported | Chromium | Supported | | Tauri macOS 14+ | WKWebView | Supported | Native WebKit | Supported | | Tauri Linux | WebKitGTK | Supported | Native WebKit | Supported | ## Qualification - **Ubuntu 24.04 KDE VM:** real application built from the merged sources, real backend and parent channel, private OpenCode daemon and deterministic provider producing actual agent tool calls. Full browser action sequence, OAuth opener messaging, downloaded-file contents, native dialog acceptance and PNG passed under X11/Xwayland. Additional native Wayland qualification uses an owned private KWin compositor; native dialog operator coverage is recorded separately. - **macOS 15.8 Intel VM:** same real-application/agent-tool path, all actions and capability smoke passed with the built UI, private profile/configuration and private OpenCode daemon. The app-window nested-scroll check also qualifies the Cocoa wheel-coordinate correction. - **Windows:** Tauri release executable under its own profile/configuration; all actions, OAuth, completed download, native dialog and PNG passed through actual agent tool calls. The user's main CodeNomad and shared OpenCode service remain untouched. - Native CI qualifies Linux, macOS Intel, macOS ARM64 and Windows. Authorized draft qualification uses no signing secrets. - A **real Mac trial remains a release qualification step**. Graphical macOS ARM64 and physical hardware capabilities retain their own qualification scope; the capability matrix describes supported contracts rather than exhaustive peripheral testing. ## Documented limits - macOS first standalone hover can omit `pointerenter`; ⌘A selects through the native responder while consuming the main A keydown. - WebKit captures script-created closed shadow roots; declarative closed shadow roots remain opaque. - WebKitGTK preserves CVE-2025-13947 dropped-file protection; use the page's native file input. No dropped URI text is converted into host file access. - GTK async clipboard reads lack requesting-frame attribution and are refused; native Ctrl+V and clipboard writing remain available. - WebKit keeps native screen/platform/pointer/hover/touch metadata. WKWebView uses DPR 2.75 when its runtime-checked native selector exists; GTK DPR follows monitor scale × fitted zoom. CSS dimensions remain stable across app zoom/DPI. No JavaScript getter shims. - Cross-origin frame contents retain the documented opaque automation boundary. Page input timeouts have unknown outcomes and are never replayed automatically. ## Integration - Merge `ef69b4085`: `feat/tauri-webkit-parity`. - Merge `e0d56d428`: `feat/tauri-webkit-capabilities` (`d58022b13`). - Activation `3c68ab566`; Cocoa wheel correction `ecf5751ec`. - The fetched `origin/dev` merges cleanly with this head. Pushed history is preserved; no force-push or merge into `dev`. The shared embedded-view layer remains available to remote-tabs work while each consumer retains its own origin/profile/login/pairing/events policy.
180 lines
13 KiB
JavaScript
180 lines
13 KiB
JavaScript
// Real packaged app + backend + native parent, with a private OpenCode daemon and
|
|
// deterministic provider producing ordinary codenomad.browser agent tool calls.
|
|
import assert from "node:assert/strict"
|
|
import { spawn, execFileSync } from "node:child_process"
|
|
import { createServer } from "node:http"
|
|
import { mkdir, readFile, writeFile, chmod, readdir } from "node:fs/promises"
|
|
import { createHash, randomUUID } from "node:crypto"
|
|
import os from "node:os"
|
|
import path from "node:path"
|
|
import { setTimeout as delay } from "node:timers/promises"
|
|
import { tsImport } from "tsx/esm/api"
|
|
import Fastify from "fastify"
|
|
import staticFiles from "@fastify/static"
|
|
import { fileURLToPath } from "node:url"
|
|
|
|
const [appPath, cli, outputPath] = process.argv.slice(2)
|
|
if (![appPath, cli, outputPath].every(value => value && path.isAbsolute(value))) throw Error("Usage: node scripts/qualify-native-browser-app.mjs <app-executable> <opencode-executable> <output-directory>")
|
|
const output = outputPath, root = path.join(output, "private"), project = path.join(root, "project")
|
|
await mkdir(project, { recursive: true })
|
|
const run = randomUUID().slice(0, 8), password = `p7-${randomUUID()}`, started = Date.now()
|
|
let desired, issued, providerError, app, daemon, daemonOutput = "", appOutput = ""
|
|
const tools = [], evidence = []
|
|
const ui = Fastify()
|
|
await ui.register(staticFiles, { root: fileURLToPath(new URL("../packages/tauri-app/src-tauri/resources/server/public", import.meta.url)) })
|
|
ui.setNotFoundHandler((_request, reply) => reply.sendFile("index.html"))
|
|
await ui.listen({ host: "127.0.0.1", port: 0 })
|
|
const fixtureHtml = await readFile(new URL("../packages/tauri-app/tests/browser-parent/app.html", import.meta.url), "utf8")
|
|
const fixture = createServer((req, res) => {
|
|
if (req.url === "/download") {
|
|
res.setHeader("Content-Disposition", `attachment; filename="p7-${run}.txt"`)
|
|
res.end(`native-download-${run}`)
|
|
} else if (req.url === "/oauth") {
|
|
res.setHeader("Content-Type", "text/html")
|
|
res.end("<title>OAuth fixture</title><script>opener.postMessage('oauth-complete',location.origin);setTimeout(()=>close(),300)</script>")
|
|
} else { res.setHeader("Content-Type", "text/html"); res.end(fixtureHtml) }
|
|
})
|
|
const provider = createServer(async (req, res) => {
|
|
try {
|
|
let raw = ""; for await (const chunk of req) raw += chunk
|
|
const body = JSON.parse(raw), catalog = (body.tools || []).map(tool => tool.function?.name)
|
|
const command = catalog.includes("codenomad_browser") ? desired : undefined
|
|
if (command) { desired = undefined; issued = true; tools.push(command) }
|
|
if (!body.stream) { res.setHeader("Content-Type", "application/json"); res.end(JSON.stringify({ id: run, choices: [{ message: { role: "assistant", content: "Done" }, finish_reason: "stop" }] })); return }
|
|
res.setHeader("Content-Type", "text/event-stream")
|
|
const chunk = (delta, finish_reason = null) => res.write(`data: ${JSON.stringify({ id: run, object: "chat.completion.chunk", model: "fixture", choices: [{ index: 0, delta, finish_reason }] })}\n\n`)
|
|
chunk({ role: "assistant" })
|
|
if (command) chunk({ tool_calls: [{ index: 0, id: `p7-${tools.length}`, type: "function", function: { name: "codenomad_browser", arguments: JSON.stringify(command) } }] })
|
|
else chunk({ content: "Done" })
|
|
chunk({}, command ? "tool_calls" : "stop"); res.end("data: [DONE]\n\n")
|
|
} catch (error) { providerError = error; res.destroy(error) }
|
|
})
|
|
await Promise.all([fixture, provider].map(server => new Promise(done => server.listen(0, "127.0.0.1", done))))
|
|
const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !/^(OPENCODE_|CODENOMAD_)/.test(key)))
|
|
for (const key of ["XDG_DATA_HOME", "XDG_CONFIG_HOME", "XDG_STATE_HOME", "XDG_CACHE_HOME"]) env[key] = path.join(root, key)
|
|
Object.assign(env, { HOME: root, USERPROFILE: root, OPENCODE_TEST_HOME: root,
|
|
XDG_RUNTIME_DIR: process.env.XDG_RUNTIME_DIR || path.join(os.homedir(), ".config"),
|
|
OPENCODE_CONFIG_DIR: path.join(root, "config"), OPENCODE_DB: path.join(root, "fixture.db"), OPENCODE_SERVER_PASSWORD: password,
|
|
OPENCODE_CONFIG_PROJECT_DISABLE: "1", OPENCODE_DISABLE_MODELS_FETCH: "1", OPENCODE_DISABLE_FFF: "1" })
|
|
await mkdir(env.OPENCODE_CONFIG_DIR, { recursive: true })
|
|
await writeFile(path.join(env.OPENCODE_CONFIG_DIR, "opencode.json"), "{}")
|
|
env.OPENCODE_CONFIG_CONTENT = JSON.stringify({ model: "fixture/fixture", permissions: [{ action: "*", resource: "*", effect: "allow" }],
|
|
providers: { fixture: { package: "@opencode/ai/providers/openai-compatible", settings: { baseURL: `http://127.0.0.1:${provider.address().port}/v1`, apiKey: "fixture" }, models: { fixture: {} } } } })
|
|
async function until(check, timeout = 60000) {
|
|
const end = Date.now() + timeout
|
|
do { if (providerError) throw providerError; const value = await check(); if (value) return value; await delay(150) } while (Date.now() < end)
|
|
throw Error("Qualification condition timed out")
|
|
}
|
|
function strings(value) {
|
|
if (typeof value === "string") return value
|
|
if (Array.isArray(value)) return value.map(strings).join("\n")
|
|
return value && typeof value === "object" ? Object.values(value).map(strings).join("\n") : ""
|
|
}
|
|
try {
|
|
daemon = spawn(cli, ["serve", "--hostname", "127.0.0.1", "--port", "0", "--print-logs"], { cwd: root, env, windowsHide: true })
|
|
daemon.stdout.on("data", chunk => { daemonOutput += chunk }); daemon.stderr.on("data", chunk => { daemonOutput += chunk })
|
|
await until(() => /http:\/\/127\.0\.0\.1:\d+/.test(daemonOutput))
|
|
const baseUrl = daemonOutput.match(/http:\/\/127\.0\.0\.1:\d+/)[0]
|
|
// The fixture CLI adapter pins lifecycle discovery to this retained private daemon.
|
|
// All non-lifecycle commands run the real CLI under the same isolated environment.
|
|
const wrapperModule = path.join(root, "cli.cjs"), wrapper = path.join(root, process.platform === "win32" ? "opencode.cmd" : "opencode")
|
|
await writeFile(wrapperModule, `const {spawnSync}=require('node:child_process'); const a=process.argv.slice(2); if(a.join(' ')==='service status')console.log(${JSON.stringify(baseUrl)}); else if(a.join(' ')==='service get password')console.log(${JSON.stringify(password)}); else if(a[0]==='service'){console.error('Fixture lifecycle is externally owned');process.exit(1)}else {const r=spawnSync(${JSON.stringify(cli)},a,{env:${JSON.stringify(env)},stdio:'inherit',windowsHide:true});process.exit(r.status??1)}`)
|
|
await writeFile(wrapper, process.platform === "win32" ? `@echo off\r\n"${process.execPath}" "${wrapperModule}" %*\r\n` : `#!/bin/sh\nexec '${process.execPath.replaceAll("'", "'\\''")}' '${wrapperModule.replaceAll("'", "'\\''")}' "$@"\n`)
|
|
if (process.platform !== "win32") await chmod(wrapper, 0o755)
|
|
const { OpenCodeCliService } = await tsImport("../packages/server/src/workspaces/opencode-cli-service.ts", import.meta.url)
|
|
const { OpenCodeSharedService } = await tsImport("../packages/server/src/workspaces/opencode-service.ts", import.meta.url)
|
|
const lifecycle = new OpenCodeCliService({ label: "Private app qualification", timeoutMs: 10000, command: args => ({ command: cli, args, options: {} }) },
|
|
{ execFile: async (_file, args) => ({ stdout: args.join(" ") === "service get password" ? password : baseUrl, stderr: "" }) })
|
|
const shared = new OpenCodeSharedService(), client = await shared.client({ kind: "lifecycle", identity: `p7-${run}`, lifecycle })
|
|
const session = await client.session.create({ location: { directory: project }, title: "Native browser qualification" })
|
|
const config = path.join(root, "config.yaml")
|
|
await writeFile(config, JSON.stringify({ server: { opencodeBinary: wrapper }, preferences: { lastUsedBinary: wrapper, locale: "en" }, opencodeBinaries: [{ path: wrapper, lastUsed: Date.now() }] }))
|
|
const appEnv = { ...process.env, CODENOMAD_PROFILE: `p7-${run}`, CLI_CONFIG: config,
|
|
VITE_DEV_SERVER_URL: `http://127.0.0.1:${ui.server.address().port}` }
|
|
delete appEnv.ELECTRON_RUN_AS_NODE
|
|
app = spawn(appPath, [project], { cwd: path.dirname(appPath), env: appEnv, windowsHide: false })
|
|
app.stdout.on("data", chunk => { appOutput += chunk }); app.stderr.on("data", chunk => { appOutput += chunk })
|
|
await until(async () => {
|
|
if (app.exitCode !== null) throw Error(`App exited ${app.exitCode}: ${appOutput.slice(-4000)}`)
|
|
return (await client.plugin.list({ location: { directory: project } })).data.some(plugin => plugin.id === "codenomad.automation" && plugin.state.status === "active")
|
|
}, 90000)
|
|
async function call(command) {
|
|
issued = false; desired = command
|
|
await client.session.prompt({ sessionID: session.id, text: `Qualification step ${evidence.length + 1}. Use the browser tool.` })
|
|
await client.session.wait({ sessionID: session.id }, { signal: AbortSignal.timeout(95000) })
|
|
assert.ok(issued, "Native model tool catalog must invoke codenomad.browser")
|
|
const messages = await client.message.list({ sessionID: session.id, limit: 100, order: "asc" })
|
|
const tool = messages.data.flatMap(message => message.type === "assistant" ? message.content : [])
|
|
.filter(part => part.type === "tool" && part.name.includes("browser")).at(-1)
|
|
assert.equal(tool?.state.status, "completed", JSON.stringify(tool?.state))
|
|
const result = tool.state.content
|
|
evidence.push({ command, toolID: tool.id, status: tool.state.status, content: result })
|
|
await writeFile(path.join(output, "tool-results.json"), JSON.stringify(evidence, null, 2))
|
|
console.log(`PASS ${command.action}`)
|
|
return result
|
|
}
|
|
const url = `http://127.0.0.1:${fixture.address().port}/`
|
|
await call({ action: "open", url })
|
|
let snapshot = strings(await call({ action: "snapshot" }))
|
|
const ref = label => {
|
|
const line = snapshot.split("\n").find(line => line.includes(`"${label}"`) && !line.trimStart().startsWith("- text "))
|
|
assert.ok(line, `Missing ${label}: ${snapshot}`)
|
|
return /\[([^\]]+)\]/.exec(line)[1]
|
|
}
|
|
await call({ action: "click", ref: ref("Increment") })
|
|
await call({ action: "type", ref: ref("Name"), text: "Native parity" })
|
|
await call({ action: "select", ref: ref("Plan"), values: ["Pro"] })
|
|
await call({ action: "press", ref: ref("Name"), key: "Enter" })
|
|
await call({ action: "hover", ref: ref("Hover target") })
|
|
await call({ action: "drag", ref: ref("Drag source"), to: ref("Drop target") })
|
|
await call({ action: "scroll", ref: ref("Scroll box"), direction: "down", amount: 150 })
|
|
for (const text of ["Count 1", "Pressed Enter", "Hovered", "Dragged", "Scrolled"]) await call({ action: "wait", text })
|
|
await call({ action: "click", ref: ref("OAuth popup") })
|
|
await call({ action: "wait", text: "OAuth complete" })
|
|
await call({ action: "click", ref: ref("Download fixture") })
|
|
const downloads = process.platform === "linux"
|
|
? execFileSync("xdg-user-dir", ["DOWNLOAD"], { encoding: "utf8" }).trim()
|
|
: path.join(os.homedir(), "Downloads")
|
|
const download = await until(async () => {
|
|
const name = (await readdir(downloads)).find(name => name.includes(`p7-${run}.txt`))
|
|
return name && await readFile(path.join(downloads, name), "utf8").catch(() => "") === `native-download-${run}` ? name : undefined
|
|
})
|
|
assert.equal(await readFile(path.join(downloads, download), "utf8"), `native-download-${run}`)
|
|
if (!process.env.P7_SKIP_NATIVE_DIALOG) {
|
|
await call({ action: "click", ref: ref("Native dialog") })
|
|
await delay(1500)
|
|
if (process.platform === "darwin") execFileSync("python3", [new URL("fixtures/accept-owned-browser-dialog.py", import.meta.url).pathname, String(app.pid)])
|
|
else if (process.platform === "linux") {
|
|
const windows = execFileSync("xdotool", ["search", "--onlyvisible", "--pid", String(app.pid)], { encoding: "utf8" }).trim().split("\n")
|
|
execFileSync("xdotool", ["windowactivate", "--sync", windows.at(-1)])
|
|
execFileSync("xdotool", ["key", "Return"])
|
|
} else {
|
|
execFileSync("pwsh", ["-NoProfile", "-File", new URL("fixtures/accept-owned-browser-dialog.ps1", import.meta.url).pathname.replace(/^\/(\w:)/, "$1"), "-AppProcessId", String(app.pid)], { windowsHide: true, timeout: 20000 })
|
|
}
|
|
await call({ action: "wait", text: "Dialog accepted" })
|
|
}
|
|
snapshot = strings(await call({ action: "snapshot" }))
|
|
assert.match(snapshot, /Native parity/)
|
|
const image = strings(await call({ action: "screenshot" }))
|
|
const png = /data:image\/png;base64,([A-Za-z0-9+/=]+)/.exec(image)
|
|
assert.ok(png, "Browser agent tool must return PNG data")
|
|
await writeFile(path.join(output, "browser.png"), Buffer.from(png[1], "base64"))
|
|
await writeFile(path.join(output, "result.json"), JSON.stringify({ passed: true, run, sessionID: session.id, project, appPath,
|
|
appSHA256: createHash("sha256").update(await readFile(appPath)).digest("hex"), tools: tools.map(command => command.action), download, screenshot: !!png,
|
|
nativeDialog: !process.env.P7_SKIP_NATIVE_DIALOG }, null, 2))
|
|
await shared.shutdown()
|
|
} finally {
|
|
app?.kill(); daemon?.kill()
|
|
if (process.platform !== "win32" && app) {
|
|
for (const [, pid] of appOutput.matchAll(/spawned pid=(\d+)/g)) {
|
|
try {
|
|
const args = execFileSync("ps", ["-ww", "-p", pid, "-o", "command="], { encoding: "utf8" })
|
|
if (args.includes(`codenomad_session_${app.pid}_`)) process.kill(-Number(pid), "SIGTERM")
|
|
} catch { /* The exact owned backend may already have stopped. */ }
|
|
}
|
|
}
|
|
await writeFile(path.join(output, "app.log"), appOutput)
|
|
await writeFile(path.join(output, "opencode.log"), daemonOutput)
|
|
await Promise.all([fixture, provider].map(server => new Promise(done => server.close(done))))
|
|
await ui.close()
|
|
}
|