diff --git a/bun.lock b/bun.lock index 0102b13a6be..491dbd16794 100644 --- a/bun.lock +++ b/bun.lock @@ -873,6 +873,7 @@ "name": "@opencode-ai/simulation", "version": "1.17.13", "dependencies": { + "@napi-rs/canvas": "1.0.2", "@opencode-ai/core": "workspace:*", "@opencode-ai/llm": "workspace:*", "@opentui/core": "catalog:", @@ -1983,6 +1984,30 @@ "@msgpackr-extract/msgpackr-extract-win32-x64": ["@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4", "", { "os": "win32", "cpu": "x64" }, "sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ=="], + "@napi-rs/canvas": ["@napi-rs/canvas@1.0.2", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "1.0.2", "@napi-rs/canvas-darwin-arm64": "1.0.2", "@napi-rs/canvas-darwin-x64": "1.0.2", "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.2", "@napi-rs/canvas-linux-arm64-gnu": "1.0.2", "@napi-rs/canvas-linux-arm64-musl": "1.0.2", "@napi-rs/canvas-linux-riscv64-gnu": "1.0.2", "@napi-rs/canvas-linux-x64-gnu": "1.0.2", "@napi-rs/canvas-linux-x64-musl": "1.0.2", "@napi-rs/canvas-win32-arm64-msvc": "1.0.2", "@napi-rs/canvas-win32-x64-msvc": "1.0.2" } }, "sha512-EYEqlMYaCbpZDz+IgDH5xp9MTd3ui4dmGqbQYryhMLnSRxrhHKq5KQWHHKxFUcEP4Hp8/BWgvqXocX4j7iSbOQ=="], + + "@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@1.0.2", "", { "os": "android", "cpu": "arm64" }, "sha512-IMXKVQod0ol4vt3gmClUfXz4JAgHYESGPCUqmH3lQxBoL0K/2greJaQE1HVBVxWWFKfLc4OLZVdxg7kXVyXv+g=="], + + "@napi-rs/canvas-darwin-arm64": ["@napi-rs/canvas-darwin-arm64@1.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Sc8tPi6cF+5lqOzCCKFALJHhDiRwyMzTPYm3bbhdXsOunU0lQO5f05ucyOzN2r55I23Hg5bsjH63uSCvWp3EgQ=="], + + "@napi-rs/canvas-darwin-x64": ["@napi-rs/canvas-darwin-x64@1.0.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-niDXZ9LhKB1zLrUdYB64RHQFDGz9rr0eGx061qtJJU3U20EMMIx28ADF5fVYbhtOgkWQrBjFicfaye1yM0U62A=="], + + "@napi-rs/canvas-linux-arm-gnueabihf": ["@napi-rs/canvas-linux-arm-gnueabihf@1.0.2", "", { "os": "linux", "cpu": "arm" }, "sha512-sgatQL9JxGRH/Amzcvu0P3t8Am3duou74CisfuJ41Dwt8cWy723z/9KZ8LlgmxfypEwEZxSTNFJtU8d281lmhQ=="], + + "@napi-rs/canvas-linux-arm64-gnu": ["@napi-rs/canvas-linux-arm64-gnu@1.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-dgKuX0peF3xwY6ZF5QxGS4wbfDqpoFAJYXiLSp+guZKARQUKMkRqZSDrXKj7nfrec3UCMzC0PFCPte0ES98AiA=="], + + "@napi-rs/canvas-linux-arm64-musl": ["@napi-rs/canvas-linux-arm64-musl@1.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-qwROoDIC9upfvDoRLuPn2aNg9CGW1x0Ygr4k2Or+8paA9d0qBLwk87U+g8KQpoOviKoPoiwl97kvBYuYD7qZoA=="], + + "@napi-rs/canvas-linux-riscv64-gnu": ["@napi-rs/canvas-linux-riscv64-gnu@1.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-fXRjnPihdnbO6qy1QQOgxAonb68A0TCEG7rj1x7v7rxNElsE8EVIKIEUTvyDtU+sthYSbX+8e7g3oZiLGnOmxw=="], + + "@napi-rs/canvas-linux-x64-gnu": ["@napi-rs/canvas-linux-x64-gnu@1.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-nPR97DXhbWIAy7yazF3jc06kEPMqYMLmPzFOVNlwKPfIoSChnI+x7dc0hTLaihz3jxrjL6j4BbA7earxfx4X3g=="], + + "@napi-rs/canvas-linux-x64-musl": ["@napi-rs/canvas-linux-x64-musl@1.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-l7zZY5+jL5qnBZtDz7CoBtY6p7EkHu422g/0zWwrOrzIwWyWxZFRfZZORY1UG7YApymPLx+UbOkN206xXn/c1Q=="], + + "@napi-rs/canvas-win32-arm64-msvc": ["@napi-rs/canvas-win32-arm64-msvc@1.0.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-yE0koHCFF4PIbMc2o2SEALhnipz7WBISh5glLvQiomtIoCcW0np3H4Lw93ceJAfJttTTeIIWFbwH84F7EVzjMQ=="], + + "@napi-rs/canvas-win32-x64-msvc": ["@napi-rs/canvas-win32-x64-msvc@1.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-okU8/t2foV6C31n0GtvEMbfD5rOFc70+/6xUNME9Guld29sgSOIGUEDScAWFlcP3k5TYQRl9TNkwJEEjh15w8A=="], + "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.4", "", { "dependencies": { "@tybys/wasm-util": "^0.10.1" }, "peerDependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1" } }, "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow=="], "@noble/hashes": ["@noble/hashes@2.2.0", "", {}, "sha512-IYqDGiTXab6FniAgnSdZwgWbomxpy9FtYvLKs7wCUs2a8RkITG+DFGO1DM9cr+E3/RgADRpFjrKVaJ1z6sjtEg=="], diff --git a/packages/simulation/package.json b/packages/simulation/package.json index 29613ff28ac..a98b6f3289c 100644 --- a/packages/simulation/package.json +++ b/packages/simulation/package.json @@ -16,6 +16,7 @@ "typecheck": "tsgo --noEmit" }, "dependencies": { + "@napi-rs/canvas": "1.0.2", "@opencode-ai/core": "workspace:*", "@opencode-ai/llm": "workspace:*", "@opentui/core": "catalog:", diff --git a/packages/simulation/src/backend/control.ts b/packages/simulation/src/backend/control.ts index a27a0cd2afe..eaacc162c1d 100644 --- a/packages/simulation/src/backend/control.ts +++ b/packages/simulation/src/backend/control.ts @@ -1,7 +1,6 @@ import { Effect } from "effect" import { SimulationProtocol } from "../protocol" import { SimulationLLMExchange } from "./llm-exchange" -import { SimulationNetwork } from "./network" /** * Backend-hosted simulation control WebSocket. @@ -19,7 +18,6 @@ import { SimulationNetwork } from "./network" * - `llm.finish` { id, reason? } finish an exchange * - `llm.disconnect` { id } abruptly terminate an exchange without a finish * - `llm.pending` list open exchanges - * - `network.log` simulated network request log */ type ControlSocket = Bun.ServerWebSocket<{ unsubscribe?: () => void }> @@ -58,8 +56,6 @@ async function handle(socket: ControlSocket, request: SimulationProtocol.Backend } case "llm.pending": return { exchanges: SimulationLLMExchange.pending() } - case "network.log": - return { entries: SimulationNetwork.log() } } } diff --git a/packages/simulation/src/frontend/actions.ts b/packages/simulation/src/frontend/actions.ts index 9e5b7bdead5..4b03cb69395 100644 --- a/packages/simulation/src/frontend/actions.ts +++ b/packages/simulation/src/frontend/actions.ts @@ -1,8 +1,11 @@ -import type { CliRenderer, Renderable } from "@opentui/core" +import { mkdtemp } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import type { CapturedFrame, CliRenderer, Renderable } from "@opentui/core" import { createMockKeys, createMockMouse, type MockInput, type MockMouse } from "@opentui/core/testing" import type { SimulationProtocol } from "../protocol" import { SimulationRenderer } from "./renderer" -import { SimulationTrace } from "./trace" +import { SimulationPng } from "./png" export type Action = SimulationProtocol.Frontend.Action export type Element = SimulationProtocol.Frontend.Element @@ -91,57 +94,95 @@ export function elements(renderer: CliRenderer): Element[] { .filter((element) => element.focusable || element.clickable || element.editor) } -export function actions(renderer: CliRenderer, options: { text?: string } = {}): Action[] { - const items = elements(renderer) - return [ - ...(renderer.currentFocusedEditor - ? ([{ type: "typeText", text: options.text ?? "hello" }, { type: "pressEnter" }] satisfies Action[]) - : []), - ...items.filter((item) => item.focusable && !item.focused).map((item) => ({ type: "focus" as const, target: item.num })), - ...items - .filter((item) => item.clickable) - .map((item) => ({ - type: "click" as const, - target: item.num, - x: Math.floor(item.x + item.width / 2), - y: Math.floor(item.y + item.height / 2), - })), - { type: "pressArrow", direction: "down" }, - { type: "pressArrow", direction: "up" }, - ] -} - export function state(harness: Harness) { return { - screen: harness.screen(), focused: { renderable: harness.renderer.currentFocusedRenderable?.num, editor: Boolean(harness.renderer.currentFocusedEditor), }, elements: elements(harness.renderer), - actions: actions(harness.renderer), } } +export async function screenshot(harness: Harness) { + await harness.renderOnce() + const image = SimulationPng.screenshot(harness.renderer) + const path = join(await mkdtemp(join(tmpdir(), "opencode-drive-")), "screenshot.png") + await Bun.write(path, image.data) + return path +} + +export function frame(harness: Harness): CapturedFrame { + const buffer = harness.renderer.currentRenderBuffer + return { + cols: buffer.width, + rows: buffer.height, + cursor: [0, 0], + lines: buffer.getSpanLines().map((line) => ({ + spans: line.spans.map((span) => ({ + text: span.text, + fg: span.fg, + bg: span.bg, + attributes: span.attributes, + width: span.width, + })), + })), + } +} + +export async function video(frames: CapturedFrame[]) { + const directory = await mkdtemp(join(tmpdir(), "opencode-drive-recording-")) + await Promise.all( + frames.map((frame, index) => + Bun.write( + join(directory, `frame-${index.toString().padStart(6, "0")}.png`), + SimulationPng.screenshotFrame(frame).data, + ), + ), + ) + const path = join(directory, "recording.mp4") + const process = Bun.spawn( + [ + "ffmpeg", + "-loglevel", + "error", + "-framerate", + "10", + "-i", + join(directory, "frame-%06d.png"), + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + "-y", + path, + ], + { stderr: "pipe" }, + ) + if ((await process.exited) !== 0) throw new Error(`ffmpeg failed: ${await new Response(process.stderr).text()}`) + return path +} + export async function execute(harness: Harness, action: Action) { - SimulationTrace.add("ui.action", { action }) switch (action.type) { - case "typeText": + case "ui.type": await harness.mockInput.typeText(action.text) break - case "pressKey": + case "ui.press": harness.mockInput.pressKey(action.key, action.modifiers) break - case "pressEnter": + case "ui.enter": harness.mockInput.pressEnter() break - case "pressArrow": + case "ui.arrow": harness.mockInput.pressArrow(action.direction) break - case "focus": + case "ui.focus": all(harness.renderer.root).find((item) => item.num === action.target)?.focus() break - case "click": + case "ui.click": await harness.mockMouse.click(action.x, action.y) break } diff --git a/packages/simulation/src/frontend/png.ts b/packages/simulation/src/frontend/png.ts new file mode 100644 index 00000000000..137a09e1be4 --- /dev/null +++ b/packages/simulation/src/frontend/png.ts @@ -0,0 +1,84 @@ +import { fileURLToPath } from "node:url" +import { GlobalFonts, createCanvas } from "@napi-rs/canvas" +import { TextAttributes, type CapturedFrame, type CliRenderer, type RGBA } from "@opentui/core" + +const CellWidth = 10 +const CellHeight = 20 +const FontSize = 16 +const FontFamily = "OpenCode Screenshot" + +GlobalFonts.registerFromPath( + fileURLToPath(new URL("../../../ui/src/assets/fonts/JetBrainsMonoNerdFontMono-Regular.woff2", import.meta.url)), + FontFamily, +) + +export function screenshot(renderer: CliRenderer) { + return screenshotFrame({ + cols: renderer.currentRenderBuffer.width, + rows: renderer.currentRenderBuffer.height, + cursor: [0, 0], + lines: renderer.currentRenderBuffer.getSpanLines(), + }) +} + +export function screenshotFrame(frame: CapturedFrame) { + const canvas = createCanvas(frame.cols * CellWidth, frame.rows * CellHeight) + const context = canvas.getContext("2d") + context.fillStyle = "#080808" + context.fillRect(0, 0, canvas.width, canvas.height) + context.textBaseline = "top" + + frame.lines.forEach((line, row) => { + let column = 0 + line.spans.forEach((span) => { + const attributes = span.attributes & 0xff + const inverse = Boolean(attributes & TextAttributes.INVERSE) + const hidden = Boolean(attributes & TextAttributes.HIDDEN) + const foreground = inverse ? span.bg : span.fg + const background = inverse ? span.fg : span.bg + const chars = [...span.text] + let remaining = span.width + + chars.forEach((char, index) => { + const cells = Math.max(1, remaining - (chars.length - index - 1)) + if (background.a) { + context.fillStyle = color(background) + context.fillRect(column * CellWidth, row * CellHeight, cells * CellWidth, CellHeight) + } + if (!hidden && char.codePointAt(0) !== 0x0a00) { + context.fillStyle = color(foreground, attributes & TextAttributes.DIM ? 0.55 : 1) + context.font = `${attributes & TextAttributes.ITALIC ? "italic " : ""}${attributes & TextAttributes.BOLD ? "bold " : ""}${FontSize}px "${FontFamily}"` + context.fillText(char, column * CellWidth, row * CellHeight + 1) + if (attributes & TextAttributes.UNDERLINE) { + context.fillRect(column * CellWidth, row * CellHeight + 17, cells * CellWidth, 1) + } + if (attributes & TextAttributes.STRIKETHROUGH) { + context.fillRect(column * CellWidth, row * CellHeight + 10, cells * CellWidth, 1) + } + } + column += cells + remaining -= cells + }) + while (remaining-- > 0) { + if (background.a) { + context.fillStyle = color(background) + context.fillRect(column * CellWidth, row * CellHeight, CellWidth, CellHeight) + } + column++ + } + }) + }) + + return { + width: canvas.width, + height: canvas.height, + data: canvas.toBuffer("image/png"), + } +} + +function color(value: RGBA, opacity = 1) { + const [red, green, blue, alpha] = value.toInts() + return `rgba(${red}, ${green}, ${blue}, ${(alpha / 255) * opacity})` +} + +export * as SimulationPng from "./png" diff --git a/packages/simulation/src/frontend/server.ts b/packages/simulation/src/frontend/server.ts index c717dbd4c6b..4edd6878873 100644 --- a/packages/simulation/src/frontend/server.ts +++ b/packages/simulation/src/frontend/server.ts @@ -1,6 +1,6 @@ +import type { CapturedFrame } from "@opentui/core" import { SimulationProtocol } from "../protocol" import { SimulationActions, type Harness } from "./actions" -import { SimulationTrace } from "./trace" export interface Server { readonly url: string @@ -11,47 +11,90 @@ function parseRequest(input: string | Buffer) { return SimulationProtocol.Frontend.decodeRequest(JSON.parse(typeof input === "string" ? input : input.toString())) } -async function handle(harness: Harness, request: SimulationProtocol.Frontend.Request, headless: boolean) { +interface Recording { + readonly frames: CapturedFrame[] + readonly timer: ReturnType + pending: Promise +} + +async function handle( + harness: Harness, + request: SimulationProtocol.Frontend.Request, + recording: { current?: Recording }, + headless: boolean, +) { switch (request.method) { + case "ui.screenshot": + return SimulationActions.screenshot(harness) case "ui.state": { - if (headless) await harness.renderOnce() - const result = SimulationActions.state(harness) - SimulationTrace.add("ui.state", { elements: result.elements.length, actions: result.actions.length }) - return result + return SimulationActions.state(harness) } - case "ui.action": - return SimulationActions.execute(harness, request.params.action) - case "trace.list": - return { records: SimulationTrace.list() } - case "trace.clear": - SimulationTrace.clear() - return { cleared: true } - case "trace.export": - return SimulationTrace.exportTrace() + case "ui.start-record": { + if (recording.current) throw new Error("UI recording is already active") + const frames = [SimulationActions.frame(harness)] + const current: Recording = { + frames, + timer: setInterval(() => { + current.pending = current.pending.then(async () => { + if (headless) await harness.renderOnce() + frames.push(SimulationActions.frame(harness)) + }) + }, 100), + pending: Promise.resolve(), + } + recording.current = current + return { recording: true } + } + case "ui.end-record": { + if (!recording.current) throw new Error("UI recording is not active") + const current = recording.current + clearInterval(current.timer) + await current.pending + if (headless) await harness.renderOnce() + current.frames.push(SimulationActions.frame(harness)) + recording.current = undefined + return SimulationActions.video(current.frames) + } + case "ui.type": + return SimulationActions.execute(harness, { type: "ui.type", text: request.params.text }) + case "ui.enter": + return SimulationActions.execute(harness, { type: "ui.enter" }) + case "ui.press": + return SimulationActions.execute(harness, { + type: "ui.press", + key: request.params.key, + modifiers: request.params.modifiers, + }) + case "ui.arrow": + return SimulationActions.execute(harness, { type: "ui.arrow", direction: request.params.direction }) + case "ui.focus": + return SimulationActions.execute(harness, { type: "ui.focus", target: request.params.target }) + case "ui.click": + return SimulationActions.execute(harness, { + type: "ui.click", + target: request.params.target, + x: request.params.x, + y: request.params.y, + }) } } export function start(harness: Harness, endpoint: string, headless: boolean): Server { const url = new URL(endpoint) - const server = Bun.serve<{ readonly drive: true }>({ + const recording: { current?: Recording } = {} + const server = Bun.serve<{ readonly drive: true; readonly headless: boolean }>({ hostname: url.hostname, port: Number(url.port), fetch(request, server) { - if (server.upgrade(request, { data: { drive: true } })) return undefined + if (server.upgrade(request, { data: { drive: true, headless } })) return undefined return new Response("opencode drive ui websocket", { status: 426 }) }, websocket: { - open() { - SimulationTrace.add("control.connect") - }, - close() { - SimulationTrace.add("control.disconnect") - }, async message(socket, message) { let request: SimulationProtocol.Frontend.Request | undefined try { request = parseRequest(message) - const result = await handle(harness, request, headless) + const result = await handle(harness, request, recording, headless) const next = SimulationProtocol.JsonRpc.success(request.id, result) if (next) socket.send(JSON.stringify(next)) } catch (error) { @@ -60,11 +103,10 @@ export function start(harness: Harness, endpoint: string, headless: boolean): Se }, }, }) - SimulationTrace.add("control.start", { url: endpoint }) return { url: endpoint, stop: () => { - SimulationTrace.add("control.stop", { url: endpoint }) + if (recording.current) clearInterval(recording.current.timer) server.stop(true) }, } diff --git a/packages/simulation/src/frontend/trace.ts b/packages/simulation/src/frontend/trace.ts deleted file mode 100644 index b0914faace5..00000000000 --- a/packages/simulation/src/frontend/trace.ts +++ /dev/null @@ -1,37 +0,0 @@ -export type TraceRecord = { - readonly id: number - readonly time: string - readonly type: string - readonly data?: unknown -} - -const records: TraceRecord[] = [] -let nextID = 0 - -export function add(type: string, data?: unknown) { - const record = { - id: ++nextID, - time: new Date().toISOString(), - type, - ...(data === undefined ? {} : { data }), - } satisfies TraceRecord - records.push(record) - return record -} - -export function list() { - return [...records] -} - -export function clear() { - records.length = 0 - nextID = 0 -} - -export function exportTrace() { - return { - records: list(), - } -} - -export * as SimulationTrace from "./trace" diff --git a/packages/simulation/src/protocol/index.ts b/packages/simulation/src/protocol/index.ts index 2534db8367c..c4acb771883 100644 --- a/packages/simulation/src/protocol/index.ts +++ b/packages/simulation/src/protocol/index.ts @@ -59,12 +59,12 @@ export namespace Frontend { export interface KeyModifiers extends Schema.Schema.Type {} export const Action = Schema.Union([ - Schema.Struct({ type: Schema.Literal("typeText"), text: Schema.String }), - Schema.Struct({ type: Schema.Literal("pressKey"), key: Schema.String, modifiers: Schema.optional(KeyModifiers) }), - Schema.Struct({ type: Schema.Literal("pressEnter") }), - Schema.Struct({ type: Schema.Literal("pressArrow"), direction: Schema.Literals(["up", "down", "left", "right"]) }), - Schema.Struct({ type: Schema.Literal("focus"), target: Schema.Number }), - Schema.Struct({ type: Schema.Literal("click"), target: Schema.Number, x: Schema.Number, y: Schema.Number }), + Schema.Struct({ type: Schema.Literal("ui.type"), text: Schema.String }), + Schema.Struct({ type: Schema.Literal("ui.press"), key: Schema.String, modifiers: Schema.optional(KeyModifiers) }), + Schema.Struct({ type: Schema.Literal("ui.enter") }), + Schema.Struct({ type: Schema.Literal("ui.arrow"), direction: Schema.Literals(["up", "down", "left", "right"]) }), + Schema.Struct({ type: Schema.Literal("ui.focus"), target: Schema.Number }), + Schema.Struct({ type: Schema.Literal("ui.click"), target: Schema.Number, x: Schema.Number, y: Schema.Number }), ]) export type Action = Schema.Schema.Type @@ -83,39 +83,58 @@ export namespace Frontend { export interface Element extends Schema.Schema.Type {} export const State = Schema.Struct({ - screen: Schema.String, focused: Schema.Struct({ renderable: Schema.optional(Schema.Number), editor: Schema.Boolean, }), elements: Schema.Array(Element), - actions: Schema.Array(Action), }) export interface State extends Schema.Schema.Type {} - export const ActionParams = Schema.Struct({ action: Action }) - export interface ActionParams extends Schema.Schema.Type {} + export const Screenshot = Schema.String + export type Screenshot = Schema.Schema.Type + + export const StartRecord = Schema.Struct({ recording: Schema.Literal(true) }) + export interface StartRecord extends Schema.Schema.Type {} + + export const EndRecord = Schema.String + export type EndRecord = Schema.Schema.Type + + export const TypeParams = Schema.Struct({ text: Schema.String }) + export interface TypeParams extends Schema.Schema.Type {} + + export const PressParams = Schema.Struct({ key: Schema.String, modifiers: Schema.optional(KeyModifiers) }) + export interface PressParams extends Schema.Schema.Type {} + + export const ArrowParams = Schema.Struct({ direction: Schema.Literals(["up", "down", "left", "right"]) }) + export interface ArrowParams extends Schema.Schema.Type {} + + export const FocusParams = Schema.Struct({ target: Schema.Number }) + export interface FocusParams extends Schema.Schema.Type {} + + export const ClickParams = Schema.Struct({ target: Schema.Number, x: Schema.Number, y: Schema.Number }) + export interface ClickParams extends Schema.Schema.Type {} export const Request = Schema.Union([ - Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.action"), params: ActionParams }), + Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.type"), params: TypeParams }), + Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.press"), params: PressParams }), + Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.arrow"), params: ArrowParams }), + Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.focus"), params: FocusParams }), + Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.click"), params: ClickParams }), Schema.Struct({ ...JsonRpc.RequestFields, - method: Schema.Literals(["ui.state", "trace.list", "trace.clear", "trace.export"]), + method: Schema.Literals([ + "ui.enter", + "ui.screenshot", + "ui.state", + "ui.start-record", + "ui.end-record", + ]), }), ]) export type Request = Schema.Schema.Type export const decodeRequest = Schema.decodeUnknownSync(Request) - export const TraceRecord = Schema.Struct({ - id: Schema.Number, - time: Schema.String, - type: Schema.String, - data: Schema.optional(Schema.Json), - }) - export interface TraceRecord extends Schema.Schema.Type {} - - export const TraceList = Schema.Struct({ records: Schema.Array(TraceRecord) }) - export interface TraceList extends Schema.Schema.Type {} } export namespace Backend { @@ -148,7 +167,7 @@ export namespace Backend { Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("llm.disconnect"), params: DisconnectParams }), Schema.Struct({ ...JsonRpc.RequestFields, - method: Schema.Literals(["llm.attach", "llm.pending", "network.log"]), + method: Schema.Literals(["llm.attach", "llm.pending"]), }), ]) export type Request = Schema.Schema.Type