vrr/opencode
2
0
Fork 0
mirror of https://github.com/anomalyco/opencode.git synced 2026-07-10 03:58:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

feat(simulation): render driven UI screenshots and videos (#35739)

Browse source
This commit is contained in:
James Long 2026-07-07 12:15:01 -04:00 • committed by GitHub
parent dd1c007877
commit 6c065ca6a5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 292 additions and 121 deletions
Download patch file Download diff file Expand all files Collapse all files

25
bun.lock
View file

@ -873,6 +873,7 @@
"name": "@opencode-ai/simulation",
"version": "1.17.13",
"dependencies": {
"@napi-rs/canvas": "1.0.2",
"@opencode-ai/core": "workspace:*",
"@opencode-ai/llm": "workspace:*",
"@opentui/core": "catalog:",
@ -1983,6 +1984,30 @@
"@msgpackr-extract/msgpackr-extract-win32-x64": ["@msgpackr-extract/msgpackr-extract-win32-x64@3.0.4", "", { "os": "win32", "cpu": "x64" }, "sha512-CmCXPQrkbwExx3j946/PtHWHbYJiCRBRDl4BlkRQcJB/YOwQxJRTpoo7aTsortjgoJ1x7opzTSxn7C+ASSLVjQ=="],
"@napi-rs/canvas": ["@napi-rs/canvas@1.0.2", "", { "optionalDependencies": { "@napi-rs/canvas-android-arm64": "1.0.2", "@napi-rs/canvas-darwin-arm64": "1.0.2", "@napi-rs/canvas-darwin-x64": "1.0.2", "@napi-rs/canvas-linux-arm-gnueabihf": "1.0.2", "@napi-rs/canvas-linux-arm64-gnu": "1.0.2", "@napi-rs/canvas-linux-arm64-musl": "1.0.2", "@napi-rs/canvas-linux-riscv64-gnu": "1.0.2", "@napi-rs/canvas-linux-x64-gnu": "1.0.2", "@napi-rs/canvas-linux-x64-musl": "1.0.2", "@napi-rs/canvas-win32-arm64-msvc": "1.0.2", "@napi-rs/canvas-win32-x64-msvc": "1.0.2" } }, "sha512-EYEqlMYaCbpZDz+IgDH5xp9MTd3ui4dmGqbQYryhMLnSRxrhHKq5KQWHHKxFUcEP4Hp8/BWgvqXocX4j7iSbOQ=="],
"@napi-rs/canvas-android-arm64": ["@napi-rs/canvas-android-arm64@1.0.2", "", { "os": "android", "cpu": "arm64" }, "sha512-IMXKVQod0ol4vt3gmClUfXz4JAgHYESGPCUqmH3lQxBoL0K/2greJaQE1HVBVxWWFKfLc4OLZVdxg7kXVyXv+g=="],
"@napi-rs/canvas-darwin-arm64": ["@napi-rs/canvas-darwin-arm64@1.0.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-Sc8tPi6cF+5lqOzCCKFALJHhDiRwyMzTPYm3bbhdXsOunU0lQO5f05ucyOzN2r55I23Hg5bsjH63uSCvWp3EgQ=="],
"@napi-rs/canvas-darwin-x64": ["@napi-rs/canvas-darwin-x64@1.0.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-niDXZ9LhKB1zLrUdYB64RHQFDGz9rr0eGx061qtJJU3U20EMMIx28ADF5fVYbhtOgkWQrBjFicfaye1yM0U62A=="],
"@napi-rs/canvas-linux-arm-gnueabihf": ["@napi-rs/canvas-linux-arm-gnueabihf@1.0.2", "", { "os": "linux", "cpu": "arm" }, "sha512-sgatQL9JxGRH/Amzcvu0P3t8Am3duou74CisfuJ41Dwt8cWy723z/9KZ8LlgmxfypEwEZxSTNFJtU8d281lmhQ=="],
"@napi-rs/canvas-linux-arm64-gnu": ["@napi-rs/canvas-linux-arm64-gnu@1.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-dgKuX0peF3xwY6ZF5QxGS4wbfDqpoFAJYXiLSp+guZKARQUKMkRqZSDrXKj7nfrec3UCMzC0PFCPte0ES98AiA=="],
"@napi-rs/canvas-linux-arm64-musl": ["@napi-rs/canvas-linux-arm64-musl@1.0.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-qwROoDIC9upfvDoRLuPn2aNg9CGW1x0Ygr4k2Or+8paA9d0qBLwk87U+g8KQpoOviKoPoiwl97kvBYuYD7qZoA=="],
"@napi-rs/canvas-linux-riscv64-gnu": ["@napi-rs/canvas-linux-riscv64-gnu@1.0.2", "", { "os": "linux", "cpu": "none" }, "sha512-fXRjnPihdnbO6qy1QQOgxAonb68A0TCEG7rj1x7v7rxNElsE8EVIKIEUTvyDtU+sthYSbX+8e7g3oZiLGnOmxw=="],
"@napi-rs/canvas-linux-x64-gnu": ["@napi-rs/canvas-linux-x64-gnu@1.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-nPR97DXhbWIAy7yazF3jc06kEPMqYMLmPzFOVNlwKPfIoSChnI+x7dc0hTLaihz3jxrjL6j4BbA7earxfx4X3g=="],
"@napi-rs/canvas-linux-x64-musl": ["@napi-rs/canvas-linux-x64-musl@1.0.2", "", { "os": "linux", "cpu": "x64" }, "sha512-l7zZY5+jL5qnBZtDz7CoBtY6p7EkHu422g/0zWwrOrzIwWyWxZFRfZZORY1UG7YApymPLx+UbOkN206xXn/c1Q=="],
"@napi-rs/canvas-win32-arm64-msvc": ["@napi-rs/canvas-win32-arm64-msvc@1.0.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-yE0koHCFF4PIbMc2o2SEALhnipz7WBISh5glLvQiomtIoCcW0np3H4Lw93ceJAfJttTTeIIWFbwH84F7EVzjMQ=="],
"@napi-rs/canvas-win32-x64-msvc": ["@napi-rs/canvas-win32-x64-msvc@1.0.2", "", { "os": "win32", "cpu": "x64" }, "sha512-okU8/t2foV6C31n0GtvEMbfD5rOFc70+/6xUNME9Guld29sgSOIGUEDScAWFlcP3k5TYQRl9TNkwJEEjh15w8A=="],
"@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.4", "", { "dependencies": { "@tybys/wasm-util": "^0.10.1" }, "peerDependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1" } }, "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow=="],
"@noble/hashes": ["@noble/hashes@2.2.0", "", {}, "sha512-IYqDGiTXab6FniAgnSdZwgWbomxpy9FtYvLKs7wCUs2a8RkITG+DFGO1DM9cr+E3/RgADRpFjrKVaJ1z6sjtEg=="],

1
packages/simulation/package.json
View file

@ -16,6 +16,7 @@
"typecheck": "tsgo --noEmit"
},
"dependencies": {
"@napi-rs/canvas": "1.0.2",
"@opencode-ai/core": "workspace:*",
"@opencode-ai/llm": "workspace:*",
"@opentui/core": "catalog:",

4
packages/simulation/src/backend/control.ts
View file

@ -1,7 +1,6 @@
import { Effect } from "effect"
import { SimulationProtocol } from "../protocol"
import { SimulationLLMExchange } from "./llm-exchange"
import { SimulationNetwork } from "./network"
/**
* Backend-hosted simulation control WebSocket.
@ -19,7 +18,6 @@ import { SimulationNetwork } from "./network"
* - `llm.finish` { id, reason? } finish an exchange
* - `llm.disconnect` { id } abruptly terminate an exchange without a finish
* - `llm.pending` list open exchanges
* - `network.log` simulated network request log
*/
type ControlSocket = Bun.ServerWebSocket<{ unsubscribe?: () => void }>
@ -58,8 +56,6 @@ async function handle(socket: ControlSocket, request: SimulationProtocol.Backend
}
case "llm.pending":
return { exchanges: SimulationLLMExchange.pending() }
case "network.log":
return { entries: SimulationNetwork.log() }
}
}

103
packages/simulation/src/frontend/actions.ts
View file

@ -1,8 +1,11 @@
import type { CliRenderer, Renderable } from "@opentui/core"
import { mkdtemp } from "node:fs/promises"
import { tmpdir } from "node:os"
import { join } from "node:path"
import type { CapturedFrame, CliRenderer, Renderable } from "@opentui/core"
import { createMockKeys, createMockMouse, type MockInput, type MockMouse } from "@opentui/core/testing"
import type { SimulationProtocol } from "../protocol"
import { SimulationRenderer } from "./renderer"
import { SimulationTrace } from "./trace"
import { SimulationPng } from "./png"
export type Action = SimulationProtocol.Frontend.Action
export type Element = SimulationProtocol.Frontend.Element
@ -91,57 +94,95 @@ export function elements(renderer: CliRenderer): Element[] {
.filter((element) => element.focusable || element.clickable || element.editor)
}
export function actions(renderer: CliRenderer, options: { text?: string } = {}): Action[] {
const items = elements(renderer)
return [
...(renderer.currentFocusedEditor
? ([{ type: "typeText", text: options.text ?? "hello" }, { type: "pressEnter" }] satisfies Action[])
: []),
...items.filter((item) => item.focusable && !item.focused).map((item) => ({ type: "focus" as const, target: item.num })),
...items
.filter((item) => item.clickable)
.map((item) => ({
type: "click" as const,
target: item.num,
x: Math.floor(item.x + item.width / 2),
y: Math.floor(item.y + item.height / 2),
})),
{ type: "pressArrow", direction: "down" },
{ type: "pressArrow", direction: "up" },
]
}
export function state(harness: Harness) {
return {
screen: harness.screen(),
focused: {
renderable: harness.renderer.currentFocusedRenderable?.num,
editor: Boolean(harness.renderer.currentFocusedEditor),
},
elements: elements(harness.renderer),
actions: actions(harness.renderer),
}
}
export async function screenshot(harness: Harness) {
await harness.renderOnce()
const image = SimulationPng.screenshot(harness.renderer)
const path = join(await mkdtemp(join(tmpdir(), "opencode-drive-")), "screenshot.png")
await Bun.write(path, image.data)
return path
}
export function frame(harness: Harness): CapturedFrame {
const buffer = harness.renderer.currentRenderBuffer
return {
cols: buffer.width,
rows: buffer.height,
cursor: [0, 0],
lines: buffer.getSpanLines().map((line) => ({
spans: line.spans.map((span) => ({
text: span.text,
fg: span.fg,
bg: span.bg,
attributes: span.attributes,
width: span.width,
})),
})),
}
}
export async function video(frames: CapturedFrame[]) {
const directory = await mkdtemp(join(tmpdir(), "opencode-drive-recording-"))
await Promise.all(
frames.map((frame, index) =>
Bun.write(
join(directory, `frame-${index.toString().padStart(6, "0")}.png`),
SimulationPng.screenshotFrame(frame).data,
),
),
)
const path = join(directory, "recording.mp4")
const process = Bun.spawn(
[
"ffmpeg",
"-loglevel",
"error",
"-framerate",
"10",
"-i",
join(directory, "frame-%06d.png"),
"-c:v",
"libx264",
"-pix_fmt",
"yuv420p",
"-movflags",
"+faststart",
"-y",
path,
],
{ stderr: "pipe" },
)
if ((await process.exited) !== 0) throw new Error(`ffmpeg failed: ${await new Response(process.stderr).text()}`)
return path
}
export async function execute(harness: Harness, action: Action) {
SimulationTrace.add("ui.action", { action })
switch (action.type) {
case "typeText":
case "ui.type":
await harness.mockInput.typeText(action.text)
break
case "pressKey":
case "ui.press":
harness.mockInput.pressKey(action.key, action.modifiers)
break
case "pressEnter":
case "ui.enter":
harness.mockInput.pressEnter()
break
case "pressArrow":
case "ui.arrow":
harness.mockInput.pressArrow(action.direction)
break
case "focus":
case "ui.focus":
all(harness.renderer.root).find((item) => item.num === action.target)?.focus()
break
case "click":
case "ui.click":
await harness.mockMouse.click(action.x, action.y)
break
}

84
packages/simulation/src/frontend/png.ts Normal file
View file

@ -0,0 +1,84 @@
import { fileURLToPath } from "node:url"
import { GlobalFonts, createCanvas } from "@napi-rs/canvas"
import { TextAttributes, type CapturedFrame, type CliRenderer, type RGBA } from "@opentui/core"
const CellWidth = 10
const CellHeight = 20
const FontSize = 16
const FontFamily = "OpenCode Screenshot"
GlobalFonts.registerFromPath(
fileURLToPath(new URL("../../../ui/src/assets/fonts/JetBrainsMonoNerdFontMono-Regular.woff2", import.meta.url)),
FontFamily,
)
export function screenshot(renderer: CliRenderer) {
return screenshotFrame({
cols: renderer.currentRenderBuffer.width,
rows: renderer.currentRenderBuffer.height,
cursor: [0, 0],
lines: renderer.currentRenderBuffer.getSpanLines(),
})
}
export function screenshotFrame(frame: CapturedFrame) {
const canvas = createCanvas(frame.cols * CellWidth, frame.rows * CellHeight)
const context = canvas.getContext("2d")
context.fillStyle = "#080808"
context.fillRect(0, 0, canvas.width, canvas.height)
context.textBaseline = "top"
frame.lines.forEach((line, row) => {
let column = 0
line.spans.forEach((span) => {
const attributes = span.attributes & 0xff
const inverse = Boolean(attributes & TextAttributes.INVERSE)
const hidden = Boolean(attributes & TextAttributes.HIDDEN)
const foreground = inverse ? span.bg : span.fg
const background = inverse ? span.fg : span.bg
const chars = [...span.text]
let remaining = span.width
chars.forEach((char, index) => {
const cells = Math.max(1, remaining - (chars.length - index - 1))
if (background.a) {
context.fillStyle = color(background)
context.fillRect(column * CellWidth, row * CellHeight, cells * CellWidth, CellHeight)
}
if (!hidden && char.codePointAt(0) !== 0x0a00) {
context.fillStyle = color(foreground, attributes & TextAttributes.DIM ? 0.55 : 1)
context.font = `${attributes & TextAttributes.ITALIC ? "italic " : ""}${attributes & TextAttributes.BOLD ? "bold " : ""}${FontSize}px "${FontFamily}"`
context.fillText(char, column * CellWidth, row * CellHeight + 1)
if (attributes & TextAttributes.UNDERLINE) {
context.fillRect(column * CellWidth, row * CellHeight + 17, cells * CellWidth, 1)
}
if (attributes & TextAttributes.STRIKETHROUGH) {
context.fillRect(column * CellWidth, row * CellHeight + 10, cells * CellWidth, 1)
}
}
column += cells
remaining -= cells
})
while (remaining-- > 0) {
if (background.a) {
context.fillStyle = color(background)
context.fillRect(column * CellWidth, row * CellHeight, CellWidth, CellHeight)
}
column++
}
})
})
return {
width: canvas.width,
height: canvas.height,
data: canvas.toBuffer("image/png"),
}
}
function color(value: RGBA, opacity = 1) {
const [red, green, blue, alpha] = value.toInts()
return `rgba(${red}, ${green}, ${blue}, ${(alpha / 255) * opacity})`
}
export * as SimulationPng from "./png"

94
packages/simulation/src/frontend/server.ts
View file

@ -1,6 +1,6 @@
import type { CapturedFrame } from "@opentui/core"
import { SimulationProtocol } from "../protocol"
import { SimulationActions, type Harness } from "./actions"
import { SimulationTrace } from "./trace"
export interface Server {
readonly url: string
@ -11,47 +11,90 @@ function parseRequest(input: string | Buffer) {
return SimulationProtocol.Frontend.decodeRequest(JSON.parse(typeof input === "string" ? input : input.toString()))
}
async function handle(harness: Harness, request: SimulationProtocol.Frontend.Request, headless: boolean) {
interface Recording {
readonly frames: CapturedFrame[]
readonly timer: ReturnType<typeof setInterval>
pending: Promise<void>
}
async function handle(
harness: Harness,
request: SimulationProtocol.Frontend.Request,
recording: { current?: Recording },
headless: boolean,
) {
switch (request.method) {
case "ui.screenshot":
return SimulationActions.screenshot(harness)
case "ui.state": {
if (headless) await harness.renderOnce()
const result = SimulationActions.state(harness)
SimulationTrace.add("ui.state", { elements: result.elements.length, actions: result.actions.length })
return result
return SimulationActions.state(harness)
}
case "ui.action":
return SimulationActions.execute(harness, request.params.action)
case "trace.list":
return { records: SimulationTrace.list() }
case "trace.clear":
SimulationTrace.clear()
return { cleared: true }
case "trace.export":
return SimulationTrace.exportTrace()
case "ui.start-record": {
if (recording.current) throw new Error("UI recording is already active")
const frames = [SimulationActions.frame(harness)]
const current: Recording = {
frames,
timer: setInterval(() => {
current.pending = current.pending.then(async () => {
if (headless) await harness.renderOnce()
frames.push(SimulationActions.frame(harness))
})
}, 100),
pending: Promise.resolve(),
}
recording.current = current
return { recording: true }
}
case "ui.end-record": {
if (!recording.current) throw new Error("UI recording is not active")
const current = recording.current
clearInterval(current.timer)
await current.pending
if (headless) await harness.renderOnce()
current.frames.push(SimulationActions.frame(harness))
recording.current = undefined
return SimulationActions.video(current.frames)
}
case "ui.type":
return SimulationActions.execute(harness, { type: "ui.type", text: request.params.text })
case "ui.enter":
return SimulationActions.execute(harness, { type: "ui.enter" })
case "ui.press":
return SimulationActions.execute(harness, {
type: "ui.press",
key: request.params.key,
modifiers: request.params.modifiers,
})
case "ui.arrow":
return SimulationActions.execute(harness, { type: "ui.arrow", direction: request.params.direction })
case "ui.focus":
return SimulationActions.execute(harness, { type: "ui.focus", target: request.params.target })
case "ui.click":
return SimulationActions.execute(harness, {
type: "ui.click",
target: request.params.target,
x: request.params.x,
y: request.params.y,
})
}
}
export function start(harness: Harness, endpoint: string, headless: boolean): Server {
const url = new URL(endpoint)
const server = Bun.serve<{ readonly drive: true }>({
const recording: { current?: Recording } = {}
const server = Bun.serve<{ readonly drive: true; readonly headless: boolean }>({
hostname: url.hostname,
port: Number(url.port),
fetch(request, server) {
if (server.upgrade(request, { data: { drive: true } })) return undefined
if (server.upgrade(request, { data: { drive: true, headless } })) return undefined
return new Response("opencode drive ui websocket", { status: 426 })
},
websocket: {
open() {
SimulationTrace.add("control.connect")
},
close() {
SimulationTrace.add("control.disconnect")
},
async message(socket, message) {
let request: SimulationProtocol.Frontend.Request | undefined
try {
request = parseRequest(message)
const result = await handle(harness, request, headless)
const result = await handle(harness, request, recording, headless)
const next = SimulationProtocol.JsonRpc.success(request.id, result)
if (next) socket.send(JSON.stringify(next))
} catch (error) {
@ -60,11 +103,10 @@ export function start(harness: Harness, endpoint: string, headless: boolean): Se
},
},
})
SimulationTrace.add("control.start", { url: endpoint })
return {
url: endpoint,
stop: () => {
SimulationTrace.add("control.stop", { url: endpoint })
if (recording.current) clearInterval(recording.current.timer)
server.stop(true)
},
}

37
packages/simulation/src/frontend/trace.ts
View file

@ -1,37 +0,0 @@
export type TraceRecord = {
readonly id: number
readonly time: string
readonly type: string
readonly data?: unknown
}
const records: TraceRecord[] = []
let nextID = 0
export function add(type: string, data?: unknown) {
const record = {
id: ++nextID,
time: new Date().toISOString(),
type,
...(data === undefined ? {} : { data }),
} satisfies TraceRecord
records.push(record)
return record
}
export function list() {
return [...records]
}
export function clear() {
records.length = 0
nextID = 0
}
export function exportTrace() {
return {
records: list(),
}
}
export * as SimulationTrace from "./trace"

65
packages/simulation/src/protocol/index.ts
View file

@ -59,12 +59,12 @@ export namespace Frontend {
export interface KeyModifiers extends Schema.Schema.Type<typeof KeyModifiers> {}
export const Action = Schema.Union([
Schema.Struct({ type: Schema.Literal("typeText"), text: Schema.String }),
Schema.Struct({ type: Schema.Literal("pressKey"), key: Schema.String, modifiers: Schema.optional(KeyModifiers) }),
Schema.Struct({ type: Schema.Literal("pressEnter") }),
Schema.Struct({ type: Schema.Literal("pressArrow"), direction: Schema.Literals(["up", "down", "left", "right"]) }),
Schema.Struct({ type: Schema.Literal("focus"), target: Schema.Number }),
Schema.Struct({ type: Schema.Literal("click"), target: Schema.Number, x: Schema.Number, y: Schema.Number }),
Schema.Struct({ type: Schema.Literal("ui.type"), text: Schema.String }),
Schema.Struct({ type: Schema.Literal("ui.press"), key: Schema.String, modifiers: Schema.optional(KeyModifiers) }),
Schema.Struct({ type: Schema.Literal("ui.enter") }),
Schema.Struct({ type: Schema.Literal("ui.arrow"), direction: Schema.Literals(["up", "down", "left", "right"]) }),
Schema.Struct({ type: Schema.Literal("ui.focus"), target: Schema.Number }),
Schema.Struct({ type: Schema.Literal("ui.click"), target: Schema.Number, x: Schema.Number, y: Schema.Number }),
])
export type Action = Schema.Schema.Type<typeof Action>
@ -83,39 +83,58 @@ export namespace Frontend {
export interface Element extends Schema.Schema.Type<typeof Element> {}
export const State = Schema.Struct({
screen: Schema.String,
focused: Schema.Struct({
renderable: Schema.optional(Schema.Number),
editor: Schema.Boolean,
}),
elements: Schema.Array(Element),
actions: Schema.Array(Action),
})
export interface State extends Schema.Schema.Type<typeof State> {}
export const ActionParams = Schema.Struct({ action: Action })
export interface ActionParams extends Schema.Schema.Type<typeof ActionParams> {}
export const Screenshot = Schema.String
export type Screenshot = Schema.Schema.Type<typeof Screenshot>
export const StartRecord = Schema.Struct({ recording: Schema.Literal(true) })
export interface StartRecord extends Schema.Schema.Type<typeof StartRecord> {}
export const EndRecord = Schema.String
export type EndRecord = Schema.Schema.Type<typeof EndRecord>
export const TypeParams = Schema.Struct({ text: Schema.String })
export interface TypeParams extends Schema.Schema.Type<typeof TypeParams> {}
export const PressParams = Schema.Struct({ key: Schema.String, modifiers: Schema.optional(KeyModifiers) })
export interface PressParams extends Schema.Schema.Type<typeof PressParams> {}
export const ArrowParams = Schema.Struct({ direction: Schema.Literals(["up", "down", "left", "right"]) })
export interface ArrowParams extends Schema.Schema.Type<typeof ArrowParams> {}
export const FocusParams = Schema.Struct({ target: Schema.Number })
export interface FocusParams extends Schema.Schema.Type<typeof FocusParams> {}
export const ClickParams = Schema.Struct({ target: Schema.Number, x: Schema.Number, y: Schema.Number })
export interface ClickParams extends Schema.Schema.Type<typeof ClickParams> {}
export const Request = Schema.Union([
Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.action"), params: ActionParams }),
Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.type"), params: TypeParams }),
Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.press"), params: PressParams }),
Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.arrow"), params: ArrowParams }),
Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.focus"), params: FocusParams }),
Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.click"), params: ClickParams }),
Schema.Struct({
...JsonRpc.RequestFields,
method: Schema.Literals(["ui.state", "trace.list", "trace.clear", "trace.export"]),
method: Schema.Literals([
"ui.enter",
"ui.screenshot",
"ui.state",
"ui.start-record",
"ui.end-record",
]),
}),
])
export type Request = Schema.Schema.Type<typeof Request>
export const decodeRequest = Schema.decodeUnknownSync(Request)
export const TraceRecord = Schema.Struct({
id: Schema.Number,
time: Schema.String,
type: Schema.String,
data: Schema.optional(Schema.Json),
})
export interface TraceRecord extends Schema.Schema.Type<typeof TraceRecord> {}
export const TraceList = Schema.Struct({ records: Schema.Array(TraceRecord) })
export interface TraceList extends Schema.Schema.Type<typeof TraceList> {}
}
export namespace Backend {
@ -148,7 +167,7 @@ export namespace Backend {
Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("llm.disconnect"), params: DisconnectParams }),
Schema.Struct({
...JsonRpc.RequestFields,
method: Schema.Literals(["llm.attach", "llm.pending", "network.log"]),
method: Schema.Literals(["llm.attach", "llm.pending"]),
}),
])
export type Request = Schema.Schema.Type<typeof Request>