From 19833ad1fdbdaa355b86bc3d6b1df2681304f933 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Thu, 3 Sep 2026 22:17:33 -0400 Subject: [PATCH] feat(simulation): expose and record real mouse input (#47194) --- packages/protocol/src/simulation.ts | 31 +++++++++ packages/simulation/src/frontend/actions.ts | 31 +++++++++ packages/simulation/src/frontend/renderer.ts | 6 +- packages/simulation/src/frontend/server.ts | 2 + packages/simulation/src/recording.ts | 41 ++++++++++- packages/simulation/test/actions.test.ts | 41 +++++++++++ packages/simulation/test/recording.test.ts | 73 +++++++++++++++++++- 7 files changed, 219 insertions(+), 6 deletions(-) diff --git a/packages/protocol/src/simulation.ts b/packages/protocol/src/simulation.ts index c3a657a2b74..e73f478a85a 100644 --- a/packages/protocol/src/simulation.ts +++ b/packages/protocol/src/simulation.ts @@ -203,6 +203,8 @@ export namespace Frontend { "ui.focus", "ui.click", "ui.click.semantic", + "ui.mouse", + "ui.recording.pointer", "ui.resize", "ui.matches", "ui.state", @@ -228,12 +230,39 @@ export namespace Frontend { }) export interface SemanticClickTarget extends Schema.Schema.Type {} + const MousePosition = { + x: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + y: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + modifiers: Schema.optionalKey( + Schema.Struct({ + shift: Schema.optionalKey(Schema.Boolean), + alt: Schema.optionalKey(Schema.Boolean), + ctrl: Schema.optionalKey(Schema.Boolean), + }), + ), + } + export const MouseParams = Schema.Union([ + Schema.Struct({ ...MousePosition, action: Schema.Literal("move") }), + Schema.Struct({ + ...MousePosition, + action: Schema.Literals(["down", "up"]), + button: Schema.optionalKey(Schema.Literals(["left", "middle", "right"])), + }), + Schema.Struct({ + ...MousePosition, + action: Schema.Literal("scroll"), + direction: Schema.Literals(["up", "down", "left", "right"]), + }), + ]) + export type MouseParams = Schema.Schema.Type + export const Action = Schema.Union([ Schema.Struct({ type: Schema.Literal("ui.type"), text: Schema.String }), Schema.Struct({ type: Schema.Literal("ui.press"), key: Schema.String, modifiers: Schema.optional(KeyModifiers) }), Schema.Struct({ type: Schema.Literal("ui.enter") }), Schema.Struct({ type: Schema.Literal("ui.arrow"), direction: Schema.Literals(["up", "down", "left", "right"]) }), Schema.Struct({ type: Schema.Literal("ui.focus"), target: Schema.Number }), + Schema.Struct({ type: Schema.Literal("ui.mouse"), params: MouseParams }), Schema.Struct({ type: Schema.Literal("ui.click"), target: Schema.Number, @@ -375,6 +404,7 @@ export namespace Frontend { Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.arrow"), params: ArrowParams }), Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.focus"), params: FocusParams }), Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.click"), params: ClickParams }), + Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.mouse"), params: MouseParams }), Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.resize"), params: ResizeParams }), Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.matches"), params: MatchesParams }), Schema.Struct({ @@ -631,6 +661,7 @@ export const UiRpcs = RpcGroup.make( request("ui.arrow", { payload: Frontend.ArrowParams, success: Frontend.State }), request("ui.focus", { payload: Frontend.FocusParams, success: Frontend.State }), request("ui.click", { payload: Frontend.ClickParams, success: Frontend.State }), + request("ui.mouse", { payload: Frontend.MouseParams, success: Frontend.State }), request("ui.resize", { payload: Frontend.ResizeParams, success: Frontend.State }), ) diff --git a/packages/simulation/src/frontend/actions.ts b/packages/simulation/src/frontend/actions.ts index 588801e0d87..03dbed9d504 100644 --- a/packages/simulation/src/frontend/actions.ts +++ b/packages/simulation/src/frontend/actions.ts @@ -184,6 +184,32 @@ export const execute = Effect.fn("SimulationActions.execute")(function* (harness .find((item) => item.num === action.target) ?.focus() break + case "ui.mouse": { + const params = action.params + if (params.x >= harness.renderer.width || params.y >= harness.renderer.height) + return yield* Effect.fail(new Error("mouse position must be within the terminal viewport")) + const options = { modifiers: params.modifiers } + SimulationRenderer.recordPointer(harness.renderer, params.action, params.x, params.y) + switch (params.action) { + case "move": + yield* Effect.tryPromise(() => harness.mockMouse.moveTo(params.x, params.y, options)) + break + case "down": + yield* Effect.tryPromise(() => + harness.mockMouse.pressDown(params.x, params.y, mouseButton(params.button), options), + ) + break + case "up": + yield* Effect.tryPromise(() => + harness.mockMouse.release(params.x, params.y, mouseButton(params.button), options), + ) + break + case "scroll": + yield* Effect.tryPromise(() => harness.mockMouse.scroll(params.x, params.y, params.direction, options)) + break + } + break + } case "ui.click": { const target = all(harness.renderer.root).find((item) => item.num === action.target) if (!target || !target.visible || target.isDestroyed) @@ -206,6 +232,7 @@ export const execute = Effect.fn("SimulationActions.execute")(function* (harness action.y >= target.height ) return yield* Effect.fail(new Error("click position must be within the target element")) + SimulationRenderer.recordPointer(harness.renderer, "click", target.screenX + action.x, target.screenY + action.y) yield* Effect.tryPromise(() => harness.mockMouse.click(target.screenX + action.x, target.screenY + action.y)) break } @@ -227,3 +254,7 @@ export const execute = Effect.fn("SimulationActions.execute")(function* (harness }) export * as SimulationActions from "./actions" + +function mouseButton(button: "left" | "middle" | "right" = "left") { + return ({ left: 0, middle: 1, right: 2 } as const)[button] +} diff --git a/packages/simulation/src/frontend/renderer.ts b/packages/simulation/src/frontend/renderer.ts index b20860c1ca4..ea1a79755a6 100644 --- a/packages/simulation/src/frontend/renderer.ts +++ b/packages/simulation/src/frontend/renderer.ts @@ -1,7 +1,7 @@ import type { CliRenderer, CliRendererConfig } from "@opentui/core" import { createTestRenderer, type TestRendererSetup } from "@opentui/core/testing" import { Effect } from "effect" -import { Timeline } from "../recording" +import { Timeline, type Pointer } from "../recording" const setups = new WeakMap() const recordings = new WeakMap() @@ -64,6 +64,10 @@ export function recordResize(renderer: CliRenderer, cols: number, rows: number) recordings.get(renderer)?.resize(cols, rows) } +export function recordPointer(renderer: CliRenderer, action: Pointer["action"], x: number, y: number) { + recordings.get(renderer)?.pointer(action, x, y) +} + export function setupFor(renderer: CliRenderer): TestRendererSetup | undefined { return setups.get(renderer) } diff --git a/packages/simulation/src/frontend/server.ts b/packages/simulation/src/frontend/server.ts index 75b75cd9d42..ddef9d9c7d2 100644 --- a/packages/simulation/src/frontend/server.ts +++ b/packages/simulation/src/frontend/server.ts @@ -47,6 +47,8 @@ function handle(harness: Harness, request: SimulationProtocol.Frontend.Request, y: request.params.y, semantic: request.params.semantic, }) + case "ui.mouse": + return SimulationActions.execute(harness, { type: "ui.mouse", params: request.params }) case "ui.resize": return SimulationActions.execute(harness, { type: "ui.resize", diff --git a/packages/simulation/src/recording.ts b/packages/simulation/src/recording.ts index 2f236afb91c..a0beffa34b9 100644 --- a/packages/simulation/src/recording.ts +++ b/packages/simulation/src/recording.ts @@ -32,6 +32,14 @@ export interface Resize extends Schema.Schema.Type {} export const Event = Schema.Union([Header, Output, Resize]) export type Event = Schema.Schema.Type +export const Pointer = Schema.Struct({ + atMs: Schema.Number, + action: Schema.Literals(["move", "down", "up", "click", "scroll"]), + x: Schema.Number, + y: Schema.Number, +}) +export interface Pointer extends Schema.Schema.Type {} + export class Timeline extends Writable { readonly isTTY = true readonly path: string @@ -41,6 +49,7 @@ export class Timeline extends Writable { private readonly started = performance.now() private readonly timestamps: number[] = [] private done?: Promise + private pointers?: WriteStream private constructor(path: string, cols: number, rows: number, output: WriteStream) { super() @@ -95,14 +104,27 @@ export class Timeline extends Writable { override _final(callback: (error?: Error | null) => void) { this.writeOutput(Buffer.alloc(0), this.elapsed(), (error) => { if (error) return callback(error) - this.output.end(callback) + this.output.end() + this.pointers?.end() + void Promise.all(this.streams().map((stream) => finished(stream, { cleanup: true }))).then( + () => callback(), + callback, + ) }) } + override _destroy(error: Error | null, callback: (error: Error | null) => void) { + const streams = this.streams() + const closed = streams.map((stream) => finished(stream, { cleanup: true })) + streams.forEach((stream) => stream.destroy()) + // Destroy joins both children even when one failed before finish() began. + void Promise.allSettled(closed).then(() => callback(error)) + } + finish() { if (this.done) return this.done this.end() - this.done = finished(this).then(() => this.path) + this.done = finished(this, { cleanup: true }).then(() => this.path) return this.done } @@ -112,6 +134,21 @@ export class Timeline extends Writable { this.output.write(`${JSON.stringify(event)}\n`) } + // Input and terminal output share one monotonic clock. A sidecar leaves + // the existing terminal timeline readable by older Drive releases. + pointer(action: Pointer["action"], x: number, y: number) { + if (this.writableEnded || this.destroyed) return + if (!this.pointers) { + this.pointers = createWriteStream(`${this.path.replace(/\.jsonl$/, "")}.pointers.jsonl`) + this.pointers.on("error", (error) => this.destroy(error)) + } + this.pointers.write(`${JSON.stringify({ atMs: this.elapsed(), action, x, y } satisfies Pointer)}\n`) + } + + private streams() { + return this.pointers ? [this.output, this.pointers] : [this.output] + } + private elapsed() { return Math.max(0, Math.round(performance.now() - this.started)) } diff --git a/packages/simulation/test/actions.test.ts b/packages/simulation/test/actions.test.ts index 5912660b410..cf8e91a6170 100644 --- a/packages/simulation/test/actions.test.ts +++ b/packages/simulation/test/actions.test.ts @@ -103,6 +103,47 @@ test("clicks a target at relative coordinates through descendant text", async () ) }) +test("mouse input drives native hover, drag, buttons and scrolling at absolute coordinates", async () => { + await Effect.runPromise( + Effect.scoped( + Effect.gen(function* () { + const renderer = yield* SimulationRenderer.create({}) + const events: Array<{ type: string; x: number; y: number; button: number }> = [] + const button = new BoxRenderable(renderer, { + position: "absolute", + left: 10, + top: 5, + width: 15, + height: 3, + onMouse: (event) => events.push({ type: event.type, x: event.x, y: event.y, button: event.button }), + }) + renderer.root.add(button) + const harness = createHarness(renderer) + yield* Effect.promise(() => harness.renderOnce()) + yield* execute(harness, { type: "ui.mouse", params: { action: "move", x: 11, y: 6 } }) + yield* execute(harness, { type: "ui.mouse", params: { action: "move", x: 12, y: 6 } }) + expect(events.map((event) => event.type)).toContain("over") + expect(events).toContainEqual(expect.objectContaining({ type: "move", x: 12, y: 6 })) + yield* execute(harness, { type: "ui.mouse", params: { action: "down", x: 12, y: 6, button: "right" } }) + yield* execute(harness, { type: "ui.mouse", params: { action: "move", x: 13, y: 6 } }) + yield* execute(harness, { type: "ui.mouse", params: { action: "up", x: 13, y: 6, button: "right" } }) + expect(events).toContainEqual(expect.objectContaining({ type: "down", button: 2 })) + expect(events).toContainEqual(expect.objectContaining({ type: "drag", x: 13, y: 6 })) + expect(events).toContainEqual(expect.objectContaining({ type: "up", x: 13, y: 6, button: 2 })) + expect(harness.mockMouse.getPressedButtons()).toEqual([]) + yield* execute(harness, { type: "ui.mouse", params: { action: "scroll", x: 12, y: 6, direction: "down" } }) + expect(events.map((event) => event.type)).toContain("scroll") + yield* execute(harness, { type: "ui.mouse", params: { action: "move", x: 1, y: 1 } }) + expect(events.map((event) => event.type)).toContain("out") + const error = yield* execute(harness, { type: "ui.mouse", params: { action: "move", x: 100, y: 40 } }).pipe( + Effect.flip, + ) + expect(error.message).toContain("within the terminal viewport") + }), + ), + ) +}) + test("rejects a semantic click when the live identity does not match", async () => { await Effect.runPromise( Effect.scoped( diff --git a/packages/simulation/test/recording.test.ts b/packages/simulation/test/recording.test.ts index 00ef08b69d2..b54f6201711 100644 --- a/packages/simulation/test/recording.test.ts +++ b/packages/simulation/test/recording.test.ts @@ -1,12 +1,14 @@ import { expect, test } from "bun:test" -import { mkdtemp, rm } from "node:fs/promises" +import { mkdir, mkdtemp, rm } from "node:fs/promises" +import { WriteStream } from "node:fs" +import { once } from "node:events" import { tmpdir } from "node:os" import { join } from "node:path" import { TextRenderable } from "@opentui/core" import { createHarness, matches } from "../src/frontend/actions" import { SimulationRenderer } from "../src/frontend/renderer" -import { Effect } from "effect" -import { Timeline, type Event } from "../src/recording" +import { Effect, Schema } from "effect" +import { Timeline, Pointer, type Event } from "../src/recording" test("streams ANSI chunks into a versioned JSONL timeline", async () => { const directory = await mkdtemp(join(tmpdir(), "simulation-recording-")) @@ -37,6 +39,71 @@ test("streams ANSI chunks into a versioned JSONL timeline", async () => { } }) +test("finishes the pointer sidecar on the output clock without changing the v1 timeline", async () => { + const directory = await mkdtemp(join(tmpdir(), "simulation-pointer-recording-")) + try { + const path = join(directory, "timeline.jsonl") + const timeline = await Timeline.create(path, 80, 24) + timeline.write("before") + timeline.pointer("move", 12, 5) + timeline.pointer("click", 15, 6) + timeline.write("after") + const first = timeline.finish() + expect(timeline.finish()).toBe(first) + expect(await first).toBe(path) + timeline.pointer("move", 30, 10) + const pointers = (await Bun.file(join(directory, "timeline.pointers.jsonl")).text()) + .trim() + .split("\n") + .map((line) => Schema.decodeUnknownSync(Schema.fromJsonString(Pointer))(line)) + expect(pointers.map(({ action, x, y }) => ({ action, x, y }))).toEqual([ + { action: "move", x: 12, y: 5 }, + { action: "click", x: 15, y: 6 }, + ]) + const output = (await Bun.file(path).text()) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Event) + const firstOutput = output[1] + const lastOutput = output.at(-1) + if (firstOutput?.type !== "output" || lastOutput?.type !== "output") throw new Error("missing output") + expect(pointers[0]?.atMs).toBeGreaterThanOrEqual(firstOutput.at_ms) + expect(pointers[1]?.atMs).toBeLessThanOrEqual(lastOutput.at_ms) + expect(output.every((event) => ["header", "output", "resize"].includes(event.type))).toBe(true) + } finally { + await rm(directory, { recursive: true, force: true }) + } +}) + +test.each(["pointer", "output"])("joins both recording streams after an early %s failure", async (failed) => { + const directory = await mkdtemp(join(tmpdir(), "simulation-pointer-failure-")) + try { + const timeline = await Timeline.create(join(directory, "timeline.jsonl"), 80, 24) + if (failed === "pointer") await mkdir(join(directory, "timeline.pointers.jsonl")) + const error = new Promise((resolve) => timeline.once("error", resolve)) + timeline.pointer("move", 10, 5) + const output: unknown = Reflect.get(timeline, "output") + const pointers: unknown = Reflect.get(timeline, "pointers") + if (!(output instanceof WriteStream) || !(pointers instanceof WriteStream)) throw new Error("missing owned streams") + if (failed === "output") { + if (pointers.pending) await once(pointers, "open") + output.destroy(new Error("output failed")) + } + const failure = await error + const finishing = timeline.finish() + expect(timeline.finish()).toBe(finishing) + await expect(finishing).rejects.toBe(failure) + expect(output.closed).toBe(true) + expect(pointers.closed).toBe(true) + expect(Reflect.get(output, "fd")).toBeNull() + expect(Reflect.get(pointers, "fd")).toBeNull() + timeline.pointer("move", 99, 99) + expect(timeline.finish()).toBe(finishing) + } finally { + await rm(directory, { recursive: true, force: true }) + } +}) + test("captures native renderer output and finishes on destroy", async () => { const directory = await mkdtemp(join(tmpdir(), "simulation-renderer-recording-")) const path = join(directory, "timeline.jsonl")