feat(simulation): expose and record real mouse input (#47194)

packages/protocol/src/simulation.ts

@@ -203,6 +203,8 @@ export namespace Frontend {
     "ui.focus",
     "ui.click",
     "ui.click.semantic",
+    "ui.mouse",
+    "ui.recording.pointer",
     "ui.resize",
     "ui.matches",
     "ui.state",
@@ -228,12 +230,39 @@ export namespace Frontend {
   })
   export interface SemanticClickTarget extends Schema.Schema.Type<typeof SemanticClickTarget> {}
 
+  const MousePosition = {
+    x: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
+    y: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)),
+    modifiers: Schema.optionalKey(
+      Schema.Struct({
+        shift: Schema.optionalKey(Schema.Boolean),
+        alt: Schema.optionalKey(Schema.Boolean),
+        ctrl: Schema.optionalKey(Schema.Boolean),
+      }),
+    ),
+  }
+  export const MouseParams = Schema.Union([
+    Schema.Struct({ ...MousePosition, action: Schema.Literal("move") }),
+    Schema.Struct({
+      ...MousePosition,
+      action: Schema.Literals(["down", "up"]),
+      button: Schema.optionalKey(Schema.Literals(["left", "middle", "right"])),
+    }),
+    Schema.Struct({
+      ...MousePosition,
+      action: Schema.Literal("scroll"),
+      direction: Schema.Literals(["up", "down", "left", "right"]),
+    }),
+  ])
+  export type MouseParams = Schema.Schema.Type<typeof MouseParams>
+
   export const Action = Schema.Union([
     Schema.Struct({ type: Schema.Literal("ui.type"), text: Schema.String }),
     Schema.Struct({ type: Schema.Literal("ui.press"), key: Schema.String, modifiers: Schema.optional(KeyModifiers) }),
     Schema.Struct({ type: Schema.Literal("ui.enter") }),
     Schema.Struct({ type: Schema.Literal("ui.arrow"), direction: Schema.Literals(["up", "down", "left", "right"]) }),
     Schema.Struct({ type: Schema.Literal("ui.focus"), target: Schema.Number }),
+    Schema.Struct({ type: Schema.Literal("ui.mouse"), params: MouseParams }),
     Schema.Struct({
       type: Schema.Literal("ui.click"),
       target: Schema.Number,
@@ -375,6 +404,7 @@ export namespace Frontend {
     Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.arrow"), params: ArrowParams }),
     Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.focus"), params: FocusParams }),
     Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.click"), params: ClickParams }),
+    Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.mouse"), params: MouseParams }),
     Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.resize"), params: ResizeParams }),
     Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.matches"), params: MatchesParams }),
     Schema.Struct({
@@ -631,6 +661,7 @@ export const UiRpcs = RpcGroup.make(
   request("ui.arrow", { payload: Frontend.ArrowParams, success: Frontend.State }),
   request("ui.focus", { payload: Frontend.FocusParams, success: Frontend.State }),
   request("ui.click", { payload: Frontend.ClickParams, success: Frontend.State }),
+  request("ui.mouse", { payload: Frontend.MouseParams, success: Frontend.State }),
   request("ui.resize", { payload: Frontend.ResizeParams, success: Frontend.State }),
 )
 

packages/simulation/src/frontend/actions.ts

@@ -184,6 +184,32 @@ export const execute = Effect.fn("SimulationActions.execute")(function* (harness
         .find((item) => item.num === action.target)
         ?.focus()
       break
+    case "ui.mouse": {
+      const params = action.params
+      if (params.x >= harness.renderer.width || params.y >= harness.renderer.height)
+        return yield* Effect.fail(new Error("mouse position must be within the terminal viewport"))
+      const options = { modifiers: params.modifiers }
+      SimulationRenderer.recordPointer(harness.renderer, params.action, params.x, params.y)
+      switch (params.action) {
+        case "move":
+          yield* Effect.tryPromise(() => harness.mockMouse.moveTo(params.x, params.y, options))
+          break
+        case "down":
+          yield* Effect.tryPromise(() =>
+            harness.mockMouse.pressDown(params.x, params.y, mouseButton(params.button), options),
+          )
+          break
+        case "up":
+          yield* Effect.tryPromise(() =>
+            harness.mockMouse.release(params.x, params.y, mouseButton(params.button), options),
+          )
+          break
+        case "scroll":
+          yield* Effect.tryPromise(() => harness.mockMouse.scroll(params.x, params.y, params.direction, options))
+          break
+      }
+      break
+    }
     case "ui.click": {
       const target = all(harness.renderer.root).find((item) => item.num === action.target)
       if (!target || !target.visible || target.isDestroyed)
@@ -206,6 +232,7 @@ export const execute = Effect.fn("SimulationActions.execute")(function* (harness
         action.y >= target.height
       )
         return yield* Effect.fail(new Error("click position must be within the target element"))
+      SimulationRenderer.recordPointer(harness.renderer, "click", target.screenX + action.x, target.screenY + action.y)
       yield* Effect.tryPromise(() => harness.mockMouse.click(target.screenX + action.x, target.screenY + action.y))
       break
     }
@@ -227,3 +254,7 @@ export const execute = Effect.fn("SimulationActions.execute")(function* (harness
 })
 
 export * as SimulationActions from "./actions"
+
+function mouseButton(button: "left" | "middle" | "right" = "left") {
+  return ({ left: 0, middle: 1, right: 2 } as const)[button]
+}

packages/simulation/src/frontend/renderer.ts

@@ -1,7 +1,7 @@
 import type { CliRenderer, CliRendererConfig } from "@opentui/core"
 import { createTestRenderer, type TestRendererSetup } from "@opentui/core/testing"
 import { Effect } from "effect"
-import { Timeline } from "../recording"
+import { Timeline, type Pointer } from "../recording"
 
 const setups = new WeakMap<CliRenderer, TestRendererSetup>()
 const recordings = new WeakMap<CliRenderer, Timeline>()
@@ -64,6 +64,10 @@ export function recordResize(renderer: CliRenderer, cols: number, rows: number)
   recordings.get(renderer)?.resize(cols, rows)
 }
 
+export function recordPointer(renderer: CliRenderer, action: Pointer["action"], x: number, y: number) {
+  recordings.get(renderer)?.pointer(action, x, y)
+}
+
 export function setupFor(renderer: CliRenderer): TestRendererSetup | undefined {
   return setups.get(renderer)
 }

packages/simulation/src/frontend/server.ts

@@ -47,6 +47,8 @@ function handle(harness: Harness, request: SimulationProtocol.Frontend.Request,
         y: request.params.y,
         semantic: request.params.semantic,
       })
+    case "ui.mouse":
+      return SimulationActions.execute(harness, { type: "ui.mouse", params: request.params })
     case "ui.resize":
       return SimulationActions.execute(harness, {
         type: "ui.resize",

packages/simulation/src/recording.ts

@@ -32,6 +32,14 @@ export interface Resize extends Schema.Schema.Type<typeof Resize> {}
 export const Event = Schema.Union([Header, Output, Resize])
 export type Event = Schema.Schema.Type<typeof Event>
 
+export const Pointer = Schema.Struct({
+  atMs: Schema.Number,
+  action: Schema.Literals(["move", "down", "up", "click", "scroll"]),
+  x: Schema.Number,
+  y: Schema.Number,
+})
+export interface Pointer extends Schema.Schema.Type<typeof Pointer> {}
+
 export class Timeline extends Writable {
   readonly isTTY = true
   readonly path: string
@@ -41,6 +49,7 @@ export class Timeline extends Writable {
   private readonly started = performance.now()
   private readonly timestamps: number[] = []
   private done?: Promise<string>
+  private pointers?: WriteStream
 
   private constructor(path: string, cols: number, rows: number, output: WriteStream) {
     super()
@@ -95,14 +104,27 @@ export class Timeline extends Writable {
   override _final(callback: (error?: Error | null) => void) {
     this.writeOutput(Buffer.alloc(0), this.elapsed(), (error) => {
       if (error) return callback(error)
-      this.output.end(callback)
+      this.output.end()
+      this.pointers?.end()
+      void Promise.all(this.streams().map((stream) => finished(stream, { cleanup: true }))).then(
+        () => callback(),
+        callback,
+      )
     })
   }
 
+  override _destroy(error: Error | null, callback: (error: Error | null) => void) {
+    const streams = this.streams()
+    const closed = streams.map((stream) => finished(stream, { cleanup: true }))
+    streams.forEach((stream) => stream.destroy())
+    // Destroy joins both children even when one failed before finish() began.
+    void Promise.allSettled(closed).then(() => callback(error))
+  }
+
   finish() {
     if (this.done) return this.done
     this.end()
-    this.done = finished(this).then(() => this.path)
+    this.done = finished(this, { cleanup: true }).then(() => this.path)
     return this.done
   }
 
@@ -112,6 +134,21 @@ export class Timeline extends Writable {
     this.output.write(`${JSON.stringify(event)}\n`)
   }
 
+  // Input and terminal output share one monotonic clock. A sidecar leaves
+  // the existing terminal timeline readable by older Drive releases.
+  pointer(action: Pointer["action"], x: number, y: number) {
+    if (this.writableEnded || this.destroyed) return
+    if (!this.pointers) {
+      this.pointers = createWriteStream(`${this.path.replace(/\.jsonl$/, "")}.pointers.jsonl`)
+      this.pointers.on("error", (error) => this.destroy(error))
+    }
+    this.pointers.write(`${JSON.stringify({ atMs: this.elapsed(), action, x, y } satisfies Pointer)}\n`)
+  }
+
+  private streams() {
+    return this.pointers ? [this.output, this.pointers] : [this.output]
+  }
+
   private elapsed() {
     return Math.max(0, Math.round(performance.now() - this.started))
   }

packages/simulation/test/actions.test.ts

@@ -103,6 +103,47 @@ test("clicks a target at relative coordinates through descendant text", async ()
   )
 })
 
+test("mouse input drives native hover, drag, buttons and scrolling at absolute coordinates", async () => {
+  await Effect.runPromise(
+    Effect.scoped(
+      Effect.gen(function* () {
+        const renderer = yield* SimulationRenderer.create({})
+        const events: Array<{ type: string; x: number; y: number; button: number }> = []
+        const button = new BoxRenderable(renderer, {
+          position: "absolute",
+          left: 10,
+          top: 5,
+          width: 15,
+          height: 3,
+          onMouse: (event) => events.push({ type: event.type, x: event.x, y: event.y, button: event.button }),
+        })
+        renderer.root.add(button)
+        const harness = createHarness(renderer)
+        yield* Effect.promise(() => harness.renderOnce())
+        yield* execute(harness, { type: "ui.mouse", params: { action: "move", x: 11, y: 6 } })
+        yield* execute(harness, { type: "ui.mouse", params: { action: "move", x: 12, y: 6 } })
+        expect(events.map((event) => event.type)).toContain("over")
+        expect(events).toContainEqual(expect.objectContaining({ type: "move", x: 12, y: 6 }))
+        yield* execute(harness, { type: "ui.mouse", params: { action: "down", x: 12, y: 6, button: "right" } })
+        yield* execute(harness, { type: "ui.mouse", params: { action: "move", x: 13, y: 6 } })
+        yield* execute(harness, { type: "ui.mouse", params: { action: "up", x: 13, y: 6, button: "right" } })
+        expect(events).toContainEqual(expect.objectContaining({ type: "down", button: 2 }))
+        expect(events).toContainEqual(expect.objectContaining({ type: "drag", x: 13, y: 6 }))
+        expect(events).toContainEqual(expect.objectContaining({ type: "up", x: 13, y: 6, button: 2 }))
+        expect(harness.mockMouse.getPressedButtons()).toEqual([])
+        yield* execute(harness, { type: "ui.mouse", params: { action: "scroll", x: 12, y: 6, direction: "down" } })
+        expect(events.map((event) => event.type)).toContain("scroll")
+        yield* execute(harness, { type: "ui.mouse", params: { action: "move", x: 1, y: 1 } })
+        expect(events.map((event) => event.type)).toContain("out")
+        const error = yield* execute(harness, { type: "ui.mouse", params: { action: "move", x: 100, y: 40 } }).pipe(
+          Effect.flip,
+        )
+        expect(error.message).toContain("within the terminal viewport")
+      }),
+    ),
+  )
+})
+
 test("rejects a semantic click when the live identity does not match", async () => {
   await Effect.runPromise(
     Effect.scoped(

packages/simulation/test/recording.test.ts

@@ -1,12 +1,14 @@
 import { expect, test } from "bun:test"
-import { mkdtemp, rm } from "node:fs/promises"
+import { mkdir, mkdtemp, rm } from "node:fs/promises"
+import { WriteStream } from "node:fs"
+import { once } from "node:events"
 import { tmpdir } from "node:os"
 import { join } from "node:path"
 import { TextRenderable } from "@opentui/core"
 import { createHarness, matches } from "../src/frontend/actions"
 import { SimulationRenderer } from "../src/frontend/renderer"
-import { Effect } from "effect"
-import { Timeline, type Event } from "../src/recording"
+import { Effect, Schema } from "effect"
+import { Timeline, Pointer, type Event } from "../src/recording"
 
 test("streams ANSI chunks into a versioned JSONL timeline", async () => {
   const directory = await mkdtemp(join(tmpdir(), "simulation-recording-"))
@@ -37,6 +39,71 @@ test("streams ANSI chunks into a versioned JSONL timeline", async () => {
   }
 })
 
+test("finishes the pointer sidecar on the output clock without changing the v1 timeline", async () => {
+  const directory = await mkdtemp(join(tmpdir(), "simulation-pointer-recording-"))
+  try {
+    const path = join(directory, "timeline.jsonl")
+    const timeline = await Timeline.create(path, 80, 24)
+    timeline.write("before")
+    timeline.pointer("move", 12, 5)
+    timeline.pointer("click", 15, 6)
+    timeline.write("after")
+    const first = timeline.finish()
+    expect(timeline.finish()).toBe(first)
+    expect(await first).toBe(path)
+    timeline.pointer("move", 30, 10)
+    const pointers = (await Bun.file(join(directory, "timeline.pointers.jsonl")).text())
+      .trim()
+      .split("\n")
+      .map((line) => Schema.decodeUnknownSync(Schema.fromJsonString(Pointer))(line))
+    expect(pointers.map(({ action, x, y }) => ({ action, x, y }))).toEqual([
+      { action: "move", x: 12, y: 5 },
+      { action: "click", x: 15, y: 6 },
+    ])
+    const output = (await Bun.file(path).text())
+      .trim()
+      .split("\n")
+      .map((line) => JSON.parse(line) as Event)
+    const firstOutput = output[1]
+    const lastOutput = output.at(-1)
+    if (firstOutput?.type !== "output" || lastOutput?.type !== "output") throw new Error("missing output")
+    expect(pointers[0]?.atMs).toBeGreaterThanOrEqual(firstOutput.at_ms)
+    expect(pointers[1]?.atMs).toBeLessThanOrEqual(lastOutput.at_ms)
+    expect(output.every((event) => ["header", "output", "resize"].includes(event.type))).toBe(true)
+  } finally {
+    await rm(directory, { recursive: true, force: true })
+  }
+})
+
+test.each(["pointer", "output"])("joins both recording streams after an early %s failure", async (failed) => {
+  const directory = await mkdtemp(join(tmpdir(), "simulation-pointer-failure-"))
+  try {
+    const timeline = await Timeline.create(join(directory, "timeline.jsonl"), 80, 24)
+    if (failed === "pointer") await mkdir(join(directory, "timeline.pointers.jsonl"))
+    const error = new Promise<Error>((resolve) => timeline.once("error", resolve))
+    timeline.pointer("move", 10, 5)
+    const output: unknown = Reflect.get(timeline, "output")
+    const pointers: unknown = Reflect.get(timeline, "pointers")
+    if (!(output instanceof WriteStream) || !(pointers instanceof WriteStream)) throw new Error("missing owned streams")
+    if (failed === "output") {
+      if (pointers.pending) await once(pointers, "open")
+      output.destroy(new Error("output failed"))
+    }
+    const failure = await error
+    const finishing = timeline.finish()
+    expect(timeline.finish()).toBe(finishing)
+    await expect(finishing).rejects.toBe(failure)
+    expect(output.closed).toBe(true)
+    expect(pointers.closed).toBe(true)
+    expect(Reflect.get(output, "fd")).toBeNull()
+    expect(Reflect.get(pointers, "fd")).toBeNull()
+    timeline.pointer("move", 99, 99)
+    expect(timeline.finish()).toBe(finishing)
+  } finally {
+    await rm(directory, { recursive: true, force: true })
+  }
+})
+
 test("captures native renderer output and finishes on destroy", async () => {
   const directory = await mkdtemp(join(tmpdir(), "simulation-renderer-recording-"))
   const path = join(directory, "timeline.jsonl")