From a8f7c5ec930b0da42f4e3c0d48825926be5830cc Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Tue, 19 May 2026 17:26:41 -0400 Subject: [PATCH] Run CLI subprocess tests concurrently (#28399) --- packages/opencode/test/cli/run/run-process.test.ts | 8 ++++---- packages/opencode/test/lib/cli-process.ts | 13 +++++++++---- perf/test-suite.md | 1 + 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/packages/opencode/test/cli/run/run-process.test.ts b/packages/opencode/test/cli/run/run-process.test.ts index 031db58531..00d2e64b37 100644 --- a/packages/opencode/test/cli/run/run-process.test.ts +++ b/packages/opencode/test/cli/run/run-process.test.ts @@ -10,7 +10,7 @@ import { cliIt } from "../../lib/cli-process" describe("opencode run (non-interactive subprocess)", () => { // Happy path: prompt completes, output reaches stdout, process exits 0. // If this fails, all the others likely will too — debug here first. - cliIt.live( + cliIt.concurrent( "exits 0 and writes the response to stdout on a successful prompt", ({ llm, opencode }) => Effect.gen(function* () { @@ -27,7 +27,7 @@ describe("opencode run (non-interactive subprocess)", () => { // makes the SDK call surface an error promptly so the process exits nonzero. // We assert nonzero exit AND wall-clock under the harness timeout — a hang // would expire the timeout and produce a different (signal-killed) failure. - cliIt.live( + cliIt.concurrent( "exits nonzero promptly when the model is unknown (regression for #27371)", ({ opencode }) => Effect.gen(function* () { @@ -47,7 +47,7 @@ describe("opencode run (non-interactive subprocess)", () => { // // This is debatable — a future cleanup might flip it to exit 1. If you're // changing this expectation, do it deliberately and say so in the PR. - cliIt.live( + cliIt.concurrent( "mid-stream LLM error still exits 0 today (contract lock-in)", ({ llm, opencode }) => Effect.gen(function* () { @@ -61,7 +61,7 @@ describe("opencode run (non-interactive subprocess)", () => { // --format json puts one JSON object per line on stdout for each emitted // event. Consumers (CI scripts, tooling) parse this stream. Asserts the // shape so a future event-emit change has to update this expectation. - cliIt.live( + cliIt.concurrent( "--format json emits parseable line-delimited JSON to stdout", ({ llm, opencode }) => Effect.gen(function* () { diff --git a/packages/opencode/test/lib/cli-process.ts b/packages/opencode/test/lib/cli-process.ts index 2f659a8fd0..e260242fd3 100644 --- a/packages/opencode/test/lib/cli-process.ts +++ b/packages/opencode/test/lib/cli-process.ts @@ -17,7 +17,7 @@ // builders (`opencode.serve(opts)`, `opencode.acp(opts)`, `opencode.auth(...)`) // without changing the fixture. Long-lived commands like `serve` will need a // different return shape — see the TODO at the bottom of OpencodeCli. -import type { TestOptions } from "bun:test" +import { test, type TestOptions } from "bun:test" import { AppFileSystem } from "@opencode-ai/core/filesystem" import { AppProcess } from "@opencode-ai/core/process" import { Deferred, Duration, Effect, Layer, Queue, Scope, Stream } from "effect" @@ -439,9 +439,9 @@ function expectExit(result: RunResult, expected: number, label = "opencode") { // `it.live(name, () => withCliFixture(fixture))` — one fewer nesting level at // every call site. Use this for any test that needs the opencode CLI fixture. // -// Only `.live` is exposed because subprocess tests must run against the real -// clock — a TestClock-paused environment can't drive a child process. If you -// need `.only` or `.skip`, fall back to `it.live` + `withCliFixture` directly. +// Subprocess tests must run against the real clock — a TestClock-paused +// environment can't drive a child process. If you need `.only` or `.skip`, fall +// back to `it.live` + `withCliFixture` directly. // Body's R is `Scope.Scope | never` so tests can yield* scope-requiring // resources (e.g. `opencode.serve`) without an extra `Effect.scoped` wrapper — // `withCliFixture`'s outer scope is the natural lifetime. @@ -451,4 +451,9 @@ export const cliIt = { body: (input: CliFixture) => Effect.Effect, opts?: number | TestOptions, ) => it.live(name, () => withCliFixture(body), opts), + concurrent: ( + name: string, + body: (input: CliFixture) => Effect.Effect, + opts?: number | TestOptions, + ) => test.concurrent(name, () => Effect.runPromise(Effect.scoped(withCliFixture(body))), opts), } diff --git a/perf/test-suite.md b/perf/test-suite.md index 7123d4f3e3..ba8fc8424c 100644 --- a/perf/test-suite.md +++ b/perf/test-suite.md @@ -84,6 +84,7 @@ Repeated setup work, long sleeps/timeouts, serial integration tests, filesystem/ | Remaining legacy tools config cases can use Effect-aware instance fixtures | Migrated allow/deny legacy `tools` permission cases to `it.instance` | 2.65s | 1.90s | keep | Single baseline before edit; after median from three sequential reruns (2.58, 1.90, 1.90). | | Oversized snapshot batch tests only need to cross the 100-file boundary | Reduced large diff/revert fixture sizes while keeping each case above the batch boundary | 4.32s | 3.66s | keep | Three affected snapshot tests; after median from three reruns (4.32, 3.66, 3.66) while still crossing the 100-file boundary. | | Prompt tests without LLM calls do not need the test LLM server | Added a no-server runner and moved obvious non-LLM prompt/shell cases to it | 25.41s | 21.03s | keep | Full prompt file after simplify pass median from three reruns (20.66, 21.03, 21.64); LLM-backed tests stay on original runner. | +| CLI run subprocess cases can run independently | Marked `run-process.test.ts` subprocess cases concurrent | 11.87s | 4.13s | keep | Newest-dev single baseline; after median from three reruns (4.13, 4.17, 4.11). Each case has an isolated temp home and LLM port. | ## Profiling Results