From 896ad7b88442fdffa5ee85b57aa783b82b78d38a Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Mon, 18 May 2026 12:18:29 -0400 Subject: [PATCH] Speed up targeted opencode tests Reduce avoidable setup costs in slow opencode tests while preserving reviewed coverage and recording the benchmark evidence for follow-up test-suite work. --- packages/opencode/package.json | 2 + packages/opencode/script/bench-test-suite.ts | 52 ++ .../opencode/script/profile-test-files.ts | 42 ++ .../src/cli/cmd/tui/plugin/runtime.ts | 21 +- .../opencode/src/control-plane/workspace.ts | 4 +- .../test/cli/tui/plugin-lifecycle.test.ts | 4 +- packages/opencode/test/config/config.test.ts | 3 - .../test/control-plane/workspace.test.ts | 14 +- packages/opencode/test/fixture/plugin.ts | 10 + .../test/plugin/install-concurrency.test.ts | 6 +- .../opencode/test/provider/provider.test.ts | 135 ++-- .../test/server/httpapi-listen.test.ts | 72 +- .../test/server/httpapi-provider.test.ts | 5 + .../opencode/test/server/httpapi-sdk.test.ts | 7 +- .../test/session/processor-effect.test.ts | 24 +- packages/opencode/test/session/prompt.test.ts | 698 ++++++++---------- packages/opencode/test/tool/skill.test.ts | 90 ++- perf/test-suite.md | 123 +++ 18 files changed, 737 insertions(+), 575 deletions(-) create mode 100644 packages/opencode/script/bench-test-suite.ts create mode 100644 packages/opencode/script/profile-test-files.ts create mode 100644 packages/opencode/test/fixture/plugin.ts create mode 100644 perf/test-suite.md diff --git a/packages/opencode/package.json b/packages/opencode/package.json index b3d7be57da..29762c604b 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -10,6 +10,8 @@ "test": "bun test --timeout 30000", "test:ci": "mkdir -p .artifacts/unit && bun test --timeout 30000 --reporter=junit --reporter-outfile=.artifacts/unit/junit.xml", "test:httpapi": "bun run script/httpapi-exercise.ts --mode coverage --fail-on-missing --fail-on-skip && bun run script/httpapi-exercise.ts --mode auth --fail-on-missing --fail-on-skip && bun run script/httpapi-exercise.ts --mode effect --fail-on-missing --fail-on-skip", + "bench:test": "bun run script/bench-test-suite.ts", + "profile:test": "bun run script/profile-test-files.ts", "build": "bun run script/build.ts", "fix-node-pty": "bun run script/fix-node-pty.ts", "dev": "bun run --conditions=browser ./src/index.ts", diff --git a/packages/opencode/script/bench-test-suite.ts b/packages/opencode/script/bench-test-suite.ts new file mode 100644 index 0000000000..27ff7750c2 --- /dev/null +++ b/packages/opencode/script/bench-test-suite.ts @@ -0,0 +1,52 @@ +// Full-suite timing harness for the test-speed research in ../../perf/test-suite.md. +// Use this for periodic sanity checks; use profile-test-files.ts for discovery. +// Env: BENCH_WARMUPS=0 BENCH_RUNS=1 bun run bench:test +const warmups = Number(Bun.env.BENCH_WARMUPS ?? 0) +const runs = Number(Bun.env.BENCH_RUNS ?? 1) +const timings: number[] = [] + +if (!Number.isInteger(warmups) || warmups < 0) { + console.error("BENCH_WARMUPS must be a non-negative integer") + process.exit(1) +} +if (!Number.isInteger(runs) || runs < 1) { + console.error("BENCH_RUNS must be a positive integer") + process.exit(1) +} + +for (const index of Array.from({ length: warmups + runs }, (_, index) => index)) { + const measured = index >= warmups + const label = measured ? `run ${index - warmups + 1}/${runs}` : `warmup ${index + 1}/${warmups}` + const start = performance.now() + console.log(`bench:test ${label}`) + + const proc = Bun.spawn(["bun", "test", "--timeout", "30000"], { + cwd: import.meta.dir + "/..", + stdout: "inherit", + stderr: "inherit", + env: Bun.env, + }) + + const exitCode = await proc.exited + if (exitCode !== 0) { + console.error(`bench:test failed during ${label} with exit code ${exitCode}`) + process.exit(exitCode) + } + + const seconds = (performance.now() - start) / 1000 + console.log(`bench:test ${label} ${seconds.toFixed(3)}s`) + if (measured) timings.push(seconds) +} + +const sorted = timings.toSorted((a, b) => a - b) +const median = sorted[Math.floor(sorted.length / 2)] +const mean = timings.reduce((sum, timing) => sum + timing, 0) / timings.length +const best = sorted[0] ?? median +const worst = sorted.at(-1) ?? median + +console.log( + `bench:test median=${median.toFixed(3)}s mean=${mean.toFixed(3)}s best=${best.toFixed(3)}s worst=${worst.toFixed(3)}s`, +) +console.log(`METRIC test_suite_seconds=${median.toFixed(3)}`) +console.log(`METRIC test_suite_best_seconds=${best.toFixed(3)}`) +console.log(`METRIC test_suite_worst_seconds=${worst.toFixed(3)}`) diff --git a/packages/opencode/script/profile-test-files.ts b/packages/opencode/script/profile-test-files.ts new file mode 100644 index 0000000000..12b9bccd84 --- /dev/null +++ b/packages/opencode/script/profile-test-files.ts @@ -0,0 +1,42 @@ +// Per-file profiler for finding candidate test-speed work; see ../../perf/test-suite.md +// for the benchmark notes, kept wins, and discarded experiments. +// Example: TEST_PROFILE_GLOB='test/server/**/*.test.ts' TEST_PROFILE_TOP=15 bun run profile:test +const pattern = Bun.env.TEST_PROFILE_GLOB ?? "test/**/*.test.{ts,tsx}" +const limit = Number(Bun.env.TEST_PROFILE_LIMIT ?? 0) +const timeout = Bun.env.TEST_PROFILE_TIMEOUT ?? "30000" +const files = Array.fromAsync(new Bun.Glob(pattern).scan({ cwd: import.meta.dir + "/..", onlyFiles: true })) + .then((files) => files.toSorted()) + .then((files) => (limit > 0 ? files.slice(0, limit) : files)) + +const results = [] +for (const file of await files) { + const start = performance.now() + const proc = Bun.spawn(["bun", "test", "--timeout", timeout, file], { + cwd: import.meta.dir + "/..", + stdout: "pipe", + stderr: "pipe", + env: Bun.env, + }) + const [output, error, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) + const seconds = (performance.now() - start) / 1000 + results.push({ file, seconds, exitCode }) + console.log(`${exitCode === 0 ? "PASS" : "FAIL"} ${seconds.toFixed(3)}s ${file}`) + if (exitCode !== 0) console.log((output + error).trim()) +} + +const sorted = results.toSorted((a, b) => b.seconds - a.seconds) +console.log("\nSlowest test files:") +for (const result of sorted.slice(0, Number(Bun.env.TEST_PROFILE_TOP ?? 20))) { + console.log(`${result.seconds.toFixed(3)}s ${result.exitCode === 0 ? "PASS" : "FAIL"} ${result.file}`) +} + +if (sorted[0]) { + console.log(`METRIC slowest_test_file_seconds=${sorted[0].seconds.toFixed(3)}`) + console.log(`METRIC profiled_test_files=${results.length}`) +} + +if (results.some((result) => result.exitCode !== 0)) process.exit(1) diff --git a/packages/opencode/src/cli/cmd/tui/plugin/runtime.ts b/packages/opencode/src/cli/cmd/tui/plugin/runtime.ts index 2a9ebc4ed2..62f04c9707 100644 --- a/packages/opencode/src/cli/cmd/tui/plugin/runtime.ts +++ b/packages/opencode/src/cli/cmd/tui/plugin/runtime.ts @@ -114,6 +114,7 @@ type RuntimeState = { plugins: PluginEntry[] plugins_by_id: Map pending: Map + dispose_timeout_ms: number } const log = Log.create({ service: "tui.plugin" }) @@ -394,7 +395,7 @@ async function syncPluginThemes(plugin: PluginEntry) { } } -function createPluginScope(load: PluginLoad, id: string) { +function createPluginScope(load: PluginLoad, id: string, disposeTimeoutMs: number) { const ctrl = new AbortController() let list: { key: symbol; fn: TuiDispose }[] = [] let done = false @@ -436,14 +437,14 @@ function createPluginScope(load: PluginLoad, id: string) { ctrl.abort() const queue = [...list].reverse() list = [] - const until = Date.now() + DISPOSE_TIMEOUT_MS + const until = Date.now() + disposeTimeoutMs for (const item of queue) { const left = until - Date.now() if (left <= 0) { fail("timed out cleaning up tui plugin", { path: load.spec, id, - timeout: DISPOSE_TIMEOUT_MS, + timeout: disposeTimeoutMs, }) break } @@ -454,7 +455,7 @@ function createPluginScope(load: PluginLoad, id: string) { fail("timed out cleaning up tui plugin", { path: load.spec, id, - timeout: DISPOSE_TIMEOUT_MS, + timeout: disposeTimeoutMs, }) break } @@ -523,7 +524,7 @@ async function activatePluginEntry(state: RuntimeState, plugin: PluginEntry, per if (persist) writePluginEnabledState(state.api, plugin.id, true) if (plugin.scope) return true - const scope = createPluginScope(plugin.load, plugin.id) + const scope = createPluginScope(plugin.load, plugin.id, state.dispose_timeout_ms) const api = pluginApi(state, plugin, scope, plugin.id) const ok = await Promise.resolve() .then(async () => { @@ -1002,7 +1003,12 @@ let loaded: Promise | undefined let runtime: RuntimeState | undefined export const Slot = View -export async function init(input: { api: HostPluginApi; config: TuiConfig.Resolved; dispose?: () => void }) { +export async function init(input: { + api: HostPluginApi + config: TuiConfig.Resolved + dispose?: () => void + disposeTimeoutMs?: number +}) { const cwd = process.cwd() if (loaded) { if (dir !== cwd) { @@ -1052,7 +1058,7 @@ export async function dispose() { state.dispose?.() } -async function load(input: { api: Api; config: TuiConfig.Resolved; dispose?: () => void }) { +async function load(input: { api: Api; config: TuiConfig.Resolved; dispose?: () => void; disposeTimeoutMs?: number }) { const { api, config } = input const cwd = process.cwd() const slots = setupSlots(api) @@ -1064,6 +1070,7 @@ async function load(input: { api: Api; config: TuiConfig.Resolved; dispose?: () plugins: [], plugins_by_id: new Map(), pending: new Map(), + dispose_timeout_ms: input.disposeTimeoutMs ?? DISPOSE_TIMEOUT_MS, } runtime = next try { diff --git a/packages/opencode/src/control-plane/workspace.ts b/packages/opencode/src/control-plane/workspace.ts index a50df578f9..f4e49ed20b 100644 --- a/packages/opencode/src/control-plane/workspace.ts +++ b/packages/opencode/src/control-plane/workspace.ts @@ -160,6 +160,7 @@ export interface Interface { workspaceID: WorkspaceID, state: Record, signal?: AbortSignal, + timeout?: number, ) => Effect.Effect readonly startWorkspaceSyncing: (projectID: ProjectID) => Effect.Effect } @@ -946,12 +947,13 @@ export const layer = Layer.effect( workspaceID: WorkspaceID, state: Record, signal?: AbortSignal, + timeout = TIMEOUT, ) { if (synced(state)) return yield* Effect.catch( waitEvent({ - timeout: TIMEOUT, + timeout, signal, fn(event) { if (event.workspace !== workspaceID && event.payload.type !== "sync") { diff --git a/packages/opencode/test/cli/tui/plugin-lifecycle.test.ts b/packages/opencode/test/cli/tui/plugin-lifecycle.test.ts index 8725fe8b9b..269c48b605 100644 --- a/packages/opencode/test/cli/tui/plugin-lifecycle.test.ts +++ b/packages/opencode/test/cli/tui/plugin-lifecycle.test.ts @@ -205,10 +205,10 @@ test( const { config, restore } = mockTuiRuntime(tmp.path, [tmp.extra.spec]) try { - await TuiPluginRuntime.init({ api: createTuiPluginApi(), config }) + await TuiPluginRuntime.init({ api: createTuiPluginApi(), config, disposeTimeoutMs: 25 }) const done = await new Promise((resolve) => { - const timer = setTimeout(() => resolve("timeout"), 7000) + const timer = setTimeout(() => resolve("timeout"), 500) void TuiPluginRuntime.dispose().then(() => { clearTimeout(timer) resolve("done") diff --git a/packages/opencode/test/config/config.test.ts b/packages/opencode/test/config/config.test.ts index 4d3ed45d2f..6684eea1a4 100644 --- a/packages/opencode/test/config/config.test.ts +++ b/packages/opencode/test/config/config.test.ts @@ -1101,9 +1101,6 @@ test("installs dependencies in writable OPENCODE_CONFIG_DIR", async () => { }, }) - // TODO: this is a hack to wait for backgruounded gitignore - await new Promise((resolve) => setTimeout(resolve, 1000)) - expect(await Filesystem.exists(path.join(tmp.extra, ".gitignore"))).toBe(true) expect(await Filesystem.readText(path.join(tmp.extra, ".gitignore"))).toContain("package-lock.json") } finally { diff --git a/packages/opencode/test/control-plane/workspace.test.ts b/packages/opencode/test/control-plane/workspace.test.ts index 26784592fb..ae8fb97a51 100644 --- a/packages/opencode/test/control-plane/workspace.test.ts +++ b/packages/opencode/test/control-plane/workspace.test.ts @@ -173,8 +173,12 @@ const startWorkspaceSyncingWithFlag = (projectID: ProjectID, experimentalWorkspa Effect.provide(workspaceLayer(experimentalWorkspaces)), ), ) -const waitForWorkspaceSync = (workspaceID: WorkspaceID, state: Record, signal?: AbortSignal) => - runWorkspace(Workspace.Service.use((workspace) => workspace.waitForSync(workspaceID, state, signal))) +const waitForWorkspaceSync = ( + workspaceID: WorkspaceID, + state: Record, + signal?: AbortSignal, + timeout?: number, +) => runWorkspace(Workspace.Service.use((workspace) => workspace.waitForSync(workspaceID, state, signal, timeout))) function captureGlobalEvents() { const events: GlobalEvent[] = [] @@ -1639,9 +1643,9 @@ describe("workspace waitForSync", () => { await withInstance(async () => { const sessionID = SessionID.descending("ses_wait_timeout") - await expect(waitForWorkspaceSync(WorkspaceID.ascending("wrk_wait_timeout"), { [sessionID]: 1 })).rejects.toThrow( - `Timed out waiting for sync fence: {"${sessionID}":1}`, - ) + await expect( + waitForWorkspaceSync(WorkspaceID.ascending("wrk_wait_timeout"), { [sessionID]: 1 }, undefined, 25), + ).rejects.toThrow(`Timed out waiting for sync fence: {"${sessionID}":1}`) }) }, 7000) }) diff --git a/packages/opencode/test/fixture/plugin.ts b/packages/opencode/test/fixture/plugin.ts new file mode 100644 index 0000000000..2dbb64a5e5 --- /dev/null +++ b/packages/opencode/test/fixture/plugin.ts @@ -0,0 +1,10 @@ +import { mkdir } from "fs/promises" +import path from "path" + +export async function markPluginDependenciesReady(dir: string) { + await mkdir(path.join(dir, "node_modules"), { recursive: true }) + await Bun.write( + path.join(dir, "package-lock.json"), + JSON.stringify({ packages: { "": { dependencies: { "@opencode-ai/plugin": "0.0.0" } } } }), + ) +} diff --git a/packages/opencode/test/plugin/install-concurrency.test.ts b/packages/opencode/test/plugin/install-concurrency.test.ts index dd9f8c9282..6160bd6e25 100644 --- a/packages/opencode/test/plugin/install-concurrency.test.ts +++ b/packages/opencode/test/plugin/install-concurrency.test.ts @@ -66,7 +66,7 @@ describe("plugin.install.concurrent", () => { test("serializes concurrent server config updates across processes", async () => { await using tmp = await tmpdir() const target = await plugin(tmp.path, ["server"]) - const all = mods("mod-server", 12) + const all = mods("mod-server", 6) const out = await Promise.all( all.map((mod) => @@ -89,7 +89,7 @@ describe("plugin.install.concurrent", () => { test("serializes concurrent server+tui config updates across processes", async () => { await using tmp = await tmpdir() const target = await plugin(tmp.path, ["server", "tui"]) - const all = mods("mod-both", 10) + const all = mods("mod-both", 6) const out = await Promise.all( all.map((mod) => @@ -118,7 +118,7 @@ describe("plugin.install.concurrent", () => { await fs.mkdir(path.dirname(cfg), { recursive: true }) await Bun.write(cfg, JSON.stringify({ plugin: ["seed@1.0.0"] }, null, 2)) - const next = mods("mod-json", 8) + const next = mods("mod-json", 5) const out = await Promise.all( next.map((mod) => run({ diff --git a/packages/opencode/test/provider/provider.test.ts b/packages/opencode/test/provider/provider.test.ts index 7ff129b3bb..c109284070 100644 --- a/packages/opencode/test/provider/provider.test.ts +++ b/packages/opencode/test/provider/provider.test.ts @@ -3,6 +3,7 @@ import { mkdir, unlink } from "fs/promises" import path from "path" import { disposeAllInstances, tmpdir, withTestInstance } from "../fixture/fixture" +import { markPluginDependenciesReady } from "../fixture/plugin" import { Global } from "@opencode-ai/core/global" import type { InstanceContext } from "../../src/project/instance-context" import { Plugin } from "../../src/plugin/index" @@ -31,12 +32,12 @@ function rememberEnv(k: string) { const set = (ctx: InstanceContext, k: string, v: string) => { rememberEnv(k) process.env[k] = v - return env.runSync((svc) => svc.set(k, v).pipe(Effect.provideService(InstanceRef, ctx))) + return env.runPromise((svc) => svc.set(k, v).pipe(Effect.provideService(InstanceRef, ctx))) } const remove = (ctx: InstanceContext, k: string) => { rememberEnv(k) delete process.env[k] - return env.runSync((svc) => svc.remove(k).pipe(Effect.provideService(InstanceRef, ctx))) + return env.runPromise((svc) => svc.remove(k).pipe(Effect.provideService(InstanceRef, ctx))) } afterEach(async () => { @@ -96,14 +97,6 @@ async function defaultModel(ctx: InstanceContext) { return run(ctx, (provider) => provider.defaultModel()) } -async function markPluginDependenciesReady(dir: string) { - await mkdir(path.join(dir, "node_modules"), { recursive: true }) - await Bun.write( - path.join(dir, "package-lock.json"), - JSON.stringify({ packages: { "": { dependencies: { "@opencode-ai/plugin": "0.0.0" } } } }), - ) -} - function paid(providers: Awaited>) { const item = providers[ProviderID.make("opencode")] expect(item).toBeDefined() @@ -149,7 +142,7 @@ test("provider loaded from env variable", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const providers = await list(ctx) expect(providers[ProviderID.anthropic]).toBeDefined() // Provider should retain its connection source even if custom loaders @@ -202,7 +195,7 @@ test("disabled_providers excludes provider", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const providers = await list(ctx) expect(providers[ProviderID.anthropic]).toBeUndefined() }, @@ -224,8 +217,8 @@ test("enabled_providers restricts to only listed providers", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") - set(ctx, "OPENAI_API_KEY", "test-openai-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "OPENAI_API_KEY", "test-openai-key") const providers = await list(ctx) expect(providers[ProviderID.anthropic]).toBeDefined() expect(providers[ProviderID.openai]).toBeUndefined() @@ -252,7 +245,7 @@ test("model whitelist filters models for provider", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const providers = await list(ctx) expect(providers[ProviderID.anthropic]).toBeDefined() const models = Object.keys(providers[ProviderID.anthropic].models) @@ -281,7 +274,7 @@ test("model blacklist excludes specific models", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const providers = await list(ctx) expect(providers[ProviderID.anthropic]).toBeDefined() const models = Object.keys(providers[ProviderID.anthropic].models) @@ -314,7 +307,7 @@ test("custom model alias via config", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const providers = await list(ctx) expect(providers[ProviderID.anthropic]).toBeDefined() expect(providers[ProviderID.anthropic].models["my-alias"]).toBeDefined() @@ -469,7 +462,7 @@ test("env variable takes precedence, config merges options", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "env-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "env-api-key") const providers = await list(ctx) expect(providers[ProviderID.anthropic]).toBeDefined() // Config options should be merged @@ -493,7 +486,7 @@ test("getModel returns model for valid provider/model", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const model = await getModel(ProviderID.anthropic, ModelID.make("claude-sonnet-4-20250514"), ctx) expect(model).toBeDefined() expect(String(model.providerID)).toBe("anthropic") @@ -518,7 +511,7 @@ test("getModel throws ModelNotFoundError for invalid model", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") expect(getModel(ProviderID.anthropic, ModelID.make("nonexistent-model"), ctx)).rejects.toThrow() }, }) @@ -569,7 +562,7 @@ test("defaultModel returns first available model when no config set", async () = await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const model = await defaultModel(ctx) expect(model.providerID).toBeDefined() expect(model.modelID).toBeDefined() @@ -592,7 +585,7 @@ test("defaultModel respects config model setting", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const model = await defaultModel(ctx) expect(String(model.providerID)).toBe("anthropic") expect(String(model.modelID)).toBe("claude-sonnet-4-20250514") @@ -725,7 +718,7 @@ test("closest finds model by partial match", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const result = await closest(ProviderID.anthropic, ["sonnet-4"], ctx) expect(result).toBeDefined() expect(String(result?.providerID)).toBe("anthropic") @@ -778,7 +771,7 @@ test("getModel uses realIdByKey for aliased models", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const providers = await list(ctx) expect(providers[ProviderID.anthropic].models["my-sonnet"]).toBeDefined() @@ -891,7 +884,7 @@ test("model inherits properties from existing database model", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const providers = await list(ctx) const model = providers[ProviderID.anthropic].models["claude-sonnet-4-20250514"] expect(model.name).toBe("Custom Name for Sonnet") @@ -917,7 +910,7 @@ test("disabled_providers prevents loading even with env var", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "OPENAI_API_KEY", "test-openai-key") + await set(ctx, "OPENAI_API_KEY", "test-openai-key") const providers = await list(ctx) expect(providers[ProviderID.openai]).toBeUndefined() }, @@ -939,8 +932,8 @@ test("enabled_providers with empty array allows no providers", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") - set(ctx, "OPENAI_API_KEY", "test-openai-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "OPENAI_API_KEY", "test-openai-key") const providers = await list(ctx) expect(Object.keys(providers).length).toBe(0) }, @@ -967,7 +960,7 @@ test("whitelist and blacklist can be combined", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const providers = await list(ctx) expect(providers[ProviderID.anthropic]).toBeDefined() const models = Object.keys(providers[ProviderID.anthropic].models) @@ -1074,7 +1067,7 @@ test("getSmallModel returns appropriate small model", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const model = await getSmallModel(ProviderID.anthropic, ctx) expect(model).toBeDefined() expect(model?.id).toContain("haiku") @@ -1097,7 +1090,7 @@ test("getSmallModel respects config small_model override", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const model = await getSmallModel(ProviderID.anthropic, ctx) expect(model).toBeDefined() expect(String(model?.providerID)).toBe("anthropic") @@ -1121,7 +1114,7 @@ test("getSmallModel ignores invalid config small_model", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") expect(await getSmallModel(ProviderID.anthropic, ctx)).toBeUndefined() }, }) @@ -1164,8 +1157,8 @@ test("multiple providers can be configured simultaneously", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-anthropic-key") - set(ctx, "OPENAI_API_KEY", "test-openai-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-anthropic-key") + await set(ctx, "OPENAI_API_KEY", "test-openai-key") const providers = await list(ctx) expect(providers[ProviderID.anthropic]).toBeDefined() expect(providers[ProviderID.openai]).toBeDefined() @@ -1241,7 +1234,7 @@ test("model alias name defaults to alias key when id differs", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const providers = await list(ctx) expect(providers[ProviderID.anthropic].models["sonnet"].name).toBe("sonnet") }, @@ -1279,7 +1272,7 @@ test("provider with multiple env var options only includes apiKey when single en await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "MULTI_ENV_KEY_1", "test-key") + await set(ctx, "MULTI_ENV_KEY_1", "test-key") const providers = await list(ctx) expect(providers[ProviderID.make("multi-env")]).toBeDefined() // When multiple env options exist, key should NOT be auto-set @@ -1319,7 +1312,7 @@ test("provider with single env var includes apiKey automatically", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "SINGLE_ENV_KEY", "my-api-key") + await set(ctx, "SINGLE_ENV_KEY", "my-api-key") const providers = await list(ctx) expect(providers[ProviderID.make("single-env")]).toBeDefined() // Single env option should auto-set key @@ -1354,7 +1347,7 @@ test("model cost overrides existing cost values", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const providers = await list(ctx) const model = providers[ProviderID.anthropic].models["claude-sonnet-4-20250514"] expect(model.cost.input).toBe(999) @@ -1431,9 +1424,9 @@ test("disabled_providers and enabled_providers interaction", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-anthropic") - set(ctx, "OPENAI_API_KEY", "test-openai") - set(ctx, "GOOGLE_GENERATIVE_AI_API_KEY", "test-google") + await set(ctx, "ANTHROPIC_API_KEY", "test-anthropic") + await set(ctx, "OPENAI_API_KEY", "test-openai") + await set(ctx, "GOOGLE_GENERATIVE_AI_API_KEY", "test-google") const providers = await list(ctx) // anthropic: in enabled, not in disabled = allowed expect(providers[ProviderID.anthropic]).toBeDefined() @@ -1588,7 +1581,7 @@ test("provider env fallback - second env var used if first missing", async () => directory: tmp.path, fn: async (ctx) => { // Only set fallback, not primary - set(ctx, "FALLBACK_KEY", "fallback-api-key") + await set(ctx, "FALLBACK_KEY", "fallback-api-key") const providers = await list(ctx) // Provider should load because fallback env var is set expect(providers[ProviderID.make("fallback-env")]).toBeDefined() @@ -1610,7 +1603,7 @@ test("getModel returns consistent results", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const model1 = await getModel(ProviderID.anthropic, ModelID.make("claude-sonnet-4-20250514"), ctx) const model2 = await getModel(ProviderID.anthropic, ModelID.make("claude-sonnet-4-20250514"), ctx) expect(model1.providerID).toEqual(model2.providerID) @@ -1669,7 +1662,7 @@ test("ModelNotFoundError includes suggestions for typos", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") try { await getModel(ProviderID.anthropic, ModelID.make("claude-sonet-4"), ctx) // typo: sonet instead of sonnet expect(true).toBe(false) // Should not reach here @@ -1695,7 +1688,7 @@ test("ModelNotFoundError for provider includes suggestions", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") try { await getModel(ProviderID.make("antropic"), ModelID.make("claude-sonnet-4"), ctx) // typo: antropic expect(true).toBe(false) // Should not reach here @@ -1721,7 +1714,7 @@ test("ModelNotFoundError suggests catalog models for unloaded providers", async await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - remove(ctx, "OPENCODE_API_KEY") + await remove(ctx, "OPENCODE_API_KEY") try { await getModel(ProviderID.opencode, ModelID.make("claude-haiku-fake-model"), ctx) throw new Error("expected model lookup to fail") @@ -1767,7 +1760,7 @@ test("getProvider returns provider info", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const provider = await getProvider(ProviderID.anthropic, ctx) expect(provider).toBeDefined() expect(String(provider?.id)).toBe("anthropic") @@ -1789,7 +1782,7 @@ test("closest returns undefined when no partial match found", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const result = await closest(ProviderID.anthropic, ["nonexistent-xyz-model"], ctx) expect(result).toBeUndefined() }, @@ -1810,7 +1803,7 @@ test("closest checks multiple query terms in order", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") // First term won't match, second will const result = await closest(ProviderID.anthropic, ["nonexistent", "haiku"], ctx) expect(result).toBeDefined() @@ -1880,7 +1873,7 @@ test("provider options are deeply merged", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const providers = await list(ctx) // Custom options should be merged expect(providers[ProviderID.anthropic].options.timeout).toBe(30000) @@ -2010,7 +2003,7 @@ test("custom model inherits npm package from models.dev provider config", async await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "OPENAI_API_KEY", "test-api-key") + await set(ctx, "OPENAI_API_KEY", "test-api-key") const providers = await list(ctx) const model = providers[ProviderID.openai].models["my-custom-model"] expect(model).toBeDefined() @@ -2043,7 +2036,7 @@ test("custom model inherits api.url from models.dev provider", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "OPENROUTER_API_KEY", "test-api-key") + await set(ctx, "OPENROUTER_API_KEY", "test-api-key") const providers = await list(ctx) expect(providers[ProviderID.openrouter]).toBeDefined() @@ -2174,7 +2167,7 @@ test("model variants are generated for reasoning models", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const providers = await list(ctx) // Claude sonnet 4 has reasoning capability const model = providers[ProviderID.anthropic].models["claude-sonnet-4-20250514"] @@ -2210,7 +2203,7 @@ test("model variants can be disabled via config", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const providers = await list(ctx) const model = providers[ProviderID.anthropic].models["claude-sonnet-4-20250514"] expect(model.variants).toBeDefined() @@ -2251,7 +2244,7 @@ test("model variants can be customized via config", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const providers = await list(ctx) const model = providers[ProviderID.anthropic].models["claude-sonnet-4-20250514"] expect(model.variants!["high"]).toBeDefined() @@ -2288,7 +2281,7 @@ test("disabled key is stripped from variant config", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const providers = await list(ctx) const model = providers[ProviderID.anthropic].models["claude-sonnet-4-20250514"] expect(model.variants!["max"]).toBeDefined() @@ -2324,7 +2317,7 @@ test("all variants can be disabled via config", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const providers = await list(ctx) const model = providers[ProviderID.anthropic].models["claude-sonnet-4-20250514"] expect(model.variants).toBeDefined() @@ -2360,7 +2353,7 @@ test("variant config merges with generated variants", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-api-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-api-key") const providers = await list(ctx) const model = providers[ProviderID.anthropic].models["claude-sonnet-4-20250514"] expect(model.variants!["high"]).toBeDefined() @@ -2396,7 +2389,7 @@ test("variants filtered in second pass for database models", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "OPENAI_API_KEY", "test-api-key") + await set(ctx, "OPENAI_API_KEY", "test-api-key") const providers = await list(ctx) const model = providers[ProviderID.openai].models["gpt-5"] expect(model.variants).toBeDefined() @@ -2498,7 +2491,7 @@ test("Google Vertex: retains baseURL for custom proxy", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "GOOGLE_APPLICATION_CREDENTIALS", "test-creds") + await set(ctx, "GOOGLE_APPLICATION_CREDENTIALS", "test-creds") const providers = await list(ctx) expect(providers[ProviderID.make("vertex-proxy")]).toBeDefined() expect(providers[ProviderID.make("vertex-proxy")].options.baseURL).toBe("https://my-proxy.com/v1") @@ -2541,7 +2534,7 @@ test("Google Vertex: supports OpenAI compatible models", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "GOOGLE_APPLICATION_CREDENTIALS", "test-creds") + await set(ctx, "GOOGLE_APPLICATION_CREDENTIALS", "test-creds") const providers = await list(ctx) const model = providers[ProviderID.make("vertex-openai")].models["gpt-4"] @@ -2565,9 +2558,9 @@ test("cloudflare-ai-gateway loads with env variables", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "CLOUDFLARE_ACCOUNT_ID", "test-account") - set(ctx, "CLOUDFLARE_GATEWAY_ID", "test-gateway") - set(ctx, "CLOUDFLARE_API_TOKEN", "test-token") + await set(ctx, "CLOUDFLARE_ACCOUNT_ID", "test-account") + await set(ctx, "CLOUDFLARE_GATEWAY_ID", "test-gateway") + await set(ctx, "CLOUDFLARE_API_TOKEN", "test-token") const providers = await list(ctx) expect(providers[ProviderID.make("cloudflare-ai-gateway")]).toBeDefined() }, @@ -2595,9 +2588,9 @@ test("cloudflare-ai-gateway forwards config metadata options", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "CLOUDFLARE_ACCOUNT_ID", "test-account") - set(ctx, "CLOUDFLARE_GATEWAY_ID", "test-gateway") - set(ctx, "CLOUDFLARE_API_TOKEN", "test-token") + await set(ctx, "CLOUDFLARE_ACCOUNT_ID", "test-account") + await set(ctx, "CLOUDFLARE_GATEWAY_ID", "test-gateway") + await set(ctx, "CLOUDFLARE_API_TOKEN", "test-token") const providers = await list(ctx) expect(providers[ProviderID.make("cloudflare-ai-gateway")]).toBeDefined() expect(providers[ProviderID.make("cloudflare-ai-gateway")].options.metadata).toEqual({ @@ -2673,8 +2666,10 @@ test("plugin config providers persist after instance dispose", async () => { test("plugin config enabled and disabled providers are honored", async () => { await using tmp = await tmpdir({ init: async (dir) => { - const root = path.join(dir, ".opencode", "plugin") + const configDir = path.join(dir, ".opencode") + const root = path.join(configDir, "plugin") await mkdir(root, { recursive: true }) + await markPluginDependenciesReady(configDir) await Bun.write( path.join(root, "provider-filter.ts"), [ @@ -2696,8 +2691,8 @@ test("plugin config enabled and disabled providers are honored", async () => { await withTestInstance({ directory: tmp.path, fn: async (ctx) => { - set(ctx, "ANTHROPIC_API_KEY", "test-anthropic-key") - set(ctx, "OPENAI_API_KEY", "test-openai-key") + await set(ctx, "ANTHROPIC_API_KEY", "test-anthropic-key") + await set(ctx, "OPENAI_API_KEY", "test-openai-key") const providers = await list(ctx) expect(providers[ProviderID.anthropic]).toBeDefined() expect(providers[ProviderID.openai]).toBeUndefined() diff --git a/packages/opencode/test/server/httpapi-listen.test.ts b/packages/opencode/test/server/httpapi-listen.test.ts index f155521384..39b7100462 100644 --- a/packages/opencode/test/server/httpapi-listen.test.ts +++ b/packages/opencode/test/server/httpapi-listen.test.ts @@ -169,7 +169,7 @@ async function openPtySocket(listener: Awaited> describe("HttpApi Server.listen", () => { testPty("serves HTTP routes and upgrades PTY websocket through Server.listen", async () => { - await using tmp = await tmpdir({ git: true, config: { formatter: false, lsp: false } }) + await using tmp = await tmpdir({ config: { formatter: false, lsp: false } }) const listener = await startListener() let stopped = false try { @@ -330,7 +330,7 @@ describe("HttpApi Server.listen", () => { }) testPty("rejects unsafe PTY ticket mint and connect requests", async () => { - await using tmp = await tmpdir({ git: true, config: { formatter: false, lsp: false } }) + await using tmp = await tmpdir({ config: { formatter: false, lsp: false } }) const listener = await startListener() try { const info = await createCat(listener, tmp.path) @@ -338,6 +338,29 @@ describe("HttpApi Server.listen", () => { expect((await requestTicket(listener, info.id, tmp.path, { ticketHeader: false })).status).toBe(403) expect((await requestTicket(listener, info.id, tmp.path, { origin: "https://evil.example" })).status).toBe(403) + // Regression for #25698: minting without a directory uses the server cwd + // and cannot find a PTY registered in a project directory. + const ambiguous = await fetch(new URL(PtyPaths.connectToken.replace(":ptyID", info.id), listener.url), { + method: "POST", + headers: { authorization: authorization(), "x-opencode-ticket": "1" }, + }) + expect(ambiguous.status).toBe(404) + + const directoryScoped = await fetch( + new URL( + `${PtyPaths.connectToken.replace(":ptyID", info.id)}?directory=${encodeURIComponent(tmp.path)}`, + listener.url, + ), + { + method: "POST", + headers: { authorization: authorization(), "x-opencode-ticket": "1" }, + }, + ) + expect(directoryScoped.status).toBe(200) + const mint = (await directoryScoped.json()) as { ticket: string } + const scopedWs = await openSocket(socketURL(listener, info.id, tmp.path, mint.ticket)) + scopedWs.close(1000) + await expectSocketRejected(socketURL(listener, info.id, tmp.path, "not-a-ticket")) const reusable = await connectTicket(listener, info.id, tmp.path) @@ -358,51 +381,8 @@ describe("HttpApi Server.listen", () => { } }) - // Regression for #25698 (Ope): the app's SDK call to - // `client.pty.connectToken({ ptyID })` originally omitted `directory`, so - // the server resolved the PTY in its own cwd context — where the project - // PTY isn't registered — and returned 404. The fix is to always pass - // `directory` from the app side; this test locks in two contracts: - // 1. Mint without directory cannot find a PTY registered in another dir. - // 2. Mint with the project directory succeeds; the resulting ticket - // consumes cleanly when the WS upgrade carries the same directory. - testPty("PTY connect token requires matching directory across mint and connect", async () => { - await using tmp = await tmpdir({ git: true, config: { formatter: false, lsp: false } }) - const listener = await startListener() - try { - const info = await createCat(listener, tmp.path) - - // Mint without directory — server uses its own cwd, can't find the PTY. - const ambiguous = await fetch(new URL(PtyPaths.connectToken.replace(":ptyID", info.id), listener.url), { - method: "POST", - headers: { authorization: authorization(), "x-opencode-ticket": "1" }, - }) - expect(ambiguous.status).toBe(404) - - // Mint with the project directory — succeeds, ticket binds to that scope. - const scoped = await fetch( - new URL( - `${PtyPaths.connectToken.replace(":ptyID", info.id)}?directory=${encodeURIComponent(tmp.path)}`, - listener.url, - ), - { - method: "POST", - headers: { authorization: authorization(), "x-opencode-ticket": "1" }, - }, - ) - expect(scoped.status).toBe(200) - const mint = (await scoped.json()) as { ticket: string } - - // Same directory on the WS upgrade → consume succeeds. - const ws = await openSocket(socketURL(listener, info.id, tmp.path, mint.ticket)) - ws.close(1000) - } finally { - await stop(listener, "timed out cleaning up directory-scope listener").catch(() => undefined) - } - }) - testPty("keeps PTY websocket tickets optional when server auth is disabled", async () => { - await using tmp = await tmpdir({ git: true, config: { formatter: false, lsp: false } }) + await using tmp = await tmpdir({ config: { formatter: false, lsp: false } }) const listener = await startNoAuthListener() try { const info = await createCat(listener, tmp.path) diff --git a/packages/opencode/test/server/httpapi-provider.test.ts b/packages/opencode/test/server/httpapi-provider.test.ts index 490b947fd9..d11ecc85ec 100644 --- a/packages/opencode/test/server/httpapi-provider.test.ts +++ b/packages/opencode/test/server/httpapi-provider.test.ts @@ -6,6 +6,7 @@ import { Server } from "../../src/server/server" import * as Log from "@opencode-ai/core/util/log" import { resetDatabase } from "../fixture/db" import { TestInstance } from "../fixture/fixture" +import { markPluginDependenciesReady } from "../fixture/plugin" import { testEffect } from "../lib/effect" void Log.init({ print: false }) @@ -118,6 +119,7 @@ function requestCallback(input: { function writeProviderAuthPlugin(dir: string) { return Effect.gen(function* () { const fs = yield* AppFileSystem.Service + yield* Effect.promise(() => markPluginDependenciesReady(path.join(dir, ".opencode"))) yield* fs.writeWithDirs( path.join(dir, ".opencode", "plugin", "provider-oauth-parity.ts"), @@ -152,6 +154,7 @@ function writeProviderAuthPlugin(dir: string) { function writeProviderAuthValidationPlugin(dir: string) { return Effect.gen(function* () { const fs = yield* AppFileSystem.Service + yield* Effect.promise(() => markPluginDependenciesReady(path.join(dir, ".opencode"))) yield* fs.writeWithDirs( path.join(dir, ".opencode", "plugin", "provider-oauth-validation.ts"), @@ -193,6 +196,7 @@ function writeProviderAuthValidationPlugin(dir: string) { function writeFunctionOptionsPlugin(dir: string) { return Effect.gen(function* () { const fs = yield* AppFileSystem.Service + yield* Effect.promise(() => markPluginDependenciesReady(path.join(dir, ".opencode"))) yield* fs.writeWithDirs( path.join(dir, ".opencode", "plugin", "provider-function-options.ts"), @@ -224,6 +228,7 @@ function writeFunctionOptionsPlugin(dir: string) { function writeProviderModelsMutationPlugin(dir: string) { return Effect.gen(function* () { const fs = yield* AppFileSystem.Service + yield* Effect.promise(() => markPluginDependenciesReady(path.join(dir, ".opencode"))) yield* fs.writeWithDirs( path.join(dir, ".opencode", "plugin", "provider-models-mutation.ts"), diff --git a/packages/opencode/test/server/httpapi-sdk.test.ts b/packages/opencode/test/server/httpapi-sdk.test.ts index a0a0d6d240..2d605da3b0 100644 --- a/packages/opencode/test/server/httpapi-sdk.test.ts +++ b/packages/opencode/test/server/httpapi-sdk.test.ts @@ -265,7 +265,7 @@ function withProject( ) { return Effect.gen(function* () { const directory = yield* tmpdirScoped({ - git: options.git ?? true, + git: options.git ?? false, config: { formatter: false, lsp: false, ...options.config }, }) yield* options.setup?.(directory) ?? Effect.void @@ -516,7 +516,7 @@ describe("HttpApi SDK", () => { ) serverPathParity("matches generated SDK instance read routes", (serverPath) => - withStandardProject(serverPath, ({ sdk, directory }) => + withProject(serverPath, { git: true, setup: writeStandardFiles }, ({ sdk, directory }) => Effect.gen(function* () { const project = yield* capture(() => sdk.project.current()) const projects = yield* capture(() => sdk.project.list()) @@ -561,6 +561,7 @@ describe("HttpApi SDK", () => { foundFile: JSON.stringify(findFiles.data).includes("hello.txt"), foundText: JSON.stringify(findText.data ?? null).includes("sdk-parity"), listedFile: JSON.stringify(files.data).includes("hello.txt"), + vcs: { hasBranch: typeof record(vcs.data).branch === "string" }, } }), ), @@ -887,7 +888,7 @@ describe("HttpApi SDK", () => { ) serverPathParity("matches generated SDK project git initialization", (serverPath) => - withProject(serverPath, { git: false }, ({ sdk, directory }) => + withProject(serverPath, {}, ({ sdk, directory }) => Effect.gen(function* () { const before = yield* capture(() => sdk.project.current()) const init = yield* capture(() => sdk.project.initGit()) diff --git a/packages/opencode/test/session/processor-effect.test.ts b/packages/opencode/test/session/processor-effect.test.ts index 78c7e4c642..ae3ca854ca 100644 --- a/packages/opencode/test/session/processor-effect.test.ts +++ b/packages/opencode/test/session/processor-effect.test.ts @@ -249,7 +249,7 @@ it.live("session.processor effect tests capture llm input cleanly", () => expect(calls).toBe(1) expect(parts.some((part) => part.type === "text" && part.text === "hello")).toBe(true) }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -331,7 +331,7 @@ it.live("session.processor effect tests preserve text start time", () => if (!text?.time?.start || !text.time.end) return expect(text.time.start).toBeLessThan(text.time.end) }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -377,7 +377,7 @@ it.live("session.processor effect tests stop after token overflow requests compa expect(parts.some((part) => part.type === "text" && part.text === "after")).toBe(true) expect(parts.some((part) => part.type === "step-finish")).toBe(true) }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -425,7 +425,7 @@ it.live("session.processor effect tests capture reasoning from http mock", () => expect(reasoning?.text).toBe("think") expect(text?.text).toBe("done") }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -472,7 +472,7 @@ it.live("session.processor effect tests reset reasoning state across retries", ( expect(reasoning.some((part) => part.text === "two")).toBe(true) expect(reasoning.some((part) => part.text === "onetwo")).toBe(false) }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -515,7 +515,7 @@ it.live("session.processor effect tests do not retry unknown json errors", () => expect(yield* llm.calls).toBe(1) expect(handle.message.error?.name).toBe("APIError") }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -562,7 +562,7 @@ it.live("session.processor effect tests retry recognized structured json errors" expect(parts.some((part) => part.type === "text" && part.text === "after")).toBe(true) expect(handle.message.error).toBeUndefined() }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -614,7 +614,7 @@ it.live("session.processor effect tests publish retry status updates", () => expect(yield* llm.calls).toBe(2) expect(states).toStrictEqual([1]) }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -657,7 +657,7 @@ it.live("session.processor effect tests compact on structured context overflow", expect(yield* llm.calls).toBe(1) expect(handle.message.error).toBeUndefined() }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -721,7 +721,7 @@ it.live("session.processor effect tests mark pending tools as aborted on cleanup expect(call.state.time.end).toBeDefined() } }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -793,7 +793,7 @@ it.live("session.processor effect tests record aborted errors and idle state", ( expect(state).toMatchObject({ type: "idle" }) expect(errs).toContain("MessageAbortedError") }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -850,6 +850,6 @@ it.live("session.processor effect tests mark interruptions aborted without manua } expect(state).toMatchObject({ type: "idle" }) }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) diff --git a/packages/opencode/test/session/prompt.test.ts b/packages/opencode/test/session/prompt.test.ts index b6a4f8a344..322f30d316 100644 --- a/packages/opencode/test/session/prompt.test.ts +++ b/packages/opencode/test/session/prompt.test.ts @@ -433,317 +433,290 @@ const boot = Effect.fn("test.boot")(function* (input?: { title?: string }) { // Loop semantics -it.instance( - "loop exits immediately when last assistant has stop finish", - () => - Effect.gen(function* () { - const { llm } = yield* useServerConfig(providerCfg) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const chat = yield* sessions.create({ title: "Pinned" }) - yield* seed(chat.id, { finish: "stop" }) +it.instance("loop exits immediately when last assistant has stop finish", () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const chat = yield* sessions.create({ title: "Pinned" }) + yield* seed(chat.id, { finish: "stop" }) - const result = yield* prompt.loop({ sessionID: chat.id }) - expect(result.info.role).toBe("assistant") - if (result.info.role === "assistant") expect(result.info.finish).toBe("stop") - expect(yield* llm.calls).toBe(0) - }), - { git: true }, + const result = yield* prompt.loop({ sessionID: chat.id }) + expect(result.info.role).toBe("assistant") + if (result.info.role === "assistant") expect(result.info.finish).toBe("stop") + expect(yield* llm.calls).toBe(0) + }), ) -it.instance( - "loop calls LLM and returns assistant message", - () => - Effect.gen(function* () { - const { llm } = yield* useServerConfig(providerCfg) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const chat = yield* sessions.create({ - title: "Pinned", - permission: [{ permission: "*", pattern: "*", action: "allow" }], - }) - yield* prompt.prompt({ - sessionID: chat.id, - agent: "build", - noReply: true, - parts: [{ type: "text", text: "hello" }], - }) - yield* llm.text("world") +it.instance("loop calls LLM and returns assistant message", () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const chat = yield* sessions.create({ + title: "Pinned", + permission: [{ permission: "*", pattern: "*", action: "allow" }], + }) + yield* prompt.prompt({ + sessionID: chat.id, + agent: "build", + noReply: true, + parts: [{ type: "text", text: "hello" }], + }) + yield* llm.text("world") - const result = yield* prompt.loop({ sessionID: chat.id }) - expect(result.info.role).toBe("assistant") - const parts = result.parts.filter((p) => p.type === "text") - expect(parts.some((p) => p.type === "text" && p.text === "world")).toBe(true) - expect(yield* llm.hits).toHaveLength(1) - }), - { git: true }, + const result = yield* prompt.loop({ sessionID: chat.id }) + expect(result.info.role).toBe("assistant") + const parts = result.parts.filter((p) => p.type === "text") + expect(parts.some((p) => p.type === "text" && p.text === "world")).toBe(true) + expect(yield* llm.hits).toHaveLength(1) + }), ) -it.instance( - "prompt emits v2 prompted and synthetic events", - () => - Effect.gen(function* () { - yield* useServerConfig(providerCfg) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const chat = yield* sessions.create({ title: "Pinned" }) +it.instance("prompt emits v2 prompted and synthetic events", () => + Effect.gen(function* () { + yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const chat = yield* sessions.create({ title: "Pinned" }) - yield* prompt.prompt({ - sessionID: chat.id, - agent: "build", - noReply: true, - parts: [ - { type: "text", text: "hello v2" }, - { - type: "file", - mime: "text/plain", - filename: "note.txt", - url: "data:text/plain;base64,bm90ZSBjb250ZW50", - }, - ], - }) - - const messages = yield* SessionV2.Service.use((session) => session.messages({ sessionID: chat.id })).pipe( - Effect.provide(SessionV2.layer), - ) - const row = Database.use((db) => - db.select().from(SessionMessageTable).where(Database.eq(SessionMessageTable.session_id, chat.id)).get(), - ) - expect(messages.find((message) => message.type === "user")).toMatchObject({ type: "user", text: "hello v2" }) - expect(typeof row?.data.time.created).toBe("number") - expect(messages).toEqual( - expect.arrayContaining([ - expect.objectContaining({ type: "synthetic", text: expect.stringContaining("Called the Read tool") }), - expect.objectContaining({ type: "synthetic", text: "note content" }), - ]), - ) - }), - { git: true }, -) - -it.instance( - "static loop returns assistant text through local provider", - () => - Effect.gen(function* () { - const { llm } = yield* useServerConfig(providerCfg) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const session = yield* sessions.create({ - title: "Prompt provider", - permission: [{ permission: "*", pattern: "*", action: "allow" }], - }) - - yield* prompt.prompt({ - sessionID: session.id, - agent: "build", - noReply: true, - parts: [{ type: "text", text: "hello" }], - }) - - yield* llm.text("world") - - const result = yield* prompt.loop({ sessionID: session.id }) - expect(result.info.role).toBe("assistant") - expect(result.parts.some((part) => part.type === "text" && part.text === "world")).toBe(true) - expect(yield* llm.hits).toHaveLength(1) - expect(yield* llm.pending).toBe(0) - }), - { git: true }, -) - -it.instance( - "static loop consumes queued replies across turns", - () => - Effect.gen(function* () { - const { llm } = yield* useServerConfig(providerCfg) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const session = yield* sessions.create({ - title: "Prompt provider turns", - permission: [{ permission: "*", pattern: "*", action: "allow" }], - }) - - yield* prompt.prompt({ - sessionID: session.id, - agent: "build", - noReply: true, - parts: [{ type: "text", text: "hello one" }], - }) - - yield* llm.text("world one") - - const first = yield* prompt.loop({ sessionID: session.id }) - expect(first.info.role).toBe("assistant") - expect(first.parts.some((part) => part.type === "text" && part.text === "world one")).toBe(true) - - yield* prompt.prompt({ - sessionID: session.id, - agent: "build", - noReply: true, - parts: [{ type: "text", text: "hello two" }], - }) - - yield* llm.text("world two") - - const second = yield* prompt.loop({ sessionID: session.id }) - expect(second.info.role).toBe("assistant") - expect(second.parts.some((part) => part.type === "text" && part.text === "world two")).toBe(true) - - expect(yield* llm.hits).toHaveLength(2) - expect(yield* llm.pending).toBe(0) - }), - { git: true }, -) - -it.instance( - "loop continues when finish is tool-calls", - () => - Effect.gen(function* () { - const { llm } = yield* useServerConfig(providerCfg) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const session = yield* sessions.create({ - title: "Pinned", - permission: [{ permission: "*", pattern: "*", action: "allow" }], - }) - yield* prompt.prompt({ - sessionID: session.id, - agent: "build", - noReply: true, - parts: [{ type: "text", text: "hello" }], - }) - yield* llm.tool("first", { value: "first" }) - yield* llm.text("second") - - const result = yield* prompt.loop({ sessionID: session.id }) - expect(yield* llm.calls).toBe(2) - expect(result.info.role).toBe("assistant") - if (result.info.role === "assistant") { - expect(result.parts.some((part) => part.type === "text" && part.text === "second")).toBe(true) - expect(result.info.finish).toBe("stop") - } - }), - { git: true }, -) - -it.instance( - "glob tool keeps instance context during prompt runs", - () => - Effect.gen(function* () { - const { dir, llm } = yield* useServerConfig(providerCfg) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const session = yield* sessions.create({ - title: "Glob context", - permission: [{ permission: "*", pattern: "*", action: "allow" }], - }) - const file = path.join(dir, "probe.txt") - yield* writeText(file, "probe") - - yield* prompt.prompt({ - sessionID: session.id, - agent: "build", - noReply: true, - parts: [{ type: "text", text: "find text files" }], - }) - yield* llm.tool("glob", { pattern: "**/*.txt" }) - yield* llm.text("done") - - const result = yield* prompt.loop({ sessionID: session.id }) - expect(result.info.role).toBe("assistant") - - const msgs = yield* MessageV2.filterCompactedEffect(session.id) - const tool = msgs - .flatMap((msg) => msg.parts) - .find( - (part): part is CompletedToolPart => - part.type === "tool" && part.tool === "glob" && part.state.status === "completed", - ) - if (!tool) return - - expect(tool.state.output).toContain(file) - expect(tool.state.output).not.toContain("No context found for instance") - expect(result.parts.some((part) => part.type === "text" && part.text === "done")).toBe(true) - }), - { git: true }, -) - -it.instance( - "loop continues when finish is stop but assistant has tool parts", - () => - Effect.gen(function* () { - const { llm } = yield* useServerConfig(providerCfg) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const session = yield* sessions.create({ - title: "Pinned", - permission: [{ permission: "*", pattern: "*", action: "allow" }], - }) - yield* prompt.prompt({ - sessionID: session.id, - agent: "build", - noReply: true, - parts: [{ type: "text", text: "hello" }], - }) - yield* llm.push(reply().tool("first", { value: "first" }).stop()) - yield* llm.text("second") - - const result = yield* prompt.loop({ sessionID: session.id }) - expect(yield* llm.calls).toBe(2) - expect(result.info.role).toBe("assistant") - if (result.info.role === "assistant") { - expect(result.parts.some((part) => part.type === "text" && part.text === "second")).toBe(true) - expect(result.info.finish).toBe("stop") - } - }), - { git: true }, -) - -it.instance( - "failed subtask preserves metadata on error tool state", - () => - Effect.gen(function* () { - const { llm } = yield* useServerConfig((url) => ({ - ...providerCfg(url), - agent: { - general: { - model: "test/missing-model", - }, + yield* prompt.prompt({ + sessionID: chat.id, + agent: "build", + noReply: true, + parts: [ + { type: "text", text: "hello v2" }, + { + type: "file", + mime: "text/plain", + filename: "note.txt", + url: "data:text/plain;base64,bm90ZSBjb250ZW50", }, - })) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const chat = yield* sessions.create({ title: "Pinned" }) - yield* llm.tool("task", { - description: "inspect bug", - prompt: "look into the cache key path", - subagent_type: "general", - }) - yield* llm.text("done") - const msg = yield* user(chat.id, "hello") - yield* addSubtask(chat.id, msg.id) + ], + }) - const result = yield* prompt.loop({ sessionID: chat.id }) - expect(result.info.role).toBe("assistant") - expect(yield* llm.calls).toBe(2) + const messages = yield* SessionV2.Service.use((session) => session.messages({ sessionID: chat.id })).pipe( + Effect.provide(SessionV2.layer), + ) + const row = Database.use((db) => + db.select().from(SessionMessageTable).where(Database.eq(SessionMessageTable.session_id, chat.id)).get(), + ) + expect(messages.find((message) => message.type === "user")).toMatchObject({ type: "user", text: "hello v2" }) + expect(typeof row?.data.time.created).toBe("number") + expect(messages).toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: "synthetic", text: expect.stringContaining("Called the Read tool") }), + expect.objectContaining({ type: "synthetic", text: "note content" }), + ]), + ) + }), +) - const msgs = yield* MessageV2.filterCompactedEffect(chat.id) - const taskMsg = msgs.find((item) => item.info.role === "assistant" && item.info.agent === "general") - expect(taskMsg?.info.role).toBe("assistant") - if (!taskMsg || taskMsg.info.role !== "assistant") return +it.instance("static loop returns assistant text through local provider", () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const session = yield* sessions.create({ + title: "Prompt provider", + permission: [{ permission: "*", pattern: "*", action: "allow" }], + }) - const tool = errorTool(taskMsg.parts) - if (!tool) return + yield* prompt.prompt({ + sessionID: session.id, + agent: "build", + noReply: true, + parts: [{ type: "text", text: "hello" }], + }) - expect(tool.state.error).toContain("Tool execution failed") - expect(tool.state.metadata).toBeDefined() - expect(tool.state.metadata?.sessionId).toBeDefined() - expect(tool.state.metadata?.model).toEqual({ - providerID: ProviderID.make("test"), - modelID: ModelID.make("missing-model"), - }) - }), - { git: true }, + yield* llm.text("world") + + const result = yield* prompt.loop({ sessionID: session.id }) + expect(result.info.role).toBe("assistant") + expect(result.parts.some((part) => part.type === "text" && part.text === "world")).toBe(true) + expect(yield* llm.hits).toHaveLength(1) + expect(yield* llm.pending).toBe(0) + }), +) + +it.instance("static loop consumes queued replies across turns", () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const session = yield* sessions.create({ + title: "Prompt provider turns", + permission: [{ permission: "*", pattern: "*", action: "allow" }], + }) + + yield* prompt.prompt({ + sessionID: session.id, + agent: "build", + noReply: true, + parts: [{ type: "text", text: "hello one" }], + }) + + yield* llm.text("world one") + + const first = yield* prompt.loop({ sessionID: session.id }) + expect(first.info.role).toBe("assistant") + expect(first.parts.some((part) => part.type === "text" && part.text === "world one")).toBe(true) + + yield* prompt.prompt({ + sessionID: session.id, + agent: "build", + noReply: true, + parts: [{ type: "text", text: "hello two" }], + }) + + yield* llm.text("world two") + + const second = yield* prompt.loop({ sessionID: session.id }) + expect(second.info.role).toBe("assistant") + expect(second.parts.some((part) => part.type === "text" && part.text === "world two")).toBe(true) + + expect(yield* llm.hits).toHaveLength(2) + expect(yield* llm.pending).toBe(0) + }), +) + +it.instance("loop continues when finish is tool-calls", () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const session = yield* sessions.create({ + title: "Pinned", + permission: [{ permission: "*", pattern: "*", action: "allow" }], + }) + yield* prompt.prompt({ + sessionID: session.id, + agent: "build", + noReply: true, + parts: [{ type: "text", text: "hello" }], + }) + yield* llm.tool("first", { value: "first" }) + yield* llm.text("second") + + const result = yield* prompt.loop({ sessionID: session.id }) + expect(yield* llm.calls).toBe(2) + expect(result.info.role).toBe("assistant") + if (result.info.role === "assistant") { + expect(result.parts.some((part) => part.type === "text" && part.text === "second")).toBe(true) + expect(result.info.finish).toBe("stop") + } + }), +) + +it.instance("glob tool keeps instance context during prompt runs", () => + Effect.gen(function* () { + const { dir, llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const session = yield* sessions.create({ + title: "Glob context", + permission: [{ permission: "*", pattern: "*", action: "allow" }], + }) + const file = path.join(dir, "probe.txt") + yield* writeText(file, "probe") + + yield* prompt.prompt({ + sessionID: session.id, + agent: "build", + noReply: true, + parts: [{ type: "text", text: "find text files" }], + }) + yield* llm.tool("glob", { pattern: "**/*.txt" }) + yield* llm.text("done") + + const result = yield* prompt.loop({ sessionID: session.id }) + expect(result.info.role).toBe("assistant") + + const msgs = yield* MessageV2.filterCompactedEffect(session.id) + const tool = msgs + .flatMap((msg) => msg.parts) + .find( + (part): part is CompletedToolPart => + part.type === "tool" && part.tool === "glob" && part.state.status === "completed", + ) + if (!tool) return + + expect(tool.state.output).toContain(file) + expect(tool.state.output).not.toContain("No context found for instance") + expect(result.parts.some((part) => part.type === "text" && part.text === "done")).toBe(true) + }), +) + +it.instance("loop continues when finish is stop but assistant has tool parts", () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const session = yield* sessions.create({ + title: "Pinned", + permission: [{ permission: "*", pattern: "*", action: "allow" }], + }) + yield* prompt.prompt({ + sessionID: session.id, + agent: "build", + noReply: true, + parts: [{ type: "text", text: "hello" }], + }) + yield* llm.push(reply().tool("first", { value: "first" }).stop()) + yield* llm.text("second") + + const result = yield* prompt.loop({ sessionID: session.id }) + expect(yield* llm.calls).toBe(2) + expect(result.info.role).toBe("assistant") + if (result.info.role === "assistant") { + expect(result.parts.some((part) => part.type === "text" && part.text === "second")).toBe(true) + expect(result.info.finish).toBe("stop") + } + }), +) + +it.instance("failed subtask preserves metadata on error tool state", () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig((url) => ({ + ...providerCfg(url), + agent: { + general: { + model: "test/missing-model", + }, + }, + })) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const chat = yield* sessions.create({ title: "Pinned" }) + yield* llm.tool("task", { + description: "inspect bug", + prompt: "look into the cache key path", + subagent_type: "general", + }) + yield* llm.text("done") + const msg = yield* user(chat.id, "hello") + yield* addSubtask(chat.id, msg.id) + + const result = yield* prompt.loop({ sessionID: chat.id }) + expect(result.info.role).toBe("assistant") + expect(yield* llm.calls).toBe(2) + + const msgs = yield* MessageV2.filterCompactedEffect(chat.id) + const taskMsg = msgs.find((item) => item.info.role === "assistant" && item.info.agent === "general") + expect(taskMsg?.info.role).toBe("assistant") + if (!taskMsg || taskMsg.info.role !== "assistant") return + + const tool = errorTool(taskMsg.parts) + if (!tool) return + + expect(tool.state.error).toContain("Tool execution failed") + expect(tool.state.metadata).toBeDefined() + expect(tool.state.metadata?.sessionId).toBeDefined() + expect(tool.state.metadata?.model).toEqual({ + providerID: ProviderID.make("test"), + modelID: ModelID.make("missing-model"), + }) + }), ) it.instance( @@ -778,7 +751,6 @@ it.instance( yield* prompt.cancel(chat.id) yield* Fiber.await(fiber) }), - { git: true }, 5_000, ) @@ -823,7 +795,6 @@ it.instance( yield* prompt.cancel(chat.id) yield* Fiber.await(fiber) }), - { git: true }, 10_000, ) @@ -848,7 +819,6 @@ it.instance( yield* Fiber.await(fiber) expect((yield* status.get(chat.id)).type).toBe("idle") }), - { git: true }, 3_000, ) @@ -877,7 +847,6 @@ it.instance( expect(exit.value.info.role).toBe("assistant") } }), - { git: true }, 3_000, ) @@ -904,7 +873,6 @@ it.instance( } } }), - { git: true }, 3_000, ) @@ -994,7 +962,6 @@ race.instance( expect(lastAssistant.info.parentID).toBe(lastUser?.info.id) } }), - { git: true }, 3_000, ) @@ -1041,7 +1008,7 @@ it.instance( expect(taskMsg.info.time.completed).toBeDefined() expect(taskMsg.info.finish).toBeDefined() }), - { git: true, config: cfg }, + { config: cfg }, 30_000, ) @@ -1077,7 +1044,6 @@ it.instance( expect((yield* status.get(chat.id)).type).toBe("idle") expect((yield* status.get(childID)).type).toBe("idle") }), - { git: true }, 10_000, ) @@ -1111,22 +1077,19 @@ it.instance( // Queue semantics -it.instance( - "concurrent loop callers get same result", - () => - Effect.gen(function* () { - const { prompt, run, chat } = yield* boot() - yield* seed(chat.id, { finish: "stop" }) +it.instance("concurrent loop callers get same result", () => + Effect.gen(function* () { + const { prompt, run, chat } = yield* boot() + yield* seed(chat.id, { finish: "stop" }) - const [a, b] = yield* Effect.all([prompt.loop({ sessionID: chat.id }), prompt.loop({ sessionID: chat.id })], { - concurrency: "unbounded", - }) + const [a, b] = yield* Effect.all([prompt.loop({ sessionID: chat.id }), prompt.loop({ sessionID: chat.id })], { + concurrency: "unbounded", + }) - expect(a.info.id).toBe(b.info.id) - expect(a.info.role).toBe("assistant") - yield* run.assertNotBusy(chat.id) - }), - { git: true }, + expect(a.info.id).toBe(b.info.id) + expect(a.info.role).toBe("assistant") + yield* run.assertNotBusy(chat.id) + }), ) it.instance( @@ -1147,7 +1110,6 @@ it.instance( expect(a.info.id).toBe(b.info.id) expect(a.info.role).toBe("assistant") }), - { git: true }, 3_000, ) @@ -1216,7 +1178,6 @@ it.instance( expect(inputs).toHaveLength(2) expect(JSON.stringify(inputs.at(-1)?.messages)).toContain("second") }), - { git: true }, 3_000, ) @@ -1246,22 +1207,18 @@ it.instance( yield* prompt.cancel(chat.id) yield* Fiber.await(fiber) }), - { git: true }, 3_000, ) -it.instance( - "assertNotBusy succeeds when idle", - () => - Effect.gen(function* () { - const run = yield* SessionRunState.Service - const sessions = yield* Session.Service +it.instance("assertNotBusy succeeds when idle", () => + Effect.gen(function* () { + const run = yield* SessionRunState.Service + const sessions = yield* Session.Service - const chat = yield* sessions.create({}) - const exit = yield* run.assertNotBusy(chat.id).pipe(Effect.exit) - expect(Exit.isSuccess(exit)).toBe(true) - }), - { git: true }, + const chat = yield* sessions.create({}) + const exit = yield* run.assertNotBusy(chat.id).pipe(Effect.exit) + expect(Exit.isSuccess(exit)).toBe(true) + }), ) // Shell semantics @@ -1290,7 +1247,6 @@ it.instance( yield* prompt.cancel(chat.id) yield* Fiber.await(fiber) }), - { git: true }, 3_000, ) @@ -1315,7 +1271,7 @@ unix( expect(tool.state.metadata.output).toContain("err") yield* run.assertNotBusy(chat.id) }), - { git: true, config: cfg }, + { config: cfg }, ) unix( @@ -1339,7 +1295,7 @@ unix( expect(tool.state.metadata.output).toContain(dir) yield* run.assertNotBusy(chat.id) }), - { git: true, config: cfg }, + { config: cfg }, ) unix( @@ -1361,7 +1317,7 @@ unix( expect(tool.state.output).toContain("configured") }), ), - { git: true, config: { ...cfg, shell: "bash" } }, + { config: { ...cfg, shell: "bash" } }, 30_000, ) @@ -1386,7 +1342,7 @@ unix( expect(tool.state.metadata.output).toContain(parent) yield* run.assertNotBusy(chat.id) }), - { git: true, config: cfg }, + { config: cfg }, ) unix( @@ -1412,7 +1368,7 @@ unix( expect(tool.state.metadata.output).toContain("README.md") yield* run.assertNotBusy(chat.id) }), - { git: true, config: cfg }, + { config: cfg }, ) unix( @@ -1434,7 +1390,7 @@ unix( expect(tool.state.metadata.output).toContain("not found") yield* run.assertNotBusy(chat.id) }), - { git: true, config: cfg }, + { config: cfg }, ) unix( @@ -1462,7 +1418,7 @@ unix( expect(Exit.isSuccess(exit)).toBe(true) }), ), - { git: true, config: cfg }, + { config: cfg }, 30_000, ) @@ -1572,7 +1528,6 @@ unix( expect(JSON.stringify(inputs.at(-1)?.messages)).toContain("configured") }), ), - { git: true }, 30_000, ) @@ -1808,7 +1763,7 @@ it.instance( const exit = yield* Fiber.await(fiber) expect(Exit.isFailure(exit)).toBe(true) }), - { git: true, config: cfg }, + { config: cfg }, 30_000, ) @@ -1843,7 +1798,7 @@ it.instance( const exit = yield* Fiber.await(fiber) expect(Exit.isFailure(exit)).toBe(true) }), - { git: true, config: cfg }, + { config: cfg }, 30_000, ) @@ -1882,7 +1837,7 @@ it.instance( yield* sessions.remove(session.id) }), - { git: true, config: cfg }, + { config: cfg }, ) it.instance( @@ -1924,7 +1879,7 @@ it.instance( yield* sessions.remove(session.id) }), - { git: true, config: cfg }, + { config: cfg }, ) it.instance( @@ -1973,7 +1928,6 @@ it.instance( expect(agents.map((agent) => agent.name)).toEqual(["build"]) }), { - git: true, config: { ...cfg, reference: { @@ -2013,7 +1967,6 @@ it.instance( yield* sessions.remove(session.id) }), { - git: true, config: { ...cfg, reference: { @@ -2079,7 +2032,6 @@ it.instance( yield* sessions.remove(session.id) }), { - git: true, config: { ...cfg, reference: { @@ -2128,31 +2080,28 @@ it.instance( // Regression: empty assistant turn loop -it.instance( - "does not loop empty assistant turns for a simple reply", - () => - Effect.gen(function* () { - const { llm } = yield* useServerConfig(providerCfg) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const session = yield* sessions.create({ title: "Prompt regression" }) +it.instance("does not loop empty assistant turns for a simple reply", () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const session = yield* sessions.create({ title: "Prompt regression" }) - yield* llm.text("packages/opencode/src/session/processor.ts") + yield* llm.text("packages/opencode/src/session/processor.ts") - const result = yield* prompt.prompt({ - sessionID: session.id, - agent: "build", - parts: [{ type: "text", text: "Where is SessionProcessor?" }], - }) + const result = yield* prompt.prompt({ + sessionID: session.id, + agent: "build", + parts: [{ type: "text", text: "Where is SessionProcessor?" }], + }) - expect(result.info.role).toBe("assistant") - expect(result.parts.some((part) => part.type === "text" && part.text.includes("processor.ts"))).toBe(true) + expect(result.info.role).toBe("assistant") + expect(result.parts.some((part) => part.type === "text" && part.text.includes("processor.ts"))).toBe(true) - const msgs = yield* sessions.messages({ sessionID: session.id }) - expect(msgs.filter((msg) => msg.info.role === "assistant")).toHaveLength(1) - expect(yield* llm.calls).toBe(1) - }), - { git: true }, + const msgs = yield* sessions.messages({ sessionID: session.id }) + expect(msgs.filter((msg) => msg.info.role === "assistant")).toHaveLength(1) + expect(yield* llm.calls).toBe(1) + }), ) it.instance( @@ -2193,7 +2142,6 @@ it.instance( expect(last.info.error?.name).toBe("MessageAbortedError") } }), - { git: true }, 3_000, ) @@ -2244,7 +2192,6 @@ it.instance( yield* sessions.remove(session.id) }), { - git: true, config: { ...cfg, provider: { @@ -2297,7 +2244,6 @@ it.instance( } } }), - { git: true }, 30_000, ) @@ -2326,7 +2272,6 @@ it.instance( } } }), - { git: true }, 30_000, ) @@ -2356,6 +2301,5 @@ it.instance( } } }), - { git: true }, 30_000, ) diff --git a/packages/opencode/test/tool/skill.test.ts b/packages/opencode/test/tool/skill.test.ts index d1538756e1..bf05fc4ab1 100644 --- a/packages/opencode/test/tool/skill.test.ts +++ b/packages/opencode/test/tool/skill.test.ts @@ -31,14 +31,13 @@ const it = testEffect(Layer.mergeAll(ToolRegistry.defaultLayer, node)) describe("tool.skill", () => { it.live("execute returns skill content block with files", () => - provideTmpdirInstance( - (dir) => - Effect.gen(function* () { - const skill = path.join(dir, ".opencode", "skill", "tool-skill") - yield* Effect.promise(() => - Bun.write( - path.join(skill, "SKILL.md"), - `--- + provideTmpdirInstance((dir) => + Effect.gen(function* () { + const skill = path.join(dir, ".opencode", "skill", "tool-skill") + yield* Effect.promise(() => + Bun.write( + path.join(skill, "SKILL.md"), + `--- name: tool-skill description: Skill for tool tests. --- @@ -47,49 +46,48 @@ description: Skill for tool tests. Use this skill. `, - ), - ) - yield* Effect.promise(() => Bun.write(path.join(skill, "scripts", "demo.txt"), "demo")) + ), + ) + yield* Effect.promise(() => Bun.write(path.join(skill, "scripts", "demo.txt"), "demo")) - const home = process.env.OPENCODE_TEST_HOME - process.env.OPENCODE_TEST_HOME = dir - yield* Effect.addFinalizer(() => + const home = process.env.OPENCODE_TEST_HOME + process.env.OPENCODE_TEST_HOME = dir + yield* Effect.addFinalizer(() => + Effect.sync(() => { + process.env.OPENCODE_TEST_HOME = home + }), + ) + + const registry = yield* ToolRegistry.Service + const agent = { name: "build", mode: "primary" as const, permission: [], options: {} } + const tool = (yield* registry.tools({ + providerID: "opencode" as any, + modelID: "gpt-5" as any, + agent, + })).find((tool) => tool.id === SkillTool.id) + if (!tool) throw new Error("Skill tool not found") + + const requests: Array> = [] + const ctx: Tool.Context = { + ...baseCtx, + ask: (req) => Effect.sync(() => { - process.env.OPENCODE_TEST_HOME = home + requests.push(req) }), - ) + } - const registry = yield* ToolRegistry.Service - const agent = { name: "build", mode: "primary" as const, permission: [], options: {} } - const tool = (yield* registry.tools({ - providerID: "opencode" as any, - modelID: "gpt-5" as any, - agent, - })).find((tool) => tool.id === SkillTool.id) - if (!tool) throw new Error("Skill tool not found") + const result = yield* tool.execute({ name: "tool-skill" }, ctx) + const file = path.resolve(skill, "scripts", "demo.txt") - const requests: Array> = [] - const ctx: Tool.Context = { - ...baseCtx, - ask: (req) => - Effect.sync(() => { - requests.push(req) - }), - } - - const result = yield* tool.execute({ name: "tool-skill" }, ctx) - const file = path.resolve(skill, "scripts", "demo.txt") - - expect(requests.length).toBe(1) - expect(requests[0].permission).toBe("skill") - expect(requests[0].patterns).toContain("tool-skill") - expect(requests[0].always).toContain("tool-skill") - expect(result.metadata.dir).toBe(skill) - expect(result.output).toContain(``) - expect(result.output).toContain(`Base directory for this skill: ${pathToFileURL(skill).href}`) - expect(result.output).toContain(`${file}`) - }), - { git: true }, + expect(requests.length).toBe(1) + expect(requests[0].permission).toBe("skill") + expect(requests[0].patterns).toContain("tool-skill") + expect(requests[0].always).toContain("tool-skill") + expect(result.metadata.dir).toBe(skill) + expect(result.output).toContain(``) + expect(result.output).toContain(`Base directory for this skill: ${pathToFileURL(skill).href}`) + expect(result.output).toContain(`${file}`) + }), ), ) }) diff --git a/perf/test-suite.md b/perf/test-suite.md new file mode 100644 index 0000000000..39c26906df --- /dev/null +++ b/perf/test-suite.md @@ -0,0 +1,123 @@ +# Test Suite Speed + +## Goal + +Speed up the `packages/opencode` test suite without reducing coverage or hiding failures. + +## Benchmark Command + +Run from `packages/opencode`: + +```sh +bun run bench:test +``` + +The full-suite benchmark defaults to one measured run. Use repeated runs only after a targeted win: + +```sh +BENCH_WARMUPS=1 BENCH_RUNS=3 bun run bench:test +``` + +To identify slow files, run: + +```sh +bun run profile:test +``` + +Scope it while exploring: + +```sh +TEST_PROFILE_GLOB='test/server/**/*.test.ts' bun run profile:test +TEST_PROFILE_LIMIT=20 bun run profile:test +``` + +## Primary Metric + +`METRIC test_suite_seconds=` + +## Secondary Metrics + +`test_suite_best_seconds`, `test_suite_worst_seconds`, failures, and noisy spread. + +For profiling: `slowest_test_file_seconds` and the slowest file list. + +## Files In Scope + +`packages/opencode/test/**`, test fixtures, package test scripts, and implementation setup paths only when a benchmarked bottleneck points there. + +## Signals To Watch + +Repeated setup work, long sleeps/timeouts, serial integration tests, filesystem/database fixture costs, and broad test globs pulling unrelated work. + +## Hypothesis Loop + +| Hypothesis | Change | Before | After | Decision | Notes | +| --------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | --------- | ------- | -------- | --------------------------------------------------------------------------------------------------------------------------- | +| Repeated full-suite runs are too expensive for discovery | Switched full-suite benchmark to one run and added per-file profiler | ~250s/run | pending | keep | Bun has no slowest-test reporter in this version; profile files directly. | +| Plugin install concurrency test spends time spawning more workers than needed to exercise lock contention | Reduced worker counts from 12/10/8 to 6/6/5; kept `holdMs: 30` | 7.800s | 6.204s | keep | Median from 3 targeted runs; still covers concurrent cross-process writes to server, server+tui, and existing json config. | +| `httpapi-listen` PTY route tests pay for git repositories they do not assert on | Removed `git: true` from temp dirs while keeping config setup | 10.554s | 7.818s | keep | Median from 3 targeted runs; HTTP routes, tickets, websocket upgrade, restart, and no-auth paths still pass. | +| `workspace.waitForSync` timeout test waits the full production timeout | Added optional timeout parameter defaulting to production timeout; timeout test uses 25ms | 12.949s | 8.305s | keep | Median from 3 targeted runs; production callers keep the 5000ms default. | +| `config.test` waits after dependencies even though `.gitignore` is written synchronously | Removed obsolete 1000ms sleep from writable `OPENCODE_CONFIG_DIR` test | 10.270s | 9.433s | keep | Median from 5 targeted runs because one run was noisy; simpler test and no fixed sleep. | +| SDK parity helpers create git repos for tests that only need files/config/session state | Changed `withProject` default to no git; explicit git init test still opts into no-git fixture | 8.011s | 5.180s | keep | Median from 5 targeted runs because first run was cold/noisy. | +| Provider plugin filter test waits on plugin dependency readiness setup | Marked local plugin dependencies ready using the existing fixture helper | 7.543s | 6.366s | keep | Median from 3 targeted runs; matches neighboring plugin provider test setup. | +| HTTP provider tests generate local plugins without dependency-ready fixture state | Marked generated `.opencode` plugin fixtures dependency-ready | 7.905s | 2.980s | keep | Median from 3 targeted runs; avoids unrelated plugin dependency setup in route tests. | +| TUI plugin lifecycle timeout coverage waits the full production cleanup timeout | Added optional runtime dispose timeout override and used 25ms in the timeout test | 7.330s | 1.507s | keep | Median from 3 targeted runs; production default remains 5000ms. | +| Skill tool test initializes git even though it only reads local skill files | Removed `git: true` from the temporary directory fixture | 2.320s | 1.425s | keep | Single targeted rerun; still exercises skill discovery, permission request, and bundled file output. | +| Prompt shell semantics tests initialize git though they only assert shell/session behavior | Removed `git: true` from shell-focused prompt fixtures while preserving config setup | 26.930s | 23.400s | keep | Three targeted reruns passed after the change: 23.80s, 23.55s, 23.40s. | +| Remaining prompt behavior tests mostly do not require repository state | Removed git setup from safe loop/reference/error fixtures; restored shell queue/cancel cases | 23.400s | 19.610s | keep | Safety review found shell runner readiness depends on git-backed setup in several tests; current single rerun passes. | +| Session processor effect tests do not require repository state | Removed git setup from all processor-effect temp server fixtures | 12.500s | 9.230s | keep | Two targeted reruns passed after the change: 9.61s, 9.23s. | +| HTTP listen PTY ticket tests restart the same listener topology twice | Folded directory-scoped ticket regression into the broader unsafe-ticket test | 7.051s | 6.170s | keep | Two targeted reruns passed after the change: 6.76s, 6.17s; still covers mint failure and successful same-directory upgrade. | + +## Profiling Results + +Command shape: + +```sh +TEST_PROFILE_GLOB='test//**/*.test.ts' TEST_PROFILE_TOP=15 bun run profile:test +``` + +Initial slowest files observed during discovery: + +| File | Seconds | Scope | +| ----------------------------------------- | ------: | ------------- | +| `test/config/config.test.ts` | 23.546 | config | +| `test/provider/provider.test.ts` | 18.747 | provider | +| `test/control-plane/workspace.test.ts` | 16.447 | control-plane | +| `test/plugin/install-concurrency.test.ts` | 14.804 | plugin | +| `test/server/httpapi-cors.test.ts` | 14.620 | server | +| `test/server/httpapi-listen.test.ts` | 10.073 | server | +| `test/server/httpapi-sdk.test.ts` | 8.661 | server | +| `test/server/httpapi-provider.test.ts` | 7.905 | server | +| `test/cli/tui/plugin-lifecycle.test.ts` | 7.330 | cli/tui | +| `test/file/index.test.ts` | 7.214 | file | + +This table is historical profiling input, not the current ranking after kept changes. + +Targeted 3-run baselines: + +| File | Runs | Median | Notes | +| ----------------------------------------- | ---------------------- | -----: | ---------------------------------------------------------------------------- | +| `test/control-plane/workspace.test.ts` | 12.949, 12.949, 12.773 | 12.949 | Stable slow target. | +| `test/server/httpapi-listen.test.ts` | 10.554, 10.631, 10.479 | 10.554 | Stable slow target; WebSocket/listener lifecycle. | +| `test/config/config.test.ts` | 10.270, 9.042, 10.737 | 10.270 | Large serial file; initial 23s was mixed-scope contention/noise. | +| `test/server/httpapi-sdk.test.ts` | 7.600, 8.011, 8.035 | 8.011 | Stable slow target. | +| `test/plugin/install-concurrency.test.ts` | 7.949, 7.800, 7.712 | 7.800 | Stable slow target; many subprocesses. | +| `test/provider/provider.test.ts` | 8.323, 7.543, 7.474 | 7.543 | Large serial file. | +| `test/server/httpapi-cors.test.ts` | 2.621, 1.682, 1.518 | 1.682 | Not a standalone top target; initial 14s was mixed-scope noise/order effect. | + +Full-suite sanity checks: + +| Command | Result | Notes | +| -------------------- | -------: | -------------------------------------------------------------------- | +| `bun run bench:test` | 225.069s | Before continuing prompt/session work. | +| `bun run bench:test` | 186.729s | After prompt, processor, and PTY wins before safety review restores. | +| `bun run bench:test` | 202.317s | After restoring prompt shell coverage and SDK VCS parity coverage. | + +## Dead Ends + +| Hypothesis | Change Tried | Before | After | Decision | Notes | +| ---------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | -----: | -----: | -------- | --------------------------------------------------------------------------------------------- | +| `file/index.test.ts` pays unnecessary per-test global instance cleanup | Removed `afterEach(disposeAllInstances)` while keeping the explicit disposal test import | 5.262s | 5.089s | discard | Improvement was within noise and the cleanup is a safety guard for many instance-state tests. | +| Socket reset retry test can shorten its idle-timeout path | Reduced Bun server idle timeout and tried forced server close | 16.46s | failed | discard | Shorter idle timeout changed the error shape; forced close hung. Keep the real socket reset. | +| `tool/webfetch` can avoid per-test instance setup | Switched local HTTP tests from `it.instance` to `it.live` | 1.219s | failed | discard | Tool execution reads instance-local agent state, so the temp instance is required. | +| LSP client interop tests can shorten coarse request-handling sleeps | Reduced fixed post-notification waits from 100ms to 10ms | 4.270s | 4.740s | discard | First run improved to 3.870s but verification was slower than baseline; not a clear win. |