diff --git a/packages/opencode/test/cli/acp-next/acp-next-process.test.ts b/packages/opencode/test/cli/acp-next/acp-next-process.test.ts deleted file mode 100644 index 5c01929d95..0000000000 --- a/packages/opencode/test/cli/acp-next/acp-next-process.test.ts +++ /dev/null @@ -1,293 +0,0 @@ -import { describe, expect } from "bun:test" -import type { - AuthenticateResponse, - CloseSessionResponse, - InitializeResponse, - LoadSessionResponse, - NewSessionResponse, - PromptResponse, - ResumeSessionResponse, - SessionNotification, - SetSessionConfigOptionResponse, -} from "@agentclientprotocol/sdk" -import { Effect } from "effect" -import { cliIt } from "../../lib/cli-process" -import { testProviderConfig } from "../../lib/test-provider" -import { createAcpClient, expectOk, firstAlternateValue, selectConfigOption } from "../acp/acp-test-client" - -describe("opencode acp-next (subprocess)", () => { - cliIt.live( - "responds to initialize behind OPENCODE_ACP_NEXT", - ({ opencode }) => - Effect.gen(function* () { - const acp = createAcpClient(yield* opencode.acp({ env: { OPENCODE_ACP_NEXT: "1" } })) - const initialized = expectOk( - yield* acp.request("initialize", { - protocolVersion: 1, - clientCapabilities: { _meta: { "terminal-auth": true } }, - }), - ) - - expect(initialized.protocolVersion).toBe(1) - expect(initialized.agentCapabilities?.promptCapabilities?.embeddedContext).toBe(true) - expect(initialized.agentCapabilities?.promptCapabilities?.image).toBe(true) - expect(initialized.agentCapabilities?.mcpCapabilities?.http).toBe(true) - expect(initialized.agentCapabilities?.mcpCapabilities?.sse).toBe(true) - expect(initialized.agentCapabilities?.loadSession).toBe(true) - expect(initialized.agentCapabilities?.sessionCapabilities?.close).toEqual({}) - expect(initialized.agentCapabilities?.sessionCapabilities?.fork).toEqual({}) - expect(initialized.agentCapabilities?.sessionCapabilities?.list).toEqual({}) - expect(initialized.agentCapabilities?.sessionCapabilities?.resume).toEqual({}) - expect(initialized.agentInfo?.name).toBe("OpenCode") - expect(initialized.authMethods?.[0]?.id).toBe("opencode-login") - expect(initialized.authMethods?.[0]?._meta?.["terminal-auth"]).toBeDefined() - }), - 60_000, - ) - - cliIt.live( - "authenticate succeeds for the advertised auth method and rejects unknown methods safely", - ({ opencode }) => - Effect.gen(function* () { - const acp = createAcpClient(yield* opencode.acp({ env: { OPENCODE_ACP_NEXT: "1" } })) - const initialized = expectOk(yield* acp.request("initialize", { protocolVersion: 1 })) - const methodId = initialized.authMethods?.[0]?.id - expect(methodId).toBe("opencode-login") - - expectOk(yield* acp.request("authenticate", { methodId })) - - const rejected = yield* acp.request("authenticate", { methodId: "missing-auth-method" }) - expect(errorCode(rejected.error)).toBe(-32602) - }), - 60_000, - ) - - cliIt.live( - "creates and loads sessions behind OPENCODE_ACP_NEXT", - ({ home, llm, opencode }) => - Effect.gen(function* () { - const acp = createAcpClient( - yield* opencode.acp({ - env: { - OPENCODE_ACP_NEXT: "1", - OPENCODE_CONFIG_CONTENT: JSON.stringify(testProviderConfig(llm.url)), - }, - }), - ) - yield* acp.request("initialize", { protocolVersion: 1 }) - - const session = expectOk(yield* acp.request("session/new", { cwd: home, mcpServers: [] })) - expect(typeof session.sessionId).toBe("string") - expect(selectConfigOption(session.configOptions, "model")?.category).toBe("model") - - const update = yield* acp.waitForNotification( - "session/update", - (params) => - params.sessionId === session.sessionId && params.update.sessionUpdate === "available_commands_update", - ) - expect(update.params?.sessionId).toBe(session.sessionId) - - const loaded = expectOk( - yield* acp.request("session/load", { - cwd: home, - sessionId: session.sessionId, - mcpServers: [], - }), - ) - expect(selectConfigOption(loaded.configOptions, "model")?.category).toBe("model") - - yield* llm.text("hello from acp-next", { usage: { input: 11, output: 7 } }) - const prompted = expectOk( - yield* acp.request("session/prompt", { - sessionId: session.sessionId, - prompt: [{ type: "text", text: "hello" }], - }), - ) - expect(prompted.stopReason).toBe("end_turn") - expect(prompted.usage?.totalTokens).toBeGreaterThan(0) - - const missing = yield* acp.request("session/prompt", { - sessionId: "ses_missing", - prompt: [{ type: "text", text: "hello" }], - }) - expect(errorCode(missing.error)).toBe(-32602) - }), - 60_000, - ) - - cliIt.live( - "switches model through config options behind OPENCODE_ACP_NEXT", - ({ home, llm, opencode }) => - Effect.gen(function* () { - const acp = createAcpClient( - yield* opencode.acp({ - env: { - OPENCODE_ACP_NEXT: "1", - OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)), - }, - }), - ) - yield* acp.request("initialize", { protocolVersion: 1 }) - const session = expectOk(yield* acp.request("session/new", { cwd: home, mcpServers: [] })) - - const updated = expectOk( - yield* acp.request("session/set_config_option", { - sessionId: session.sessionId, - configId: "model", - value: "test/second-model", - }), - ) - - expect(selectConfigOption(updated.configOptions, "model")?.currentValue).toBe("test/second-model") - }), - 60_000, - ) - - cliIt.live( - "switches effort through config options behind OPENCODE_ACP_NEXT", - ({ home, llm, opencode }) => - Effect.gen(function* () { - const acp = createAcpClient( - yield* opencode.acp({ - env: { - OPENCODE_ACP_NEXT: "1", - OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)), - }, - }), - ) - yield* acp.request("initialize", { protocolVersion: 1 }) - const session = expectOk(yield* acp.request("session/new", { cwd: home, mcpServers: [] })) - const effort = selectConfigOption(session.configOptions, "effort") - expect(effort?.category).toBe("thought_level") - const nextEffort = effort ? firstAlternateValue(effort) : undefined - expect(nextEffort).toBe("high") - - const updated = expectOk( - yield* acp.request("session/set_config_option", { - sessionId: session.sessionId, - configId: "effort", - value: nextEffort, - }), - ) - - expect(selectConfigOption(updated.configOptions, "effort")?.currentValue).toBe(nextEffort) - }), - 60_000, - ) - - cliIt.live( - "advertises and supports close behind OPENCODE_ACP_NEXT", - ({ home, llm, opencode }) => - Effect.gen(function* () { - const acp = createAcpClient( - yield* opencode.acp({ - env: { - OPENCODE_ACP_NEXT: "1", - OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)), - }, - }), - ) - const initialized = expectOk(yield* acp.request("initialize", { protocolVersion: 1 })) - expect(initialized.agentCapabilities?.sessionCapabilities?.close).toEqual({}) - const session = expectOk(yield* acp.request("session/new", { cwd: home, mcpServers: [] })) - - expectOk(yield* acp.request("session/close", { sessionId: session.sessionId })) - }), - 60_000, - ) - - cliIt.live( - "advertises and supports resume behind OPENCODE_ACP_NEXT", - ({ home, llm, opencode }) => - Effect.gen(function* () { - const acp = createAcpClient( - yield* opencode.acp({ - env: { - OPENCODE_ACP_NEXT: "1", - OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)), - }, - }), - ) - const initialized = expectOk(yield* acp.request("initialize", { protocolVersion: 1 })) - expect(initialized.agentCapabilities?.sessionCapabilities?.resume).toEqual({}) - const session = expectOk(yield* acp.request("session/new", { cwd: home, mcpServers: [] })) - const resumed = expectOk( - yield* acp.request("session/resume", { - cwd: home, - sessionId: session.sessionId, - mcpServers: [], - }), - ) - - expect(selectConfigOption(resumed.configOptions, "model")?.category).toBe("model") - }), - 60_000, - ) - - cliIt.live( - "exits cleanly when flagged stdin is closed", - ({ opencode }) => - Effect.gen(function* () { - const exitedPromise = yield* Effect.scoped( - Effect.gen(function* () { - const acp = yield* opencode.acp({ env: { OPENCODE_ACP_NEXT: "1" } }) - return acp.exited - }), - ) - - const code = yield* Effect.promise(() => exitedPromise) - expect(typeof code === "number" || code === null).toBe(true) - }), - 60_000, - ) - - cliIt.live( - "default unflagged path still uses production ACP", - ({ opencode }) => - Effect.gen(function* () { - const acp = createAcpClient(yield* opencode.acp()) - const initialized = expectOk(yield* acp.request("initialize", { protocolVersion: 1 })) - - expect(initialized.agentCapabilities?.sessionCapabilities?.close).toEqual({}) - expect(initialized.agentCapabilities?.sessionCapabilities?.resume).toEqual({}) - }), - 60_000, - ) -}) - -function errorCode(error: unknown) { - if (!error || typeof error !== "object") return undefined - if (!("code" in error)) return undefined - return typeof error.code === "number" ? error.code : undefined -} - -function verifierConfig(llmUrl: string) { - const config = testProviderConfig(llmUrl) - return { - ...config, - model: "test/test-model", - provider: { - test: { - ...config.provider.test, - models: { - "test-model": { - ...config.provider.test.models["test-model"], - variants: { - low: {}, - high: {}, - }, - }, - "second-model": { - ...config.provider.test.models["test-model"], - id: "second-model", - name: "Second Test Model", - variants: { - medium: {}, - max: {}, - }, - }, - }, - }, - }, - } -} diff --git a/packages/opencode/test/cli/acp-next/config-options.test.ts b/packages/opencode/test/cli/acp-next/config-options.test.ts new file mode 100644 index 0000000000..1fb69d5ef8 --- /dev/null +++ b/packages/opencode/test/cli/acp-next/config-options.test.ts @@ -0,0 +1,103 @@ +import { describe, expect } from "bun:test" +import type { SetSessionConfigOptionResponse } from "@agentclientprotocol/sdk" +import { Effect } from "effect" +import { cliIt } from "../../lib/cli-process" +import { expectOk, flattenSelectOptions, selectConfigOption } from "../acp/acp-test-client" +import { + createAcpNextClient, + expectAlternateValue, + expectSelectOption, + initialize, + newSession, + verifierConfig, +} from "./helpers" + +describe("opencode acp-next config option subprocess", () => { + cliIt.live( + 'model option is listed with category "model"', + ({ home, llm, opencode }) => + Effect.gen(function* () { + const acp = yield* createAcpNextClient( + { opencode }, + { OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)) }, + ) + yield* initialize(acp) + const model = expectSelectOption((yield* newSession(acp, home)).configOptions, "model") + + expect(model.category).toBe("model") + expect(model.currentValue).toBe("test/test-model") + expect(flattenSelectOptions(model).length).toBeGreaterThanOrEqual(2) + }), + 60_000, + ) + + cliIt.live( + "model switch updates currentValue", + ({ home, llm, opencode }) => + Effect.gen(function* () { + const acp = yield* createAcpNextClient( + { opencode }, + { OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)) }, + ) + yield* initialize(acp) + const session = yield* newSession(acp, home) + const model = expectSelectOption(session.configOptions, "model") + const nextModel = flattenSelectOptions(model).find((option) => option.value === "test/second-model")?.value + expect(nextModel).toBe("test/second-model") + + const updated = expectOk( + yield* acp.request("session/set_config_option", { + sessionId: session.sessionId, + configId: "model", + value: nextModel, + }), + ) + + expect(selectConfigOption(updated.configOptions, "model")?.currentValue).toBe(nextModel) + }), + 60_000, + ) + + cliIt.live( + 'effort option is listed with category "thought_level" when selected model supports variants', + ({ home, llm, opencode }) => + Effect.gen(function* () { + const acp = yield* createAcpNextClient( + { opencode }, + { OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)) }, + ) + yield* initialize(acp) + const effort = expectSelectOption((yield* newSession(acp, home)).configOptions, "effort") + + expect(effort.category).toBe("thought_level") + expect(effort.currentValue).toBe("low") + expect(flattenSelectOptions(effort).map((option) => option.value)).toEqual(["low", "high"]) + }), + 60_000, + ) + + cliIt.live( + "effort switch updates currentValue", + ({ home, llm, opencode }) => + Effect.gen(function* () { + const acp = yield* createAcpNextClient( + { opencode }, + { OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)) }, + ) + yield* initialize(acp) + const session = yield* newSession(acp, home) + const nextEffort = expectAlternateValue(expectSelectOption(session.configOptions, "effort")) + + const updated = expectOk( + yield* acp.request("session/set_config_option", { + sessionId: session.sessionId, + configId: "effort", + value: nextEffort, + }), + ) + + expect(selectConfigOption(updated.configOptions, "effort")?.currentValue).toBe(nextEffort) + }), + 60_000, + ) +}) diff --git a/packages/opencode/test/cli/acp-next/helpers.ts b/packages/opencode/test/cli/acp-next/helpers.ts new file mode 100644 index 0000000000..4bd1bb9e97 --- /dev/null +++ b/packages/opencode/test/cli/acp-next/helpers.ts @@ -0,0 +1,111 @@ +import { expect } from "bun:test" +import type { InitializeResponse, NewSessionResponse, SessionConfigOption } from "@agentclientprotocol/sdk" +import { Effect } from "effect" +import type { CliFixture } from "../../lib/cli-process" +import { testProviderConfig } from "../../lib/test-provider" +import { + createAcpClient, + expectOk, + flattenSelectOptions, + selectConfigOption, + type AcpClient, +} from "../acp/acp-test-client" + +export const diagnosticFirstSessionThresholdMs = 15_000 +export const diagnosticFastPathThresholdMs = 15_000 + +// TODO: tighten to the public verifier target of 500ms once acp-next startup is optimized. +export const finalFirstSessionThresholdMs = 500 +// TODO: tighten warm session/config/skill fast paths to the public verifier target of 100ms. +export const finalFastPathThresholdMs = 100 + +export function createAcpNextClient(input: Pick, env?: Record) { + return Effect.gen(function* () { + return createAcpClient( + yield* input.opencode.acp({ + env: { + OPENCODE_ACP_NEXT: "1", + ...env, + }, + }), + ) + }) +} + +export function initialize(acp: AcpClient) { + return Effect.gen(function* () { + return expectOk( + yield* acp.request("initialize", { + protocolVersion: 1, + clientCapabilities: { _meta: { "terminal-auth": true } }, + clientInfo: { name: "opencode-local-acp-next", version: "0.1.0" }, + }), + ) + }) +} + +export function newSession(acp: AcpClient, cwd: string) { + return Effect.gen(function* () { + return expectOk(yield* acp.request("session/new", { cwd, mcpServers: [] })) + }) +} + +export function verifierConfig(llmUrl: string, skills?: string) { + const config = testProviderConfig(llmUrl) + return { + ...config, + model: "test/test-model", + ...(skills ? { skills: { paths: [skills] } } : {}), + provider: { + test: { + ...config.provider.test, + models: { + "test-model": { + ...config.provider.test.models["test-model"], + variants: { + low: {}, + high: {}, + }, + }, + "second-model": { + ...config.provider.test.models["test-model"], + id: "second-model", + name: "Second Test Model", + variants: { + medium: {}, + max: {}, + }, + }, + }, + }, + }, + } +} + +export function expectErrorCode(error: unknown, code: number) { + if (!error || typeof error !== "object" || !("code" in error)) { + expect(error).toEqual({ code }) + return + } + expect(error.code).toBe(code) +} + +export function expectSelectOption(options: SessionConfigOption[] | null | undefined, id: string) { + const option = selectConfigOption(options, id) + expect(option).toBeDefined() + return option! +} + +export function expectAlternateValue(option: ReturnType) { + const value = flattenSelectOptions(option).find((item) => item.value !== option.currentValue)?.value + expect(value).toBeDefined() + return value! +} + +export const verifierSkill = `--- +name: verifier-skill +description: Verifier compatibility skill. +--- + +# Verifier Skill +` diff --git a/packages/opencode/test/cli/acp-next/initialize-auth.test.ts b/packages/opencode/test/cli/acp-next/initialize-auth.test.ts new file mode 100644 index 0000000000..72b1187656 --- /dev/null +++ b/packages/opencode/test/cli/acp-next/initialize-auth.test.ts @@ -0,0 +1,61 @@ +import { describe, expect } from "bun:test" +import type { AuthenticateResponse, InitializeResponse } from "@agentclientprotocol/sdk" +import { Effect } from "effect" +import { cliIt } from "../../lib/cli-process" +import { createAcpNextClient, expectErrorCode, initialize } from "./helpers" + +describe("opencode acp-next initialize/auth subprocess", () => { + cliIt.live( + "initialize responds with capabilities", + ({ opencode }) => + Effect.gen(function* () { + const initialized = yield* initialize(yield* createAcpNextClient({ opencode })) + + expect(initialized.protocolVersion).toBe(1) + expect(initialized.agentCapabilities?.promptCapabilities?.embeddedContext).toBe(true) + expect(initialized.agentCapabilities?.promptCapabilities?.image).toBe(true) + expect(initialized.agentCapabilities?.mcpCapabilities?.http).toBe(true) + expect(initialized.agentCapabilities?.mcpCapabilities?.sse).toBe(true) + expect(initialized.agentCapabilities?.loadSession).toBe(true) + expect(initialized.agentCapabilities?.sessionCapabilities?.close).toEqual({}) + expect(initialized.agentCapabilities?.sessionCapabilities?.fork).toEqual({}) + expect(initialized.agentCapabilities?.sessionCapabilities?.list).toEqual({}) + expect(initialized.agentCapabilities?.sessionCapabilities?.resume).toEqual({}) + expect(initialized.agentInfo?.name).toBe("OpenCode") + }), + 60_000, + ) + + cliIt.live( + "auth negotiation is explicit and safe", + ({ opencode }) => + Effect.gen(function* () { + const acp = yield* createAcpNextClient({ opencode }) + const initialized = yield* initialize(acp) + + expect(initialized.authMethods?.[0]?.id).toBe("opencode-login") + expect(initialized.authMethods?.[0]?._meta?.["terminal-auth"]).toBeDefined() + expect(yield* acp.request("authenticate", { methodId: "opencode-login" })).toMatchObject( + { result: {} }, + ) + + const rejected = yield* acp.request("authenticate", { methodId: "missing-auth-method" }) + expectErrorCode(rejected.error, -32602) + expect(JSON.stringify(rejected.error)).not.toContain(process.env.OPENCODE_AUTH_CONTENT ?? "not-present") + }), + 60_000, + ) + + cliIt.live( + "initialize without terminal-auth metadata keeps auth command implicit", + ({ opencode }) => + Effect.gen(function* () { + const acp = yield* createAcpNextClient({ opencode }) + const initialized = yield* acp.request("initialize", { protocolVersion: 1 }) + + expect(initialized.result?.authMethods?.[0]?.id).toBe("opencode-login") + expect(initialized.result?.authMethods?.[0]?._meta?.["terminal-auth"]).toBeUndefined() + }), + 60_000, + ) +}) diff --git a/packages/opencode/test/cli/acp-next/lifecycle.test.ts b/packages/opencode/test/cli/acp-next/lifecycle.test.ts new file mode 100644 index 0000000000..8e9793921f --- /dev/null +++ b/packages/opencode/test/cli/acp-next/lifecycle.test.ts @@ -0,0 +1,96 @@ +import { describe, expect } from "bun:test" +import type { CloseSessionResponse, LoadSessionResponse, ResumeSessionResponse } from "@agentclientprotocol/sdk" +import { Duration, Effect } from "effect" +import { cliIt } from "../../lib/cli-process" +import { expectOk, selectConfigOption } from "../acp/acp-test-client" +import { createAcpNextClient, initialize, newSession, verifierConfig } from "./helpers" + +describe("opencode acp-next lifecycle subprocess", () => { + cliIt.live( + "stdin EOF exits cleanly", + ({ opencode }) => + Effect.gen(function* () { + const acp = yield* opencode.acp({ env: { OPENCODE_ACP_NEXT: "1" } }) + acp.close() + + const code = yield* Effect.promise(() => acp.exited).pipe(Effect.timeout(Duration.seconds(5))) + expect(code).toBe(0) + }), + 60_000, + ) + + cliIt.live( + "close capability and close request", + ({ home, llm, opencode }) => + Effect.gen(function* () { + const acp = yield* createAcpNextClient( + { opencode }, + { OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)) }, + ) + const initialized = yield* initialize(acp) + expect(initialized.agentCapabilities?.sessionCapabilities?.close).toEqual({}) + + const session = yield* newSession(acp, home) + expectOk(yield* acp.request("session/close", { sessionId: session.sessionId })) + }), + 60_000, + ) + + cliIt.live( + "loadSession capability and load request return session config options", + ({ home, llm, opencode }) => + Effect.gen(function* () { + const acp = yield* createAcpNextClient( + { opencode }, + { OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)) }, + ) + const initialized = yield* initialize(acp) + expect(initialized.agentCapabilities?.loadSession).toBe(true) + const session = yield* newSession(acp, home) + const loaded = expectOk( + yield* acp.request("session/load", { + cwd: home, + sessionId: session.sessionId, + mcpServers: [], + }), + ) + + expect(selectConfigOption(loaded.configOptions, "model")?.category).toBe("model") + }), + 60_000, + ) + + cliIt.live( + "resume capability advertisement", + ({ opencode }) => + Effect.gen(function* () { + const initialized = yield* initialize(yield* createAcpNextClient({ opencode })) + + expect(initialized.agentCapabilities?.sessionCapabilities?.resume).toEqual({}) + }), + 60_000, + ) + + cliIt.live( + "resume request returns session config options", + ({ home, llm, opencode }) => + Effect.gen(function* () { + const acp = yield* createAcpNextClient( + { opencode }, + { OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)) }, + ) + yield* initialize(acp) + const session = yield* newSession(acp, home) + const resumed = expectOk( + yield* acp.request("session/resume", { + cwd: home, + sessionId: session.sessionId, + mcpServers: [], + }), + ) + + expect(selectConfigOption(resumed.configOptions, "model")?.category).toBe("model") + }), + 60_000, + ) +}) diff --git a/packages/opencode/test/cli/acp-next/skills.test.ts b/packages/opencode/test/cli/acp-next/skills.test.ts new file mode 100644 index 0000000000..0485fa56d4 --- /dev/null +++ b/packages/opencode/test/cli/acp-next/skills.test.ts @@ -0,0 +1,38 @@ +import { describe, expect } from "bun:test" +import type { SessionNotification } from "@agentclientprotocol/sdk" +import { Effect } from "effect" +import { mkdir } from "node:fs/promises" +import path from "node:path" +import { cliIt } from "../../lib/cli-process" +import { createAcpNextClient, initialize, newSession, verifierConfig, verifierSkill } from "./helpers" + +describe("opencode acp-next skills subprocess", () => { + cliIt.live( + "skill slash command appears through available_commands_update", + ({ home, llm, opencode }) => + Effect.gen(function* () { + const skills = path.join(home, "skills") + yield* Effect.promise(() => mkdir(path.join(skills, "verifier-skill"), { recursive: true })) + yield* Effect.promise(() => Bun.write(path.join(skills, "verifier-skill", "SKILL.md"), verifierSkill)) + const acp = yield* createAcpNextClient( + { opencode }, + { OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url, skills)) }, + ) + yield* initialize(acp) + const session = yield* newSession(acp, home) + + const update = yield* acp.waitForNotification( + "session/update", + (params) => + params.sessionId === session.sessionId && + params.update.sessionUpdate === "available_commands_update" && + params.update.availableCommands.some( + (command) => command.name === "verifier-skill" && command.description.length > 0, + ), + ) + + expect(update.params?.sessionId).toBe(session.sessionId) + }), + 60_000, + ) +}) diff --git a/packages/opencode/test/cli/acp-next/timing-diagnostics.test.ts b/packages/opencode/test/cli/acp-next/timing-diagnostics.test.ts new file mode 100644 index 0000000000..c70a82d6b5 --- /dev/null +++ b/packages/opencode/test/cli/acp-next/timing-diagnostics.test.ts @@ -0,0 +1,157 @@ +import { describe, expect } from "bun:test" +import type { SessionNotification, SetSessionConfigOptionResponse } from "@agentclientprotocol/sdk" +import { Effect } from "effect" +import { mkdir } from "node:fs/promises" +import path from "node:path" +import { cliIt } from "../../lib/cli-process" +import { expectOk, flattenSelectOptions } from "../acp/acp-test-client" +import { + createAcpNextClient, + diagnosticFastPathThresholdMs, + diagnosticFirstSessionThresholdMs, + expectAlternateValue, + expectSelectOption, + finalFastPathThresholdMs, + finalFirstSessionThresholdMs, + initialize, + newSession, + verifierConfig, + verifierSkill, +} from "./helpers" + +describe("opencode acp-next verifier timing diagnostics", () => { + cliIt.live( + "first session timing diagnostic stays below generous threshold", + ({ home, llm, opencode }) => + Effect.gen(function* () { + const acp = yield* createAcpNextClient( + { opencode }, + { OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)) }, + ) + const started = performance.now() + yield* initialize(acp) + const session = yield* newSession(acp, home) + const durationMs = Math.round(performance.now() - started) + + expect(session.sessionId).toBeTruthy() + // TODO: replace this diagnostic assertion with finalFirstSessionThresholdMs. + expect(durationMs).toBeLessThan(diagnosticFirstSessionThresholdMs) + expect(finalFirstSessionThresholdMs).toBe(500) + }), + 60_000, + ) + + cliIt.live( + "warm new session timing diagnostic stays below generous threshold", + ({ home, llm, opencode }) => + Effect.gen(function* () { + const acp = yield* createAcpNextClient( + { opencode }, + { OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)) }, + ) + yield* initialize(acp) + yield* newSession(acp, home) + + const started = performance.now() + const session = yield* newSession(acp, home) + const durationMs = Math.round(performance.now() - started) + + expect(session.sessionId).toBeTruthy() + // TODO: replace this diagnostic assertion with finalFastPathThresholdMs. + expect(durationMs).toBeLessThan(diagnosticFastPathThresholdMs) + expect(finalFastPathThresholdMs).toBe(100) + }), + 60_000, + ) + + cliIt.live( + "model switch timing diagnostic updates currentValue below generous threshold", + ({ home, llm, opencode }) => + Effect.gen(function* () { + const acp = yield* createAcpNextClient( + { opencode }, + { OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)) }, + ) + yield* initialize(acp) + const session = yield* newSession(acp, home) + const model = expectSelectOption(session.configOptions, "model") + const nextModel = flattenSelectOptions(model).find((option) => option.value === "test/second-model")?.value + if (!nextModel) throw new Error("expected second test model") + + const started = performance.now() + const updated = expectOk( + yield* acp.request("session/set_config_option", { + sessionId: session.sessionId, + configId: "model", + value: nextModel, + }), + ) + const durationMs = Math.round(performance.now() - started) + + expect(expectSelectOption(updated.configOptions, "model").currentValue).toBe(nextModel) + // TODO: replace this diagnostic assertion with finalFastPathThresholdMs. + expect(durationMs).toBeLessThan(diagnosticFastPathThresholdMs) + }), + 60_000, + ) + + cliIt.live( + "effort switch timing diagnostic updates currentValue below generous threshold", + ({ home, llm, opencode }) => + Effect.gen(function* () { + const acp = yield* createAcpNextClient( + { opencode }, + { OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)) }, + ) + yield* initialize(acp) + const session = yield* newSession(acp, home) + const nextEffort = expectAlternateValue(expectSelectOption(session.configOptions, "effort")) + + const started = performance.now() + const updated = expectOk( + yield* acp.request("session/set_config_option", { + sessionId: session.sessionId, + configId: "effort", + value: nextEffort, + }), + ) + const durationMs = Math.round(performance.now() - started) + + expect(expectSelectOption(updated.configOptions, "effort").currentValue).toBe(nextEffort) + // TODO: replace this diagnostic assertion with finalFastPathThresholdMs. + expect(durationMs).toBeLessThan(diagnosticFastPathThresholdMs) + }), + 60_000, + ) + + cliIt.live( + "warm skill command timing diagnostic stays below generous threshold", + ({ home, llm, opencode }) => + Effect.gen(function* () { + const skills = path.join(home, "skills") + yield* Effect.promise(() => mkdir(path.join(skills, "verifier-skill"), { recursive: true })) + yield* Effect.promise(() => Bun.write(path.join(skills, "verifier-skill", "SKILL.md"), verifierSkill)) + const acp = yield* createAcpNextClient( + { opencode }, + { OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url, skills)) }, + ) + yield* initialize(acp) + yield* newSession(acp, home) + const secondSession = yield* newSession(acp, home) + + const started = performance.now() + yield* acp.waitForNotification( + "session/update", + (params) => + params.sessionId === secondSession.sessionId && + params.update.sessionUpdate === "available_commands_update" && + params.update.availableCommands.some((command) => command.name === "verifier-skill"), + ) + const durationMs = Math.round(performance.now() - started) + + // TODO: replace this diagnostic assertion with finalFastPathThresholdMs. + expect(durationMs).toBeLessThan(diagnosticFastPathThresholdMs) + }), + 60_000, + ) +})