From 756c7c60bd31254abcf43b79f7d50931b36c02ec Mon Sep 17 00:00:00 2001 From: Shoubhit Dash Date: Mon, 25 May 2026 20:19:06 +0530 Subject: [PATCH] test(acp): add compatibility baseline (#29222) --- .../acp/acp-compatibility-baseline.test.ts | 322 ++++++++++++++++++ .../opencode/test/cli/acp/acp-test-client.ts | 96 ++++++ 2 files changed, 418 insertions(+) create mode 100644 packages/opencode/test/cli/acp/acp-compatibility-baseline.test.ts create mode 100644 packages/opencode/test/cli/acp/acp-test-client.ts diff --git a/packages/opencode/test/cli/acp/acp-compatibility-baseline.test.ts b/packages/opencode/test/cli/acp/acp-compatibility-baseline.test.ts new file mode 100644 index 0000000000..5352039383 --- /dev/null +++ b/packages/opencode/test/cli/acp/acp-compatibility-baseline.test.ts @@ -0,0 +1,322 @@ +import { describe, expect } from "bun:test" +import type { + CloseSessionResponse, + InitializeResponse, + NewSessionResponse, + ResumeSessionResponse, + SessionNotification, + SetSessionConfigOptionResponse, +} from "@agentclientprotocol/sdk" +import { Effect } from "effect" +import { mkdir } from "node:fs/promises" +import path from "node:path" +import { cliIt } from "../../lib/cli-process" +import { testProviderConfig } from "../../lib/test-provider" +import { + createAcpClient, + expectOk, + firstAlternateValue, + flattenSelectOptions, + selectConfigOption, +} from "./acp-test-client" + +describe("opencode acp verifier compatibility baseline", () => { + cliIt.live( + "initialize advertises close and resume capabilities", + ({ opencode }) => + Effect.gen(function* () { + const acp = createAcpClient(yield* opencode.acp()) + const initialized = expectOk( + yield* acp.request("initialize", { + protocolVersion: 1, + }), + ) + + expect(initialized.protocolVersion).toBe(1) + expect(initialized.agentCapabilities?.sessionCapabilities?.close).toEqual({}) + expect(initialized.agentCapabilities?.sessionCapabilities?.resume).toEqual({}) + }), + 60_000, + ) + + cliIt.live( + "first session timing diagnostic stays bounded and returns model options", + ({ home, llm, opencode }) => + Effect.gen(function* () { + const acp = createAcpClient( + yield* opencode.acp({ + env: { + OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)), + }, + }), + ) + const started = Date.now() + yield* acp.request("initialize", { + protocolVersion: 1, + clientCapabilities: {}, + clientInfo: { name: "opencode-local-acp-baseline", version: "0.1.0" }, + }) + const session = expectOk( + yield* acp.request("session/new", { + cwd: home, + mcpServers: [], + }), + ) + const durationMs = Date.now() - started + expect(durationMs).toBeLessThan(15_000) + + const model = selectConfigOption(session.configOptions, "model") + expect(model?.category).toBe("model") + expect(model?.currentValue).toBe("test/test-model") + expect(model ? flattenSelectOptions(model).length : 0).toBeGreaterThanOrEqual(2) + }), + 60_000, + ) + + cliIt.live( + "warm newSession timing diagnostic stays bounded", + ({ home, llm, opencode }) => + Effect.gen(function* () { + const acp = createAcpClient( + yield* opencode.acp({ + env: { + OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)), + }, + }), + ) + yield* acp.request("initialize", { protocolVersion: 1 }) + yield* acp.request("session/new", { cwd: home, mcpServers: [] }) + + const started = Date.now() + const session = expectOk( + yield* acp.request("session/new", { + cwd: home, + mcpServers: [], + }), + ) + const durationMs = Date.now() - started + expect(durationMs).toBeLessThan(15_000) + expect(session.sessionId).toBeTruthy() + }), + 60_000, + ) + + cliIt.live( + "model switch timing diagnostic updates currentValue", + ({ home, llm, opencode }) => + Effect.gen(function* () { + const acp = createAcpClient( + yield* opencode.acp({ + env: { + OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)), + }, + }), + ) + yield* acp.request("initialize", { protocolVersion: 1 }) + const session = expectOk(yield* acp.request("session/new", { cwd: home, mcpServers: [] })) + const model = selectConfigOption(session.configOptions, "model") + expect(model).toBeDefined() + const nextModel = model + ? flattenSelectOptions(model).find((option) => option.value === "test/second-model")?.value + : undefined + expect(nextModel).toBe("test/second-model") + + const started = Date.now() + const updated = expectOk( + yield* acp.request("session/set_config_option", { + sessionId: session.sessionId, + configId: "model", + value: nextModel, + }), + ) + const durationMs = Date.now() - started + + expect(durationMs).toBeLessThan(15_000) + expect(selectConfigOption(updated.configOptions, "model")?.currentValue).toBe(nextModel) + }), + 60_000, + ) + + cliIt.live( + "effort option is listed for variant-capable models and can switch", + ({ home, llm, opencode }) => + Effect.gen(function* () { + const acp = createAcpClient( + yield* opencode.acp({ + env: { + OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)), + }, + }), + ) + yield* acp.request("initialize", { protocolVersion: 1 }) + const session = expectOk(yield* acp.request("session/new", { cwd: home, mcpServers: [] })) + const effort = selectConfigOption(session.configOptions, "effort") + expect(effort?.category).toBe("thought_level") + const nextEffort = effort ? firstAlternateValue(effort) : undefined + expect(nextEffort).toBe("high") + + const updated = expectOk( + yield* acp.request("session/set_config_option", { + sessionId: session.sessionId, + configId: "effort", + value: nextEffort, + }), + ) + + expect(selectConfigOption(updated.configOptions, "effort")?.currentValue).toBe(nextEffort) + }), + 60_000, + ) + + cliIt.live( + "default test provider documents missing effort option when the model has no variants", + ({ home, llm, opencode }) => + Effect.gen(function* () { + const acp = createAcpClient( + yield* opencode.acp({ + env: { + OPENCODE_CONFIG_CONTENT: JSON.stringify(noVariantConfig(llm.url)), + }, + }), + ) + yield* acp.request("initialize", { protocolVersion: 1 }) + const session = expectOk(yield* acp.request("session/new", { cwd: home, mcpServers: [] })) + + expect(selectConfigOption(session.configOptions, "model")?.currentValue).toBe("test/test-model") + expect(selectConfigOption(session.configOptions, "effort")).toBeUndefined() + }), + 60_000, + ) + + cliIt.live( + "skill slash command timing diagnostic appears through available_commands_update", + ({ home, llm, opencode }) => + Effect.gen(function* () { + const skills = path.join(home, "skills") + yield* Effect.promise(() => mkdir(path.join(skills, "verifier-skill"), { recursive: true })) + yield* Effect.promise(() => Bun.write(path.join(skills, "verifier-skill", "SKILL.md"), verifierSkill)) + const acp = createAcpClient( + yield* opencode.acp({ + env: { + OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url, skills)), + }, + }), + ) + yield* acp.request("initialize", { protocolVersion: 1 }) + const session = expectOk(yield* acp.request("session/new", { cwd: home, mcpServers: [] })) + + const update = yield* acp.waitForNotification( + "session/update", + (params) => + params.sessionId === session.sessionId && + params.update.sessionUpdate === "available_commands_update" && + params.update.availableCommands.some((command) => command.name === "verifier-skill"), + ) + + expect(update.params?.sessionId).toBe(session.sessionId) + + const secondSession = expectOk( + yield* acp.request("session/new", { cwd: home, mcpServers: [] }), + ) + const started = Date.now() + yield* acp.waitForNotification( + "session/update", + (params) => + params.sessionId === secondSession.sessionId && + params.update.sessionUpdate === "available_commands_update" && + params.update.availableCommands.some((command) => command.name === "verifier-skill"), + ) + const durationMs = Date.now() - started + expect(durationMs).toBeLessThan(15_000) + }), + 60_000, + ) + + cliIt.live( + "close request succeeds for a live session", + ({ home, opencode }) => + Effect.gen(function* () { + const acp = createAcpClient(yield* opencode.acp()) + yield* acp.request("initialize", { protocolVersion: 1 }) + const session = expectOk(yield* acp.request("session/new", { cwd: home, mcpServers: [] })) + + expectOk(yield* acp.request("session/close", { sessionId: session.sessionId })) + }), + 60_000, + ) + + cliIt.live( + "resume request succeeds for a created session", + ({ home, opencode }) => + Effect.gen(function* () { + const acp = createAcpClient(yield* opencode.acp()) + yield* acp.request("initialize", { protocolVersion: 1 }) + const session = expectOk(yield* acp.request("session/new", { cwd: home, mcpServers: [] })) + + const resumed = expectOk( + yield* acp.request("session/resume", { + sessionId: session.sessionId, + cwd: home, + mcpServers: [], + }), + ) + expect(resumed.configOptions?.length).toBeGreaterThan(0) + }), + 60_000, + ) +}) + +function verifierConfig(llmUrl: string, skills?: string) { + const config = testProviderConfig(llmUrl) + return { + ...config, + model: "test/test-model", + ...(skills ? { skills: { paths: [skills] } } : {}), + provider: { + test: { + ...config.provider.test, + models: { + "test-model": { + ...config.provider.test.models["test-model"], + variants: { + low: {}, + high: {}, + }, + }, + "second-model": { + ...config.provider.test.models["test-model"], + id: "second-model", + name: "Second Test Model", + }, + }, + }, + }, + } +} + +function noVariantConfig(llmUrl: string) { + const config = verifierConfig(llmUrl) + return { + ...config, + provider: { + test: { + ...config.provider.test, + models: { + "test-model": { + ...config.provider.test.models["test-model"], + variants: undefined, + }, + "second-model": config.provider.test.models["second-model"], + }, + }, + }, + } +} + +const verifierSkill = `--- +name: verifier-skill +description: Verifier compatibility skill. +--- + +# Verifier Skill +` diff --git a/packages/opencode/test/cli/acp/acp-test-client.ts b/packages/opencode/test/cli/acp/acp-test-client.ts new file mode 100644 index 0000000000..f8a5700b8b --- /dev/null +++ b/packages/opencode/test/cli/acp/acp-test-client.ts @@ -0,0 +1,96 @@ +import { expect } from "bun:test" +import type { SessionConfigOption, SessionConfigSelectOption } from "@agentclientprotocol/sdk" +import { Duration, Effect } from "effect" +import type { AcpHandle } from "../../lib/cli-process" + +type JsonRpcRequest = { + readonly jsonrpc: "2.0" + readonly id: number + readonly method: string + readonly params?: unknown +} + +type JsonRpcResponse = { + readonly jsonrpc: "2.0" + readonly id: number + readonly result?: T + readonly error?: unknown +} + +type JsonRpcNotification = { + readonly jsonrpc: "2.0" + readonly method: string + readonly params?: T +} + +export type AcpClient = { + readonly request: (method: string, params?: unknown) => Effect.Effect, unknown> + readonly receive: Effect.Effect + readonly waitForNotification: ( + method: string, + predicate: (params: T) => boolean, + timeoutMs?: number, + ) => Effect.Effect, unknown> +} + +export function createAcpClient(acp: AcpHandle): AcpClient { + const state = { nextId: 1 } + + const request = (method: string, params?: unknown) => + Effect.gen(function* () { + const id = state.nextId++ + const message: JsonRpcRequest = + params === undefined ? { jsonrpc: "2.0", id, method } : { jsonrpc: "2.0", id, method, params } + yield* acp.send(message) + + while (true) { + const received = yield* acp.receive.pipe(Effect.timeout(Duration.seconds(15))) + if (isJsonRpcResponse(received) && received.id === id) return received + } + }) + + const waitForNotification = (method: string, predicate: (params: T) => boolean, timeoutMs = 15_000) => + Effect.gen(function* () { + while (true) { + const received = yield* acp.receive.pipe(Effect.timeout(Duration.millis(timeoutMs))) + if (!isJsonRpcNotification(received)) continue + if (received.method === method && predicate(received.params as T)) return received + } + }) + + return { + request, + receive: acp.receive, + waitForNotification, + } +} + +export function expectOk(response: JsonRpcResponse) { + expect(response.error).toBeUndefined() + expect(response.result).toBeDefined() + return response.result as T +} + +export function selectConfigOption(options: SessionConfigOption[] | null | undefined, id: string) { + return options?.find( + (option): option is Extract => option.id === id && option.type === "select", + ) +} + +export function firstAlternateValue(option: Extract) { + return flattenSelectOptions(option).find((item) => item.value !== option.currentValue)?.value +} + +export function flattenSelectOptions(option: Extract) { + return option.options.flatMap((item): SessionConfigSelectOption[] => ("value" in item ? [item] : item.options)) +} + +function isJsonRpcResponse(input: unknown): input is JsonRpcResponse { + if (!input || typeof input !== "object") return false + return "id" in input && "jsonrpc" in input +} + +function isJsonRpcNotification(input: unknown): input is JsonRpcNotification { + if (!input || typeof input !== "object") return false + return "method" in input && !("id" in input) && "jsonrpc" in input +}