mirror of
https://github.com/anomalyco/opencode.git
synced 2026-05-31 05:15:32 +00:00
test(acp): remove timing diagnostics (#29741)
This commit is contained in:
Shoubhit Dash
GitHub
parent
30a66dfc31
commit
c3562a4a51
3 changed files with 4 additions and 188 deletions
|
|
@ -11,14 +11,6 @@ import {
|
|||
type AcpClient,
|
||||
} from "../acp/acp-test-client"
|
||||
|
||||
export const diagnosticFirstSessionThresholdMs = 5_000
|
||||
export const diagnosticFastPathThresholdMs = 15_000
|
||||
|
||||
// TODO: tighten to the public verifier target of 500ms once acp-next startup is optimized.
|
||||
export const finalFirstSessionThresholdMs = 500
|
||||
// TODO: tighten warm session/config/skill fast paths to the public verifier target of 100ms.
|
||||
export const finalFastPathThresholdMs = 100
|
||||
|
||||
export function createAcpNextClient(input: Pick<CliFixture, "opencode">, env?: Record<string, string>) {
|
||||
return Effect.gen(function* () {
|
||||
return createAcpClient(
|
||||
|
|
|
|||
|
|
@ -1,153 +0,0 @@
|
|||
import { describe, expect } from "bun:test"
|
||||
import type { SessionNotification, SetSessionConfigOptionResponse } from "@agentclientprotocol/sdk"
|
||||
import { Effect } from "effect"
|
||||
import { mkdir } from "node:fs/promises"
|
||||
import path from "node:path"
|
||||
import { cliIt } from "../../lib/cli-process"
|
||||
import { expectOk, flattenSelectOptions } from "../acp/acp-test-client"
|
||||
import {
|
||||
createAcpNextClient,
|
||||
diagnosticFastPathThresholdMs,
|
||||
diagnosticFirstSessionThresholdMs,
|
||||
expectAlternateValue,
|
||||
expectSelectOption,
|
||||
finalFastPathThresholdMs,
|
||||
finalFirstSessionThresholdMs,
|
||||
initialize,
|
||||
newSession,
|
||||
verifierConfig,
|
||||
verifierSkill,
|
||||
} from "./helpers"
|
||||
|
||||
describe("opencode acp-next verifier timing diagnostics", () => {
|
||||
cliIt.live(
|
||||
"first session timing diagnostic stays below generous threshold",
|
||||
({ home, llm, opencode }) =>
|
||||
Effect.gen(function* () {
|
||||
const acp = yield* createAcpNextClient(
|
||||
{ opencode },
|
||||
{ OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)) },
|
||||
)
|
||||
const started = performance.now()
|
||||
yield* initialize(acp)
|
||||
const session = yield* newSession(acp, home)
|
||||
const durationMs = Math.round(performance.now() - started)
|
||||
|
||||
expect(session.sessionId).toBeTruthy()
|
||||
// TODO: replace this diagnostic assertion with finalFirstSessionThresholdMs.
|
||||
expect(durationMs).toBeLessThan(diagnosticFirstSessionThresholdMs)
|
||||
expect(finalFirstSessionThresholdMs).toBe(500)
|
||||
}),
|
||||
60_000,
|
||||
)
|
||||
|
||||
cliIt.live(
|
||||
"warm new session stays below verifier threshold",
|
||||
({ home, llm, opencode }) =>
|
||||
Effect.gen(function* () {
|
||||
const acp = yield* createAcpNextClient(
|
||||
{ opencode },
|
||||
{ OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)) },
|
||||
)
|
||||
yield* initialize(acp)
|
||||
yield* newSession(acp, home)
|
||||
|
||||
const started = performance.now()
|
||||
const session = yield* newSession(acp, home)
|
||||
const durationMs = Math.round(performance.now() - started)
|
||||
|
||||
expect(session.sessionId).toBeTruthy()
|
||||
expect(durationMs).toBeLessThan(finalFastPathThresholdMs)
|
||||
}),
|
||||
60_000,
|
||||
)
|
||||
|
||||
cliIt.live(
|
||||
"model switch updates currentValue below verifier threshold",
|
||||
({ home, llm, opencode }) =>
|
||||
Effect.gen(function* () {
|
||||
const acp = yield* createAcpNextClient(
|
||||
{ opencode },
|
||||
{ OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)) },
|
||||
)
|
||||
yield* initialize(acp)
|
||||
const session = yield* newSession(acp, home)
|
||||
const model = expectSelectOption(session.configOptions, "model")
|
||||
const nextModel = flattenSelectOptions(model).find((option) => option.value === "test/second-model")?.value
|
||||
if (!nextModel) throw new Error("expected second test model")
|
||||
|
||||
const started = performance.now()
|
||||
const updated = expectOk(
|
||||
yield* acp.request<SetSessionConfigOptionResponse>("session/set_config_option", {
|
||||
sessionId: session.sessionId,
|
||||
configId: "model",
|
||||
value: nextModel,
|
||||
}),
|
||||
)
|
||||
const durationMs = Math.round(performance.now() - started)
|
||||
|
||||
expect(expectSelectOption(updated.configOptions, "model").currentValue).toBe(nextModel)
|
||||
expect(durationMs).toBeLessThan(finalFastPathThresholdMs)
|
||||
}),
|
||||
60_000,
|
||||
)
|
||||
|
||||
cliIt.live(
|
||||
"effort switch updates currentValue below verifier threshold",
|
||||
({ home, llm, opencode }) =>
|
||||
Effect.gen(function* () {
|
||||
const acp = yield* createAcpNextClient(
|
||||
{ opencode },
|
||||
{ OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url)) },
|
||||
)
|
||||
yield* initialize(acp)
|
||||
const session = yield* newSession(acp, home)
|
||||
const nextEffort = expectAlternateValue(expectSelectOption(session.configOptions, "effort"))
|
||||
|
||||
const started = performance.now()
|
||||
const updated = expectOk(
|
||||
yield* acp.request<SetSessionConfigOptionResponse>("session/set_config_option", {
|
||||
sessionId: session.sessionId,
|
||||
configId: "effort",
|
||||
value: nextEffort,
|
||||
}),
|
||||
)
|
||||
const durationMs = Math.round(performance.now() - started)
|
||||
|
||||
expect(expectSelectOption(updated.configOptions, "effort").currentValue).toBe(nextEffort)
|
||||
expect(durationMs).toBeLessThan(finalFastPathThresholdMs)
|
||||
}),
|
||||
60_000,
|
||||
)
|
||||
|
||||
cliIt.live(
|
||||
"warm skill command timing diagnostic stays below generous threshold",
|
||||
({ home, llm, opencode }) =>
|
||||
Effect.gen(function* () {
|
||||
const skills = path.join(home, "skills")
|
||||
yield* Effect.promise(() => mkdir(path.join(skills, "verifier-skill"), { recursive: true }))
|
||||
yield* Effect.promise(() => Bun.write(path.join(skills, "verifier-skill", "SKILL.md"), verifierSkill))
|
||||
const acp = yield* createAcpNextClient(
|
||||
{ opencode },
|
||||
{ OPENCODE_CONFIG_CONTENT: JSON.stringify(verifierConfig(llm.url, skills)) },
|
||||
)
|
||||
yield* initialize(acp)
|
||||
yield* newSession(acp, home)
|
||||
const secondSession = yield* newSession(acp, home)
|
||||
|
||||
const started = performance.now()
|
||||
yield* acp.waitForNotification<SessionNotification>(
|
||||
"session/update",
|
||||
(params) =>
|
||||
params.sessionId === secondSession.sessionId &&
|
||||
params.update.sessionUpdate === "available_commands_update" &&
|
||||
params.update.availableCommands.some((command) => command.name === "verifier-skill"),
|
||||
)
|
||||
const durationMs = Math.round(performance.now() - started)
|
||||
|
||||
// TODO: replace this diagnostic assertion with finalFastPathThresholdMs.
|
||||
expect(durationMs).toBeLessThan(diagnosticFastPathThresholdMs)
|
||||
}),
|
||||
60_000,
|
||||
)
|
||||
})
|
||||
|
|
@ -40,7 +40,7 @@ describe("opencode acp verifier compatibility baseline", () => {
|
|||
)
|
||||
|
||||
cliIt.live(
|
||||
"first session timing diagnostic stays bounded and returns model options",
|
||||
"first session returns model options",
|
||||
({ home, llm, opencode }) =>
|
||||
Effect.gen(function* () {
|
||||
const acp = createAcpClient(
|
||||
|
|
@ -50,7 +50,6 @@ describe("opencode acp verifier compatibility baseline", () => {
|
|||
},
|
||||
}),
|
||||
)
|
||||
const started = Date.now()
|
||||
yield* acp.request<InitializeResponse>("initialize", {
|
||||
protocolVersion: 1,
|
||||
clientCapabilities: {},
|
||||
|
|
@ -62,9 +61,6 @@ describe("opencode acp verifier compatibility baseline", () => {
|
|||
mcpServers: [],
|
||||
}),
|
||||
)
|
||||
const durationMs = Date.now() - started
|
||||
expect(durationMs).toBeLessThan(15_000)
|
||||
|
||||
const model = selectConfigOption(session.configOptions, "model")
|
||||
expect(model?.category).toBe("model")
|
||||
expect(model?.currentValue).toBe("test/test-model")
|
||||
|
|
@ -74,7 +70,7 @@ describe("opencode acp verifier compatibility baseline", () => {
|
|||
)
|
||||
|
||||
cliIt.live(
|
||||
"warm newSession timing diagnostic stays bounded",
|
||||
"newSession can be called repeatedly",
|
||||
({ home, llm, opencode }) =>
|
||||
Effect.gen(function* () {
|
||||
const acp = createAcpClient(
|
||||
|
|
@ -87,22 +83,19 @@ describe("opencode acp verifier compatibility baseline", () => {
|
|||
yield* acp.request<InitializeResponse>("initialize", { protocolVersion: 1 })
|
||||
yield* acp.request<NewSessionResponse>("session/new", { cwd: home, mcpServers: [] })
|
||||
|
||||
const started = Date.now()
|
||||
const session = expectOk(
|
||||
yield* acp.request<NewSessionResponse>("session/new", {
|
||||
cwd: home,
|
||||
mcpServers: [],
|
||||
}),
|
||||
)
|
||||
const durationMs = Date.now() - started
|
||||
expect(durationMs).toBeLessThan(15_000)
|
||||
expect(session.sessionId).toBeTruthy()
|
||||
}),
|
||||
60_000,
|
||||
)
|
||||
|
||||
cliIt.live(
|
||||
"model switch timing diagnostic updates currentValue",
|
||||
"model switch updates currentValue",
|
||||
({ home, llm, opencode }) =>
|
||||
Effect.gen(function* () {
|
||||
const acp = createAcpClient(
|
||||
|
|
@ -121,7 +114,6 @@ describe("opencode acp verifier compatibility baseline", () => {
|
|||
: undefined
|
||||
expect(nextModel).toBe("test/second-model")
|
||||
|
||||
const started = Date.now()
|
||||
const updated = expectOk(
|
||||
yield* acp.request<SetSessionConfigOptionResponse>("session/set_config_option", {
|
||||
sessionId: session.sessionId,
|
||||
|
|
@ -129,9 +121,7 @@ describe("opencode acp verifier compatibility baseline", () => {
|
|||
value: nextModel,
|
||||
}),
|
||||
)
|
||||
const durationMs = Date.now() - started
|
||||
|
||||
expect(durationMs).toBeLessThan(15_000)
|
||||
expect(selectConfigOption(updated.configOptions, "model")?.currentValue).toBe(nextModel)
|
||||
}),
|
||||
60_000,
|
||||
|
|
@ -189,7 +179,7 @@ describe("opencode acp verifier compatibility baseline", () => {
|
|||
)
|
||||
|
||||
cliIt.live(
|
||||
"skill slash command timing diagnostic appears through available_commands_update",
|
||||
"skill slash command appears through available_commands_update",
|
||||
({ home, llm, opencode }) =>
|
||||
Effect.gen(function* () {
|
||||
const skills = path.join(home, "skills")
|
||||
|
|
@ -215,19 +205,6 @@ describe("opencode acp verifier compatibility baseline", () => {
|
|||
|
||||
expect(update.params?.sessionId).toBe(session.sessionId)
|
||||
|
||||
const secondSession = expectOk(
|
||||
yield* acp.request<NewSessionResponse>("session/new", { cwd: home, mcpServers: [] }),
|
||||
)
|
||||
const started = Date.now()
|
||||
yield* acp.waitForNotification<SessionNotification>(
|
||||
"session/update",
|
||||
(params) =>
|
||||
params.sessionId === secondSession.sessionId &&
|
||||
params.update.sessionUpdate === "available_commands_update" &&
|
||||
params.update.availableCommands.some((command) => command.name === "verifier-skill"),
|
||||
)
|
||||
const durationMs = Date.now() - started
|
||||
expect(durationMs).toBeLessThan(15_000)
|
||||
}),
|
||||
60_000,
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue