diff --git a/packages/core/src/session/model-request.ts b/packages/core/src/session/model-request.ts index 4cf3e4f8de7..3cd0a264203 100644 --- a/packages/core/src/session/model-request.ts +++ b/packages/core/src/session/model-request.ts @@ -3,7 +3,7 @@ export * as SessionModelRequest from "./model-request" import { LLM, Message, SystemPart, type LLMRequest } from "@opencode-ai/ai" import type { Content } from "@opencode-ai/schema/tool" import { SessionError } from "@opencode-ai/schema/session-error" -import { Cause, Context, Effect, Layer, Result } from "effect" +import { Cause, Config, Context, Effect, Layer, Result } from "effect" import { makeLocationNode } from "@opencode-ai/util/effect/app-node" import { App } from "../app" import { Model } from "../model" @@ -13,6 +13,7 @@ import { QuestionTool } from "../tool/plugin/question" import { Tool } from "../tool" import { SessionContext } from "./context" import { SessionModelHeaders } from "./model-headers" +import { PromptCacheDiagnostics } from "./prompt-cache-diagnostics" import { MAX_STEPS_PROMPT } from "./runner/max-steps" import PROMPT_DEFAULT from "./runner/prompt/base.txt" import { toLLMMessages } from "./runner/to-llm-message" @@ -109,6 +110,11 @@ export const layer = Layer.effect( Effect.gen(function* () { const hooks = yield* PluginHooks.Service const app = yield* App.Metadata + const diagnostics = yield* Config.boolean("OPENCODE_PROMPT_CACHE_DIAGNOSTICS").pipe( + Config.withDefault(false), + Effect.orDie, + ) + const promptCacheSnapshots = diagnostics ? new Map() : undefined const prepare = Effect.fn("SessionModelRequest.prepare")(function* (input: PrepareInput) { const session = input.context.session @@ -156,6 +162,23 @@ export const layer = Layer.effect( tools: hookedTools, toolChoice: stepLimitReached ? "none" : undefined, }) + if (promptCacheSnapshots) { + const current = PromptCacheDiagnostics.snapshot(request) + const comparison = PromptCacheDiagnostics.compare(promptCacheSnapshots.get(session.id), current) + promptCacheSnapshots.delete(session.id) + promptCacheSnapshots.set(session.id, current) + const oldest = promptCacheSnapshots.keys().next().value + if (promptCacheSnapshots.size > 100 && oldest !== undefined) promptCacheSnapshots.delete(oldest) + yield* Effect.logInfo("prompt cache prefix").pipe( + Effect.annotateLogs({ + sessionID: session.id, + toolCount: current.tools.length, + systemParts: current.system.length, + messageCount: current.messages.length, + ...comparison, + }), + ) + } const executeTool: Prepared["executeTool"] = (executeInput) => { if (stepLimitReached) return new Tool.Error({ message: "Tools are disabled after the maximum agent steps" }) diff --git a/packages/core/src/session/prompt-cache-diagnostics.ts b/packages/core/src/session/prompt-cache-diagnostics.ts new file mode 100644 index 00000000000..99ab54576a1 --- /dev/null +++ b/packages/core/src/session/prompt-cache-diagnostics.ts @@ -0,0 +1,95 @@ +export * as PromptCacheDiagnostics from "./prompt-cache-diagnostics" + +import type { LLMRequest } from "@opencode-ai/ai" +import { Hash } from "@opencode-ai/util/hash" + +interface Entry { + readonly label: string + readonly hash: string +} + +export interface Snapshot { + readonly settings: string + readonly tools: ReadonlyArray + readonly system: ReadonlyArray + readonly messages: ReadonlyArray +} + +export type Comparison = + | { readonly status: "initial" } + | { readonly status: "stable"; readonly messages: number } + | { readonly status: "append-only"; readonly previousMessages: number; readonly currentMessages: number } + | { + readonly status: "changed" + readonly component: "settings" | "tools" | "system" | "messages" + readonly index: number + readonly label: string + } + +const hash = (value: unknown) => Hash.sha256(JSON.stringify(value)).slice(0, 16) + +export function snapshot(request: LLMRequest): Snapshot { + return { + settings: hash({ + route: request.model.route.id, + provider: request.model.provider, + model: request.model.id, + modelDefaults: request.model.defaults, + compatibility: request.model.compatibility, + routeDefaults: { + generation: request.model.route.defaults.generation, + providerOptions: request.model.route.defaults.providerOptions, + http: request.model.route.defaults.http, + }, + generation: request.generation, + providerOptions: request.providerOptions, + http: request.http, + toolChoice: request.toolChoice, + cache: request.cache, + }), + tools: request.tools.map((tool) => ({ label: tool.name, hash: hash(tool) })), + system: request.system.map((part, index) => ({ label: `system[${index}]`, hash: hash(part) })), + messages: request.messages.map((message, index) => ({ + label: message.id ?? `${message.role}[${index}]`, + hash: hash(message), + })), + } +} + +export function compare(previous: Snapshot | undefined, current: Snapshot): Comparison { + if (!previous) return { status: "initial" } + if (previous.settings !== current.settings) + return { + status: "changed", + component: "settings", + index: 0, + label: "model settings", + } + const tools = firstChange(previous.tools, current.tools, false) + if (tools) return { status: "changed", component: "tools", ...tools } + const system = firstChange(previous.system, current.system, false) + if (system) return { status: "changed", component: "system", ...system } + const messages = firstChange(previous.messages, current.messages, true) + if (messages) return { status: "changed", component: "messages", ...messages } + if (previous.messages.length === current.messages.length) + return { status: "stable", messages: current.messages.length } + return { + status: "append-only", + previousMessages: previous.messages.length, + currentMessages: current.messages.length, + } +} + +function firstChange(previous: ReadonlyArray, current: ReadonlyArray, allowAppend: boolean) { + const index = previous.findIndex((entry, index) => entry.hash !== current[index]?.hash) + if (index >= 0) + return { + index, + label: current[index]?.label ?? previous[index]?.label ?? `entry[${index}]`, + } + if (current.length === previous.length || (allowAppend && current.length > previous.length)) return + return { + index: previous.length, + label: current[previous.length]?.label ?? `entry[${previous.length}]`, + } +} diff --git a/packages/core/test/prompt-cache-diagnostics.test.ts b/packages/core/test/prompt-cache-diagnostics.test.ts new file mode 100644 index 00000000000..4049c99465f --- /dev/null +++ b/packages/core/test/prompt-cache-diagnostics.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, test } from "bun:test" +import { GenerationOptions, LLM, LLMRequest, Message, Model, ToolDefinition } from "@opencode-ai/ai" +import { OpenAIChat } from "@opencode-ai/ai/protocols" +import { PromptCacheDiagnostics } from "@opencode-ai/core/session/prompt-cache-diagnostics" + +const model = Model.make({ id: "test", provider: "test", route: OpenAIChat.route }) +const tool = ToolDefinition.make({ + name: "read", + description: "Read a file", + inputSchema: { type: "object", properties: {} }, +}) + +const request = LLM.request({ + model, + system: "System", + prompt: "First", + tools: [tool], +}) +const compare = (current: LLMRequest) => + PromptCacheDiagnostics.compare(PromptCacheDiagnostics.snapshot(request), PromptCacheDiagnostics.snapshot(current)) + +describe("PromptCacheDiagnostics", () => { + test("distinguishes initial and stable requests", () => { + const snapshot = PromptCacheDiagnostics.snapshot(request) + expect(PromptCacheDiagnostics.compare(undefined, snapshot)).toEqual({ status: "initial" }) + expect(PromptCacheDiagnostics.compare(snapshot, snapshot)).toEqual({ status: "stable", messages: 1 }) + }) + + test("recognizes append-only history", () => { + const current = LLMRequest.update(request, { messages: [...request.messages, Message.assistant("Second")] }) + expect(compare(current)).toEqual({ status: "append-only", previousMessages: 1, currentMessages: 2 }) + }) + + test("detects cache-sensitive setting changes", () => { + const current = LLMRequest.update(request, { generation: GenerationOptions.make({ temperature: 0.5 }) }) + expect(compare(current)).toEqual({ status: "changed", component: "settings", index: 0, label: "model settings" }) + }) + + test("finds the first changed prefix component", () => { + const changedTool = ToolDefinition.make({ ...tool, description: "Read one file" }) + const current = LLMRequest.update(request, { tools: [changedTool] }) + expect(compare(current)).toEqual({ status: "changed", component: "tools", index: 0, label: "read" }) + }) + + test("treats appended tools as a prefix change", () => { + const write = ToolDefinition.make({ + name: "write", + description: "Write a file", + inputSchema: { type: "object", properties: {} }, + }) + const current = LLMRequest.update(request, { tools: [...request.tools, write] }) + expect(compare(current)).toEqual({ status: "changed", component: "tools", index: 1, label: "write" }) + }) +}) diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index 7be6e0cc5ef..e7c8e057835 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -42,6 +42,7 @@ import { SessionRunCoordinator } from "@opencode-ai/core/session/run-coordinator import { SessionRunner } from "@opencode-ai/core/session/runner" import * as SessionRunnerLLM from "@opencode-ai/core/session/runner/llm" import { SessionRunnerModel } from "@opencode-ai/core/session/runner/model" +import { PromptCacheDiagnostics } from "@opencode-ai/core/session/prompt-cache-diagnostics" import { SessionUsage } from "@opencode-ai/core/session/usage" import { PluginSupervisor } from "@opencode-ai/core/plugin/supervisor" import { PluginHooks } from "@opencode-ai/core/plugin/hooks" @@ -1315,6 +1316,12 @@ describe("SessionRunnerLLM", () => { yield* admit(session, "Second") yield* session.resume(sessionID) + expect( + PromptCacheDiagnostics.compare( + PromptCacheDiagnostics.snapshot(requests[0]!), + PromptCacheDiagnostics.snapshot(requests[1]!), + ), + ).toEqual({ status: "append-only", previousMessages: 1, currentMessages: 3 }) expect(requests.map((request) => request.system.map((part) => part.text))).toEqual([ [defaultSystem, "Initial context"], [defaultSystem, "Initial context"],