feat(coding-agent): add prompt cache miss tracking (#6427)
Some checks are pending
CI / build-check-test (push) Waiting to run

Detect prompt cache misses per turn by comparing each assistant message's
cache reads against the previous request's prompt tokens (core/cache-stats.ts).
Significant misses emit a warning-colored transcript notice at the turn they
occur, noting idle gaps past the cache TTL and model switches when relevant.

/session gained cache statistics: a compact token/cache breakdown with hit
rate, a $-prefixed cost section with per-model cost breakdown, and the
cumulative cost re-billed due to cache misses.
This commit is contained in:
Armin Ronacher 2026-07-09 14:12:57 +02:00 committed by GitHub
parent 57d96d72ed
commit 3f9aa5d10b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 471 additions and 36 deletions

View file

@ -8,6 +8,7 @@
- Added public SDK exports for CLI-equivalent model and scoped-model resolution ([#6201](https://github.com/earendil-works/pi/issues/6201)).
- Added extension and RPC `agent_settled` events plus session-level idle waiting for fully settled agent runs ([#6363](https://github.com/earendil-works/pi/issues/6363)).
- Added extension entry renderers for persisted display-only session entries that are rendered in interactive mode without being sent to the model context.
- Added a `showCacheMissNotices` setting and `/settings` toggle for significant prompt-cache miss transcript notices.
### Fixed

View file

@ -31,6 +31,7 @@ Use `/trust` in interactive mode to save a project trust decision for future ses
| `defaultModel` | string | - | Default model ID |
| `defaultThinkingLevel` | string | - | `"off"`, `"minimal"`, `"low"`, `"medium"`, `"high"`, `"xhigh"` |
| `hideThinkingBlock` | boolean | `false` | Hide thinking blocks in output |
| `showCacheMissNotices` | boolean | `false` | Show transcript notices for significant prompt-cache misses |
| `thinkingBudgets` | object | - | Custom token budgets per thinking level |
#### thinkingBudgets

View file

@ -3016,14 +3016,15 @@ export class AgentSession {
}
/**
* Get session statistics.
* Get session statistics. Aggregates over ALL session entries (including
* history that was compacted away), so token/cost totals reflect what was
* actually billed across the session.
*/
getSessionStats(): SessionStats {
const state = this.state;
const userMessages = state.messages.filter((m) => m.role === "user").length;
const assistantMessages = state.messages.filter((m) => m.role === "assistant").length;
const toolResults = state.messages.filter((m) => m.role === "toolResult").length;
let userMessages = 0;
let assistantMessages = 0;
let toolResults = 0;
let totalMessages = 0;
let toolCalls = 0;
let totalInput = 0;
let totalOutput = 0;
@ -3031,15 +3032,26 @@ export class AgentSession {
let totalCacheWrite = 0;
let totalCost = 0;
for (const message of state.messages) {
if (message.role === "assistant") {
for (const entry of this.sessionManager.getEntries()) {
if (entry.type !== "message") continue;
totalMessages++;
const message = entry.message;
if (message.role === "user") {
userMessages++;
} else if (message.role === "toolResult") {
toolResults++;
} else if (message.role === "assistant") {
assistantMessages++;
const assistantMsg = message as AssistantMessage;
toolCalls += assistantMsg.content.filter((c) => c.type === "toolCall").length;
totalInput += assistantMsg.usage.input;
totalOutput += assistantMsg.usage.output;
totalCacheRead += assistantMsg.usage.cacheRead;
totalCacheWrite += assistantMsg.usage.cacheWrite;
totalCost += assistantMsg.usage.cost.total;
if (Array.isArray(assistantMsg.content)) {
toolCalls += assistantMsg.content.filter((c) => c.type === "toolCall").length;
}
const usage = assistantMsg.usage;
totalInput += usage.input;
totalOutput += usage.output;
totalCacheRead += usage.cacheRead;
totalCacheWrite += usage.cacheWrite;
totalCost += usage.cost.total;
}
}
@ -3050,7 +3062,7 @@ export class AgentSession {
assistantMessages,
toolCalls,
toolResults,
totalMessages: state.messages.length,
totalMessages,
tokens: {
input: totalInput,
output: totalOutput,

View file

@ -0,0 +1,164 @@
import type { AssistantMessage } from "@earendil-works/pi-ai";
import type { SessionEntry } from "./session-manager.ts";
/**
* Prompt-cache TTL: idle gaps longer than this are worth mentioning as the
* likely cause of a miss. Anthropic's default cache TTL is 5 minutes.
*/
export const CACHE_TTL_MS = 5 * 60 * 1000;
/** Per-turn misses at or below this are cache breakpoint granularity noise. */
const NOISE_FLOOR_TOKENS = 1024;
/** A counted cache miss on a single assistant message. */
export interface CacheMiss {
/** Prompt tokens that were in the previous turn's prompt but not read from cache. */
missedTokens: number;
/** Extra dollars paid vs. a full cache hit; 0 when pricing is unknown. */
missedCost: number;
/** Milliseconds since the previous request (which last refreshed the cache). */
idleMs: number;
/** True when the model changed relative to the previous request. */
modelChanged: boolean;
}
export interface CacheWasteTotals {
missedTokens: number;
missedCost: number;
/** Number of counted misses (turns above the noise floor). */
missCount: number;
}
/** Minimal pricing lookup, satisfied by ModelRegistry. Cost is $/million tokens. */
export interface ModelPriceSource {
find(provider: string, modelId: string): { cost: { cacheRead: number } } | undefined;
}
/** The last request seen by the scan; everything in its prompt should be cached. */
interface PreviousRequest {
promptTokens: number;
modelKey: string;
timestamp: number;
/**
* Sticky: some earlier request in this scan segment reported cache activity.
* Distinguishes a total miss on a cache-read-only provider (OpenAI-style,
* writes unreported) from a provider that never reports caching at all.
*/
reportedCache: boolean;
}
/**
* Compute the cache miss for one assistant message relative to the previous
* request. Returns undefined when nothing is counted: first turn, after a
* reset, no cache activity ever reported (provider without cache support), or
* miss below the noise floor.
*/
function detectMiss(
prev: PreviousRequest | undefined,
message: AssistantMessage,
models: ModelPriceSource,
): CacheMiss | undefined {
const usage = message.usage;
const promptTokens = usage.input + usage.cacheRead + usage.cacheWrite;
// A zero-cache turn only counts when cache activity was reported before:
// on cache-read-only providers that is a total miss, while on providers
// that never report caching it means nothing.
if (!prev || promptTokens <= 0 || (usage.cacheRead + usage.cacheWrite === 0 && !prev.reportedCache)) {
return undefined;
}
const missedTokens = Math.min(prev.promptTokens, promptTokens) - usage.cacheRead;
if (missedTokens <= NOISE_FLOOR_TOKENS) return undefined;
// Extra cost = missed tokens billed at the actual paid rate (input/cacheWrite,
// incl. write premium) instead of the cache-read rate. Missed tokens can only
// land in the input or cacheWrite buckets, so the paid rate comes straight
// from this message's own cost breakdown.
const paidTokens = usage.input + usage.cacheWrite;
const paidPerToken = paidTokens > 0 ? (usage.cost.input + usage.cost.cacheWrite) / paidTokens : 0;
const readPerToken =
usage.cacheRead > 0
? usage.cost.cacheRead / usage.cacheRead
: (models.find(message.provider, message.model)?.cost.cacheRead ?? 0) / 1_000_000;
return {
missedTokens,
missedCost: missedTokens * Math.max(0, paidPerToken - readPerToken),
idleMs: Math.max(0, message.timestamp - prev.timestamp),
modelChanged: `${message.provider}/${message.model}` !== prev.modelKey,
};
}
function asPreviousRequest(message: AssistantMessage, reportedCache: boolean): PreviousRequest | undefined {
const usage = message.usage;
const promptTokens = usage.input + usage.cacheRead + usage.cacheWrite;
if (promptTokens <= 0) return undefined;
return {
promptTokens,
modelKey: `${message.provider}/${message.model}`,
timestamp: message.timestamp,
reportedCache: reportedCache || usage.cacheRead + usage.cacheWrite > 0,
};
}
function scan(
entries: SessionEntry[],
models: ModelPriceSource,
): { prev: PreviousRequest | undefined; totals: CacheWasteTotals; misses: Map<AssistantMessage, CacheMiss> } {
let prev: PreviousRequest | undefined;
const totals: CacheWasteTotals = { missedTokens: 0, missedCost: 0, missCount: 0 };
const misses = new Map<AssistantMessage, CacheMiss>();
for (const entry of entries) {
if (entry.type === "compaction" || entry.type === "branch_summary") {
// The context legitimately changed; the next turn's prompt is new content,
// not re-billed content. Model switches are NOT exempt: they re-bill the
// full prompt and should be counted.
prev = undefined;
continue;
}
if (entry.type === "message" && entry.message.role === "assistant") {
const miss = detectMiss(prev, entry.message, models);
if (miss) {
totals.missedTokens += miss.missedTokens;
totals.missedCost += miss.missedCost;
totals.missCount += 1;
misses.set(entry.message, miss);
}
prev = asPreviousRequest(entry.message, prev?.reportedCache ?? false) ?? prev;
}
}
return { prev, totals, misses };
}
/**
* Cumulative cache waste across a session: prompt tokens that should have been
* cache reads (they were in the previous turn's prompt) but were re-billed.
*/
export function computeCacheWaste(entries: SessionEntry[], models: ModelPriceSource): CacheWasteTotals {
return scan(entries, models).totals;
}
/**
* All counted cache misses across a session, keyed by the assistant message
* (by reference) that paid for them. Used to re-derive transcript notices when
* rebuilding the chat from entries (resume, post-compaction rebuild).
*/
export function collectCacheMisses(
entries: SessionEntry[],
models: ModelPriceSource,
): Map<AssistantMessage, CacheMiss> {
return scan(entries, models).misses;
}
/**
* Detect a cache miss on a just-completed assistant message.
* `entries` must not yet contain `message` (message_end fires before persistence).
*/
export function detectCacheMiss(
entries: SessionEntry[],
message: AssistantMessage,
models: ModelPriceSource,
): CacheMiss | undefined {
return detectMiss(scan(entries, models).prev, message, models);
}

View file

@ -92,6 +92,7 @@ export interface Settings {
branchSummary?: BranchSummarySettings;
retry?: RetrySettings;
hideThinkingBlock?: boolean;
showCacheMissNotices?: boolean; // default: false - show transcript notices for significant prompt-cache misses
externalEditor?: string; // Command for Ctrl+G external editor; takes precedence over VISUAL/EDITOR
shellPath?: string; // Custom shell path (e.g., for Cygwin users on Windows)
quietStartup?: boolean;
@ -845,6 +846,10 @@ export class SettingsManager {
return this.settings.hideThinkingBlock ?? false;
}
getShowCacheMissNotices(): boolean {
return this.settings.showCacheMissNotices ?? false;
}
getExternalEditorCommand(): string | undefined {
const configuredEditor = this.settings.externalEditor;
if (typeof configuredEditor === "string" && configuredEditor.trim() !== "") {
@ -863,6 +868,12 @@ export class SettingsManager {
this.save();
}
setShowCacheMissNotices(show: boolean): void {
this.globalSettings.showCacheMissNotices = show;
this.markModified("showCacheMissNotices");
this.save();
}
getShellPath(): string | undefined {
return this.settings.shellPath;
}

View file

@ -20,7 +20,7 @@ function sanitizeStatusText(text: string): string {
/**
* Format token counts for compact footer display.
*/
function formatTokens(count: number): string {
export function formatTokens(count: number): string {
if (count < 1000) return count.toString();
if (count < 10000) return `${(count / 1000).toFixed(1)}k`;
if (count < 1000000) return `${Math.round(count / 1000)}k`;
@ -137,7 +137,6 @@ export class FooterComponent implements Component {
if ((totalCacheRead > 0 || totalCacheWrite > 0) && latestCacheHitRate !== undefined) {
statsParts.push(`CH${latestCacheHitRate.toFixed(1)}%`);
}
// Show cost with "(sub)" indicator if using OAuth subscription
const usingSubscription = state.model ? this.session.modelRegistry.isUsingOAuth(state.model) : false;
if (totalCost || usingSubscription) {

View file

@ -65,6 +65,7 @@ export interface SettingsConfig {
terminalTheme: TerminalTheme;
availableThemes: string[];
hideThinkingBlock: boolean;
showCacheMissNotices: boolean;
collapseChangelog: boolean;
enableInstallTelemetry: boolean;
doubleEscapeAction: "fork" | "tree" | "none";
@ -95,6 +96,7 @@ export interface SettingsCallbacks {
onThemeChange: (theme: string) => void;
onThemePreview?: (theme: string) => void;
onHideThinkingBlockChange: (hidden: boolean) => void;
onShowCacheMissNoticesChange: (shown: boolean) => void;
onCollapseChangelogChange: (collapsed: boolean) => void;
onEnableInstallTelemetryChange: (enabled: boolean) => void;
onDoubleEscapeActionChange: (action: "fork" | "tree" | "none") => void;
@ -521,6 +523,13 @@ export class SettingsSelectorComponent extends Container {
currentValue: config.hideThinkingBlock ? "true" : "false",
values: ["true", "false"],
},
{
id: "cache-miss-notices",
label: "Cache miss notices",
description: "Show transcript notices for significant prompt-cache misses",
currentValue: config.showCacheMissNotices ? "true" : "false",
values: ["true", "false"],
},
{
id: "collapse-changelog",
label: "Collapse changelog",
@ -764,6 +773,9 @@ export class SettingsSelectorComponent extends Container {
case "hide-thinking":
callbacks.onHideThinkingBlockChange(newValue === "true");
break;
case "cache-miss-notices":
callbacks.onShowCacheMissNoticesChange(newValue === "true");
break;
case "collapse-changelog":
callbacks.onCollapseChangelogChange(newValue === "true");
break;

View file

@ -60,6 +60,13 @@ import {
} from "../../config.ts";
import { type AgentSession, type AgentSessionEvent, parseSkillBlock } from "../../core/agent-session.ts";
import { type AgentSessionRuntime, SessionImportFileNotFoundError } from "../../core/agent-session-runtime.ts";
import {
CACHE_TTL_MS,
type CacheMiss,
collectCacheMisses,
computeCacheWaste,
detectCacheMiss,
} from "../../core/cache-stats.ts";
import type {
AutocompleteProviderFactory,
EditorFactory,
@ -111,7 +118,7 @@ import { EarendilAnnouncementComponent } from "./components/earendil-announcemen
import { ExtensionEditorComponent } from "./components/extension-editor.ts";
import { ExtensionInputComponent } from "./components/extension-input.ts";
import { ExtensionSelectorComponent } from "./components/extension-selector.ts";
import { FooterComponent } from "./components/footer.ts";
import { FooterComponent, formatTokens } from "./components/footer.ts";
import { formatKeyText, keyDisplayText, keyHint, keyText, rawKeyHint } from "./components/keybinding-hints.ts";
import { LoginDialogComponent } from "./components/login-dialog.ts";
import { ModelSelectorComponent } from "./components/model-selector.ts";
@ -2961,6 +2968,7 @@ export class InteractiveMode {
for (const [, component] of this.pendingTools.entries()) {
component.setArgsComplete();
}
this.maybeShowCacheMissNotice(this.streamingMessage);
}
this.streamingComponent = undefined;
this.streamingMessage = undefined;
@ -3277,6 +3285,11 @@ export class InteractiveMode {
): void {
this.pendingTools.clear();
const renderedPendingTools = new Map<string, ToolExecutionComponent>();
// Cache-miss notices are not persisted; re-derive them from the full entry
// list and re-inject them after the assistant messages that paid for them.
const cacheMisses = this.settingsManager.getShowCacheMissNotices()
? collectCacheMisses(this.sessionManager.getEntries(), this.session.modelRegistry)
: new Map<AssistantMessage, CacheMiss>();
if (options.updateFooter) {
this.footer.invalidate();
@ -3328,6 +3341,10 @@ export class InteractiveMode {
}
}
}
if (message.stopReason !== "aborted" && message.stopReason !== "error") {
const miss = cacheMisses.get(message);
if (miss) this.addCacheMissNotice(miss);
}
} else if (message.role === "toolResult") {
// Match tool results to pending tool components
const component = renderedPendingTools.get(message.toolCallId);
@ -3366,6 +3383,35 @@ export class InteractiveMode {
this.renderSessionItems(items, options);
}
/**
* Show a transcript notice when a completed assistant message paid for a
* significant cache miss. Only states observable facts: the miss itself,
* a model switch, or an idle gap past the cache TTL.
*/
private maybeShowCacheMissNotice(message: AssistantMessage): void {
if (!this.settingsManager.getShowCacheMissNotices()) return;
// Entries don't contain `message` yet: message_end fires before persistence.
const miss = detectCacheMiss(this.sessionManager.getEntries(), message, this.session.modelRegistry);
if (miss) this.addCacheMissNotice(miss);
}
private addCacheMissNotice(miss: CacheMiss): void {
if (miss.missedTokens < 20_000 && miss.missedCost < 0.1) return;
const cost = miss.missedCost >= 0.01 ? ` (~$${miss.missedCost.toFixed(2)})` : "";
const reBilled = `${formatTokens(miss.missedTokens)} tokens re-billed${cost}`;
let label = "Cache miss";
if (miss.modelChanged) {
label = "Cache miss after model switch";
} else if (miss.idleMs >= CACHE_TTL_MS) {
label = `Cache miss after ${Math.round(miss.idleMs / 60_000)}m idle`;
}
const text = theme.fg("warning", `${label}: ${reBilled}`);
this.chatContainer.addChild(new Spacer(1));
this.chatContainer.addChild(new Text(text, 1, 0));
}
renderInitialMessages(): void {
const entries = this.sessionManager.buildContextEntries();
this.renderSessionEntries(entries, {
@ -4088,6 +4134,7 @@ export class InteractiveMode {
doubleEscapeAction: this.settingsManager.getDoubleEscapeAction(),
treeFilterMode: this.settingsManager.getTreeFilterMode(),
showHardwareCursor: this.settingsManager.getShowHardwareCursor(),
showCacheMissNotices: this.settingsManager.getShowCacheMissNotices(),
defaultProjectTrust: this.settingsManager.getDefaultProjectTrust(),
editorPaddingX: this.settingsManager.getEditorPaddingX(),
outputPad: this.settingsManager.getOutputPad(),
@ -4164,6 +4211,10 @@ export class InteractiveMode {
this.chatContainer.clear();
this.rebuildChatFromMessages();
},
onShowCacheMissNoticesChange: (shown) => {
this.settingsManager.setShowCacheMissNotices(shown);
this.rebuildChatFromMessages();
},
onCollapseChangelogChange: (collapsed) => {
this.settingsManager.setCollapseChangelog(collapsed);
},
@ -5561,6 +5612,26 @@ export class InteractiveMode {
private handleSessionCommand(): void {
const stats = this.session.getSessionStats();
const sessionName = this.sessionManager.getSessionName();
const entries = this.sessionManager.getEntries();
const cacheWaste = computeCacheWaste(entries, this.session.modelRegistry);
// Cost/token totals per provider/model actually used (e.g. OpenRouter `auto`
// resolves to a concrete responseModel), sorted by cost descending.
const perModelMap = new Map<string, { key: string; cost: number; tokens: number }>();
for (const entry of entries) {
if (entry.type !== "message" || entry.message.role !== "assistant") continue;
const message = entry.message;
const usage = message.usage;
const key = `${message.provider}/${message.responseModel ?? message.model}`;
let bucket = perModelMap.get(key);
if (!bucket) {
bucket = { key, cost: 0, tokens: 0 };
perModelMap.set(key, bucket);
}
bucket.cost += usage.cost.total;
bucket.tokens += usage.input + usage.output + usage.cacheRead + usage.cacheWrite;
}
const perModel = Array.from(perModelMap.values()).sort((a, b) => b.cost - a.cost);
let info = `${theme.bold("Session Info")}\n\n`;
if (sessionName) {
@ -5569,25 +5640,44 @@ export class InteractiveMode {
info += `${theme.fg("dim", "File:")} ${stats.sessionFile ?? "In-memory"}\n`;
info += `${theme.fg("dim", "ID:")} ${stats.sessionId}\n\n`;
info += `${theme.bold("Messages")}\n`;
info += `${theme.fg("dim", "Total:")} ${stats.totalMessages}\n`;
info += `${theme.fg("dim", "User:")} ${stats.userMessages}\n`;
info += `${theme.fg("dim", "Assistant:")} ${stats.assistantMessages}\n`;
info += `${theme.fg("dim", "Tool Calls:")} ${stats.toolCalls}\n`;
info += `${theme.fg("dim", "Tool Results:")} ${stats.toolResults}\n`;
info += `${theme.fg("dim", "Total:")} ${stats.totalMessages}\n\n`;
info += `${theme.fg("dim", "Tools:")} ${stats.toolCalls} calls, ${stats.toolResults} results\n\n`;
info += `${theme.bold("Tokens")}\n`;
info += `${theme.fg("dim", "Input:")} ${stats.tokens.input.toLocaleString()}\n`;
// "Input" is the full prompt volume. With cache activity, split it into
// cached (served from cache) vs uncached (everything else) - the only
// provider-independent split. Cache writes, where reported, are a detail
// of the uncached portion.
const { input, cacheRead, cacheWrite } = stats.tokens;
const promptTokens = input + cacheRead + cacheWrite;
info += `${theme.fg("dim", "Input:")} ${promptTokens.toLocaleString()}\n`;
if (promptTokens > 0 && (cacheRead > 0 || cacheWrite > 0)) {
const hitRate = theme.fg("dim", `(${((cacheRead / promptTokens) * 100).toFixed(1)}%)`);
info += ` ${theme.fg("dim", "Cached:")} ${cacheRead.toLocaleString()} ${hitRate}\n`;
const written =
cacheWrite > 0 ? ` ${theme.fg("dim", `(${cacheWrite.toLocaleString()} written to cache)`)}` : "";
info += ` ${theme.fg("dim", "Uncached:")} ${(input + cacheWrite).toLocaleString()}${written}\n`;
}
info += `${theme.fg("dim", "Output:")} ${stats.tokens.output.toLocaleString()}\n`;
if (stats.tokens.cacheRead > 0) {
info += `${theme.fg("dim", "Cache Read:")} ${stats.tokens.cacheRead.toLocaleString()}\n`;
}
if (stats.tokens.cacheWrite > 0) {
info += `${theme.fg("dim", "Cache Write:")} ${stats.tokens.cacheWrite.toLocaleString()}\n`;
}
info += `${theme.fg("dim", "Total:")} ${stats.tokens.total.toLocaleString()}\n`;
if (stats.cost > 0) {
if (stats.cost > 0 || cacheWaste.missedTokens > 0) {
info += `\n${theme.bold("Cost")}\n`;
info += `${theme.fg("dim", "Total:")} ${stats.cost.toFixed(4)}`;
info += `${theme.fg("dim", "Total:")} $${stats.cost.toFixed(3)}`;
if (perModel.length > 1) {
for (const entry of perModel) {
info += `\n ${theme.fg("dim", `${entry.key}:`)} $${entry.cost.toFixed(3)} ${theme.fg("dim", `(${formatTokens(entry.tokens)} tokens)`)}`;
}
}
if (cacheWaste.missedTokens > 0) {
const missLabel = cacheWaste.missCount === 1 ? "1 miss" : `${cacheWaste.missCount} misses`;
const detail = `${cacheWaste.missedTokens.toLocaleString()} tokens, ${missLabel}`;
info +=
cacheWaste.missedCost >= 0.0001
? `\n${theme.fg("dim", "Cache Re-billed:")} $${cacheWaste.missedCost.toFixed(3)} ${theme.fg("dim", `(${detail})`)}`
: `\n${theme.fg("dim", "Cache Re-billed:")} ${detail}`;
}
}
this.chatContainer.addChild(new Spacer(1));

View file

@ -109,7 +109,8 @@ describe("AgentSession.getSessionStats", () => {
syncAgentMessages(session, sessionManager);
const stats = session.getSessionStats();
expect(stats.tokens.input).toBe(195_000);
// Totals cover ALL entries, including history compacted away (180k + 195k).
expect(stats.tokens.input).toBe(375_000);
expect(stats.contextUsage).toBeDefined();
expect(stats.contextUsage?.tokens).toBeNull();
expect(stats.contextUsage?.percent).toBeNull();
@ -132,7 +133,8 @@ describe("AgentSession.getSessionStats", () => {
syncAgentMessages(session, sessionManager);
const stats = session.getSessionStats();
expect(stats.tokens.input).toBe(220_000);
// Totals cover ALL entries, including history compacted away (180k + 195k + 25k).
expect(stats.tokens.input).toBe(400_000);
expect(stats.contextUsage).toBeDefined();
expect(stats.contextUsage?.tokens).toBe(25_000);
expect(stats.contextUsage?.percent).toBe((25_000 / model.contextWindow) * 100);

View file

@ -0,0 +1,143 @@
import type { AssistantMessage } from "@earendil-works/pi-ai";
import { describe, expect, it } from "vitest";
import {
collectCacheMisses,
computeCacheWaste,
detectCacheMiss,
type ModelPriceSource,
} from "../src/core/cache-stats.ts";
import type { SessionEntry } from "../src/core/session-manager.ts";
const zeroCost = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 };
const models: ModelPriceSource = {
// $/million tokens; used as cache-read price fallback on full-miss turns
find: () => ({ cost: { cacheRead: 0.3 } }),
};
function assistant(options: {
input?: number;
cacheRead?: number;
cacheWrite?: number;
cost?: Partial<typeof zeroCost>;
model?: string;
timestamp?: number;
}): AssistantMessage {
return {
role: "assistant",
content: [],
api: "anthropic-messages",
provider: "test",
model: options.model ?? "test-model",
usage: {
input: options.input ?? 0,
output: 10,
cacheRead: options.cacheRead ?? 0,
cacheWrite: options.cacheWrite ?? 0,
totalTokens: 0,
cost: { ...zeroCost, ...options.cost },
},
stopReason: "stop",
timestamp: options.timestamp ?? 0,
} as AssistantMessage;
}
function entry(message: AssistantMessage): SessionEntry {
return { type: "message", id: "x", parentId: null, timestamp: "", message } as SessionEntry;
}
// Turn 1: fresh 100k cache write at $3.75/M
const turn1 = assistant({ cacheWrite: 100_000, cost: { cacheWrite: 0.375 }, timestamp: 0 });
// Turn 2: healthy, everything read back at $0.30/M
const turn2 = assistant({
cacheRead: 100_000,
cacheWrite: 5_000,
cost: { cacheRead: 0.03, cacheWrite: 0.019 },
timestamp: 60_000,
});
describe("computeCacheWaste", () => {
it("accumulates missed tokens and cost across turns", () => {
// Turn 3: full miss, previous 105k prompt re-billed at $3.75/M write
const turn3 = assistant({ cacheWrite: 110_000, cost: { cacheWrite: 0.4125 }, timestamp: 120_000 });
const totals = computeCacheWaste([entry(turn1), entry(turn2), entry(turn3)], models);
expect(totals.missedTokens).toBe(105_000);
// 105k at ($3.75 - $0.30)/M
expect(totals.missedCost).toBeCloseTo(0.36225, 5);
});
it("counts nothing for healthy sessions", () => {
const totals = computeCacheWaste([entry(turn1), entry(turn2)], models);
expect(totals.missedTokens).toBe(0);
expect(totals.missedCost).toBe(0);
});
it("skips the turn after a compaction reset", () => {
const reset = { type: "compaction", id: "c", parentId: null, timestamp: "" } as SessionEntry;
const afterReset = assistant({ cacheWrite: 20_000, cost: { cacheWrite: 0.075 } });
const totals = computeCacheWaste([entry(turn1), reset, entry(afterReset)], models);
expect(totals.missedTokens).toBe(0);
});
it("counts misses caused by model switches", () => {
const otherModel = assistant({ cacheWrite: 100_000, cost: { cacheWrite: 0.375 }, model: "other-model" });
const totals = computeCacheWaste([entry(turn1), entry(otherModel)], models);
expect(totals.missedTokens).toBe(100_000);
expect(totals.missCount).toBe(1);
});
it("skips providers that report no cache activity", () => {
const a = assistant({ input: 100_000 });
const b = assistant({ input: 110_000 });
const totals = computeCacheWaste([entry(a), entry(b)], models);
expect(totals.missedTokens).toBe(0);
});
});
describe("collectCacheMisses", () => {
it("maps counted misses to their assistant messages by reference", () => {
const missTurn = assistant({ cacheWrite: 110_000, cost: { cacheWrite: 0.4125 }, timestamp: 120_000 });
const misses = collectCacheMisses([entry(turn1), entry(turn2), entry(missTurn)], models);
expect(misses.size).toBe(1);
expect(misses.get(missTurn)?.missedTokens).toBe(105_000);
});
});
describe("detectCacheMiss", () => {
it("detects a miss on a just-completed message with idle time", () => {
const missMessage = assistant({ cacheWrite: 110_000, cost: { cacheWrite: 0.4125 }, timestamp: 600_000 });
const miss = detectCacheMiss([entry(turn1), entry(turn2)], missMessage, models);
expect(miss).toBeDefined();
expect(miss?.missedTokens).toBe(105_000);
expect(miss?.missedCost).toBeCloseTo(0.36225, 5);
// 600s - 60s since the previous request
expect(miss?.idleMs).toBe(540_000);
expect(miss?.modelChanged).toBe(false);
});
it("flags model switches on detected misses", () => {
const otherModel = assistant({
cacheWrite: 110_000,
cost: { cacheWrite: 0.4125 },
model: "other-model",
timestamp: 120_000,
});
const miss = detectCacheMiss([entry(turn1), entry(turn2)], otherModel, models);
expect(miss?.missedTokens).toBe(105_000);
expect(miss?.modelChanged).toBe(true);
});
it("returns undefined for healthy turns", () => {
const healthy = assistant({
cacheRead: 105_000,
cacheWrite: 2_000,
cost: { cacheRead: 0.0315, cacheWrite: 0.0075 },
timestamp: 120_000,
});
expect(detectCacheMiss([entry(turn1), entry(turn2)], healthy, models)).toBeUndefined();
});
it("returns undefined for the first turn of a session", () => {
expect(detectCacheMiss([], turn1, models)).toBeUndefined();
});
});

View file

@ -42,8 +42,8 @@ type RenderSessionContextThis = {
getShowImages(): boolean;
getImageWidthCells(): number;
};
sessionManager: { getCwd(): string };
session: { retryAttempt: number };
sessionManager: { getCwd(): string; getEntries(): SessionEntry[] };
session: { retryAttempt: number; modelRegistry: { find(provider: string, modelId: string): undefined } };
toolOutputExpanded: boolean;
isInitialized: boolean;
updateEditorBorderColor(): void;
@ -71,8 +71,8 @@ function createFakeInteractiveModeThis(): RenderSessionContextThis {
getShowImages: () => false,
getImageWidthCells: () => 60,
},
sessionManager: { getCwd: () => process.cwd() },
session: { retryAttempt: 0 },
sessionManager: { getCwd: () => process.cwd(), getEntries: () => [] },
session: { retryAttempt: 0, modelRegistry: { find: () => undefined } },
toolOutputExpanded: false,
isInitialized: true,
updateEditorBorderColor: vi.fn(),