mirror of
https://github.com/badlogic/pi-mono.git
synced 2026-07-09 17:28:45 +00:00
feat(coding-agent): add prompt cache miss tracking (#6427)
Some checks are pending
CI / build-check-test (push) Waiting to run
Some checks are pending
CI / build-check-test (push) Waiting to run
Detect prompt cache misses per turn by comparing each assistant message's cache reads against the previous request's prompt tokens (core/cache-stats.ts). Significant misses emit a warning-colored transcript notice at the turn they occur, noting idle gaps past the cache TTL and model switches when relevant. /session gained cache statistics: a compact token/cache breakdown with hit rate, a $-prefixed cost section with per-model cost breakdown, and the cumulative cost re-billed due to cache misses.
This commit is contained in:
parent
57d96d72ed
commit
3f9aa5d10b
11 changed files with 471 additions and 36 deletions
|
|
@ -8,6 +8,7 @@
|
|||
- Added public SDK exports for CLI-equivalent model and scoped-model resolution ([#6201](https://github.com/earendil-works/pi/issues/6201)).
|
||||
- Added extension and RPC `agent_settled` events plus session-level idle waiting for fully settled agent runs ([#6363](https://github.com/earendil-works/pi/issues/6363)).
|
||||
- Added extension entry renderers for persisted display-only session entries that are rendered in interactive mode without being sent to the model context.
|
||||
- Added a `showCacheMissNotices` setting and `/settings` toggle for significant prompt-cache miss transcript notices.
|
||||
|
||||
### Fixed
|
||||
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ Use `/trust` in interactive mode to save a project trust decision for future ses
|
|||
| `defaultModel` | string | - | Default model ID |
|
||||
| `defaultThinkingLevel` | string | - | `"off"`, `"minimal"`, `"low"`, `"medium"`, `"high"`, `"xhigh"` |
|
||||
| `hideThinkingBlock` | boolean | `false` | Hide thinking blocks in output |
|
||||
| `showCacheMissNotices` | boolean | `false` | Show transcript notices for significant prompt-cache misses |
|
||||
| `thinkingBudgets` | object | - | Custom token budgets per thinking level |
|
||||
|
||||
#### thinkingBudgets
|
||||
|
|
|
|||
|
|
@ -3016,14 +3016,15 @@ export class AgentSession {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get session statistics.
|
||||
* Get session statistics. Aggregates over ALL session entries (including
|
||||
* history that was compacted away), so token/cost totals reflect what was
|
||||
* actually billed across the session.
|
||||
*/
|
||||
getSessionStats(): SessionStats {
|
||||
const state = this.state;
|
||||
const userMessages = state.messages.filter((m) => m.role === "user").length;
|
||||
const assistantMessages = state.messages.filter((m) => m.role === "assistant").length;
|
||||
const toolResults = state.messages.filter((m) => m.role === "toolResult").length;
|
||||
|
||||
let userMessages = 0;
|
||||
let assistantMessages = 0;
|
||||
let toolResults = 0;
|
||||
let totalMessages = 0;
|
||||
let toolCalls = 0;
|
||||
let totalInput = 0;
|
||||
let totalOutput = 0;
|
||||
|
|
@ -3031,15 +3032,26 @@ export class AgentSession {
|
|||
let totalCacheWrite = 0;
|
||||
let totalCost = 0;
|
||||
|
||||
for (const message of state.messages) {
|
||||
if (message.role === "assistant") {
|
||||
for (const entry of this.sessionManager.getEntries()) {
|
||||
if (entry.type !== "message") continue;
|
||||
totalMessages++;
|
||||
const message = entry.message;
|
||||
if (message.role === "user") {
|
||||
userMessages++;
|
||||
} else if (message.role === "toolResult") {
|
||||
toolResults++;
|
||||
} else if (message.role === "assistant") {
|
||||
assistantMessages++;
|
||||
const assistantMsg = message as AssistantMessage;
|
||||
toolCalls += assistantMsg.content.filter((c) => c.type === "toolCall").length;
|
||||
totalInput += assistantMsg.usage.input;
|
||||
totalOutput += assistantMsg.usage.output;
|
||||
totalCacheRead += assistantMsg.usage.cacheRead;
|
||||
totalCacheWrite += assistantMsg.usage.cacheWrite;
|
||||
totalCost += assistantMsg.usage.cost.total;
|
||||
if (Array.isArray(assistantMsg.content)) {
|
||||
toolCalls += assistantMsg.content.filter((c) => c.type === "toolCall").length;
|
||||
}
|
||||
const usage = assistantMsg.usage;
|
||||
totalInput += usage.input;
|
||||
totalOutput += usage.output;
|
||||
totalCacheRead += usage.cacheRead;
|
||||
totalCacheWrite += usage.cacheWrite;
|
||||
totalCost += usage.cost.total;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -3050,7 +3062,7 @@ export class AgentSession {
|
|||
assistantMessages,
|
||||
toolCalls,
|
||||
toolResults,
|
||||
totalMessages: state.messages.length,
|
||||
totalMessages,
|
||||
tokens: {
|
||||
input: totalInput,
|
||||
output: totalOutput,
|
||||
|
|
|
|||
164
packages/coding-agent/src/core/cache-stats.ts
Normal file
164
packages/coding-agent/src/core/cache-stats.ts
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
import type { AssistantMessage } from "@earendil-works/pi-ai";
|
||||
import type { SessionEntry } from "./session-manager.ts";
|
||||
|
||||
/**
|
||||
* Prompt-cache TTL: idle gaps longer than this are worth mentioning as the
|
||||
* likely cause of a miss. Anthropic's default cache TTL is 5 minutes.
|
||||
*/
|
||||
export const CACHE_TTL_MS = 5 * 60 * 1000;
|
||||
|
||||
/** Per-turn misses at or below this are cache breakpoint granularity noise. */
|
||||
const NOISE_FLOOR_TOKENS = 1024;
|
||||
|
||||
/** A counted cache miss on a single assistant message. */
|
||||
export interface CacheMiss {
|
||||
/** Prompt tokens that were in the previous turn's prompt but not read from cache. */
|
||||
missedTokens: number;
|
||||
/** Extra dollars paid vs. a full cache hit; 0 when pricing is unknown. */
|
||||
missedCost: number;
|
||||
/** Milliseconds since the previous request (which last refreshed the cache). */
|
||||
idleMs: number;
|
||||
/** True when the model changed relative to the previous request. */
|
||||
modelChanged: boolean;
|
||||
}
|
||||
|
||||
export interface CacheWasteTotals {
|
||||
missedTokens: number;
|
||||
missedCost: number;
|
||||
/** Number of counted misses (turns above the noise floor). */
|
||||
missCount: number;
|
||||
}
|
||||
|
||||
/** Minimal pricing lookup, satisfied by ModelRegistry. Cost is $/million tokens. */
|
||||
export interface ModelPriceSource {
|
||||
find(provider: string, modelId: string): { cost: { cacheRead: number } } | undefined;
|
||||
}
|
||||
|
||||
/** The last request seen by the scan; everything in its prompt should be cached. */
|
||||
interface PreviousRequest {
|
||||
promptTokens: number;
|
||||
modelKey: string;
|
||||
timestamp: number;
|
||||
/**
|
||||
* Sticky: some earlier request in this scan segment reported cache activity.
|
||||
* Distinguishes a total miss on a cache-read-only provider (OpenAI-style,
|
||||
* writes unreported) from a provider that never reports caching at all.
|
||||
*/
|
||||
reportedCache: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the cache miss for one assistant message relative to the previous
|
||||
* request. Returns undefined when nothing is counted: first turn, after a
|
||||
* reset, no cache activity ever reported (provider without cache support), or
|
||||
* miss below the noise floor.
|
||||
*/
|
||||
function detectMiss(
|
||||
prev: PreviousRequest | undefined,
|
||||
message: AssistantMessage,
|
||||
models: ModelPriceSource,
|
||||
): CacheMiss | undefined {
|
||||
const usage = message.usage;
|
||||
const promptTokens = usage.input + usage.cacheRead + usage.cacheWrite;
|
||||
// A zero-cache turn only counts when cache activity was reported before:
|
||||
// on cache-read-only providers that is a total miss, while on providers
|
||||
// that never report caching it means nothing.
|
||||
if (!prev || promptTokens <= 0 || (usage.cacheRead + usage.cacheWrite === 0 && !prev.reportedCache)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const missedTokens = Math.min(prev.promptTokens, promptTokens) - usage.cacheRead;
|
||||
if (missedTokens <= NOISE_FLOOR_TOKENS) return undefined;
|
||||
|
||||
// Extra cost = missed tokens billed at the actual paid rate (input/cacheWrite,
|
||||
// incl. write premium) instead of the cache-read rate. Missed tokens can only
|
||||
// land in the input or cacheWrite buckets, so the paid rate comes straight
|
||||
// from this message's own cost breakdown.
|
||||
const paidTokens = usage.input + usage.cacheWrite;
|
||||
const paidPerToken = paidTokens > 0 ? (usage.cost.input + usage.cost.cacheWrite) / paidTokens : 0;
|
||||
const readPerToken =
|
||||
usage.cacheRead > 0
|
||||
? usage.cost.cacheRead / usage.cacheRead
|
||||
: (models.find(message.provider, message.model)?.cost.cacheRead ?? 0) / 1_000_000;
|
||||
|
||||
return {
|
||||
missedTokens,
|
||||
missedCost: missedTokens * Math.max(0, paidPerToken - readPerToken),
|
||||
idleMs: Math.max(0, message.timestamp - prev.timestamp),
|
||||
modelChanged: `${message.provider}/${message.model}` !== prev.modelKey,
|
||||
};
|
||||
}
|
||||
|
||||
function asPreviousRequest(message: AssistantMessage, reportedCache: boolean): PreviousRequest | undefined {
|
||||
const usage = message.usage;
|
||||
const promptTokens = usage.input + usage.cacheRead + usage.cacheWrite;
|
||||
if (promptTokens <= 0) return undefined;
|
||||
return {
|
||||
promptTokens,
|
||||
modelKey: `${message.provider}/${message.model}`,
|
||||
timestamp: message.timestamp,
|
||||
reportedCache: reportedCache || usage.cacheRead + usage.cacheWrite > 0,
|
||||
};
|
||||
}
|
||||
|
||||
function scan(
|
||||
entries: SessionEntry[],
|
||||
models: ModelPriceSource,
|
||||
): { prev: PreviousRequest | undefined; totals: CacheWasteTotals; misses: Map<AssistantMessage, CacheMiss> } {
|
||||
let prev: PreviousRequest | undefined;
|
||||
const totals: CacheWasteTotals = { missedTokens: 0, missedCost: 0, missCount: 0 };
|
||||
const misses = new Map<AssistantMessage, CacheMiss>();
|
||||
|
||||
for (const entry of entries) {
|
||||
if (entry.type === "compaction" || entry.type === "branch_summary") {
|
||||
// The context legitimately changed; the next turn's prompt is new content,
|
||||
// not re-billed content. Model switches are NOT exempt: they re-bill the
|
||||
// full prompt and should be counted.
|
||||
prev = undefined;
|
||||
continue;
|
||||
}
|
||||
if (entry.type === "message" && entry.message.role === "assistant") {
|
||||
const miss = detectMiss(prev, entry.message, models);
|
||||
if (miss) {
|
||||
totals.missedTokens += miss.missedTokens;
|
||||
totals.missedCost += miss.missedCost;
|
||||
totals.missCount += 1;
|
||||
misses.set(entry.message, miss);
|
||||
}
|
||||
prev = asPreviousRequest(entry.message, prev?.reportedCache ?? false) ?? prev;
|
||||
}
|
||||
}
|
||||
return { prev, totals, misses };
|
||||
}
|
||||
|
||||
/**
|
||||
* Cumulative cache waste across a session: prompt tokens that should have been
|
||||
* cache reads (they were in the previous turn's prompt) but were re-billed.
|
||||
*/
|
||||
export function computeCacheWaste(entries: SessionEntry[], models: ModelPriceSource): CacheWasteTotals {
|
||||
return scan(entries, models).totals;
|
||||
}
|
||||
|
||||
/**
|
||||
* All counted cache misses across a session, keyed by the assistant message
|
||||
* (by reference) that paid for them. Used to re-derive transcript notices when
|
||||
* rebuilding the chat from entries (resume, post-compaction rebuild).
|
||||
*/
|
||||
export function collectCacheMisses(
|
||||
entries: SessionEntry[],
|
||||
models: ModelPriceSource,
|
||||
): Map<AssistantMessage, CacheMiss> {
|
||||
return scan(entries, models).misses;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect a cache miss on a just-completed assistant message.
|
||||
* `entries` must not yet contain `message` (message_end fires before persistence).
|
||||
*/
|
||||
export function detectCacheMiss(
|
||||
entries: SessionEntry[],
|
||||
message: AssistantMessage,
|
||||
models: ModelPriceSource,
|
||||
): CacheMiss | undefined {
|
||||
return detectMiss(scan(entries, models).prev, message, models);
|
||||
}
|
||||
|
|
@ -92,6 +92,7 @@ export interface Settings {
|
|||
branchSummary?: BranchSummarySettings;
|
||||
retry?: RetrySettings;
|
||||
hideThinkingBlock?: boolean;
|
||||
showCacheMissNotices?: boolean; // default: false - show transcript notices for significant prompt-cache misses
|
||||
externalEditor?: string; // Command for Ctrl+G external editor; takes precedence over VISUAL/EDITOR
|
||||
shellPath?: string; // Custom shell path (e.g., for Cygwin users on Windows)
|
||||
quietStartup?: boolean;
|
||||
|
|
@ -845,6 +846,10 @@ export class SettingsManager {
|
|||
return this.settings.hideThinkingBlock ?? false;
|
||||
}
|
||||
|
||||
getShowCacheMissNotices(): boolean {
|
||||
return this.settings.showCacheMissNotices ?? false;
|
||||
}
|
||||
|
||||
getExternalEditorCommand(): string | undefined {
|
||||
const configuredEditor = this.settings.externalEditor;
|
||||
if (typeof configuredEditor === "string" && configuredEditor.trim() !== "") {
|
||||
|
|
@ -863,6 +868,12 @@ export class SettingsManager {
|
|||
this.save();
|
||||
}
|
||||
|
||||
setShowCacheMissNotices(show: boolean): void {
|
||||
this.globalSettings.showCacheMissNotices = show;
|
||||
this.markModified("showCacheMissNotices");
|
||||
this.save();
|
||||
}
|
||||
|
||||
getShellPath(): string | undefined {
|
||||
return this.settings.shellPath;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ function sanitizeStatusText(text: string): string {
|
|||
/**
|
||||
* Format token counts for compact footer display.
|
||||
*/
|
||||
function formatTokens(count: number): string {
|
||||
export function formatTokens(count: number): string {
|
||||
if (count < 1000) return count.toString();
|
||||
if (count < 10000) return `${(count / 1000).toFixed(1)}k`;
|
||||
if (count < 1000000) return `${Math.round(count / 1000)}k`;
|
||||
|
|
@ -137,7 +137,6 @@ export class FooterComponent implements Component {
|
|||
if ((totalCacheRead > 0 || totalCacheWrite > 0) && latestCacheHitRate !== undefined) {
|
||||
statsParts.push(`CH${latestCacheHitRate.toFixed(1)}%`);
|
||||
}
|
||||
|
||||
// Show cost with "(sub)" indicator if using OAuth subscription
|
||||
const usingSubscription = state.model ? this.session.modelRegistry.isUsingOAuth(state.model) : false;
|
||||
if (totalCost || usingSubscription) {
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@ export interface SettingsConfig {
|
|||
terminalTheme: TerminalTheme;
|
||||
availableThemes: string[];
|
||||
hideThinkingBlock: boolean;
|
||||
showCacheMissNotices: boolean;
|
||||
collapseChangelog: boolean;
|
||||
enableInstallTelemetry: boolean;
|
||||
doubleEscapeAction: "fork" | "tree" | "none";
|
||||
|
|
@ -95,6 +96,7 @@ export interface SettingsCallbacks {
|
|||
onThemeChange: (theme: string) => void;
|
||||
onThemePreview?: (theme: string) => void;
|
||||
onHideThinkingBlockChange: (hidden: boolean) => void;
|
||||
onShowCacheMissNoticesChange: (shown: boolean) => void;
|
||||
onCollapseChangelogChange: (collapsed: boolean) => void;
|
||||
onEnableInstallTelemetryChange: (enabled: boolean) => void;
|
||||
onDoubleEscapeActionChange: (action: "fork" | "tree" | "none") => void;
|
||||
|
|
@ -521,6 +523,13 @@ export class SettingsSelectorComponent extends Container {
|
|||
currentValue: config.hideThinkingBlock ? "true" : "false",
|
||||
values: ["true", "false"],
|
||||
},
|
||||
{
|
||||
id: "cache-miss-notices",
|
||||
label: "Cache miss notices",
|
||||
description: "Show transcript notices for significant prompt-cache misses",
|
||||
currentValue: config.showCacheMissNotices ? "true" : "false",
|
||||
values: ["true", "false"],
|
||||
},
|
||||
{
|
||||
id: "collapse-changelog",
|
||||
label: "Collapse changelog",
|
||||
|
|
@ -764,6 +773,9 @@ export class SettingsSelectorComponent extends Container {
|
|||
case "hide-thinking":
|
||||
callbacks.onHideThinkingBlockChange(newValue === "true");
|
||||
break;
|
||||
case "cache-miss-notices":
|
||||
callbacks.onShowCacheMissNoticesChange(newValue === "true");
|
||||
break;
|
||||
case "collapse-changelog":
|
||||
callbacks.onCollapseChangelogChange(newValue === "true");
|
||||
break;
|
||||
|
|
|
|||
|
|
@ -60,6 +60,13 @@ import {
|
|||
} from "../../config.ts";
|
||||
import { type AgentSession, type AgentSessionEvent, parseSkillBlock } from "../../core/agent-session.ts";
|
||||
import { type AgentSessionRuntime, SessionImportFileNotFoundError } from "../../core/agent-session-runtime.ts";
|
||||
import {
|
||||
CACHE_TTL_MS,
|
||||
type CacheMiss,
|
||||
collectCacheMisses,
|
||||
computeCacheWaste,
|
||||
detectCacheMiss,
|
||||
} from "../../core/cache-stats.ts";
|
||||
import type {
|
||||
AutocompleteProviderFactory,
|
||||
EditorFactory,
|
||||
|
|
@ -111,7 +118,7 @@ import { EarendilAnnouncementComponent } from "./components/earendil-announcemen
|
|||
import { ExtensionEditorComponent } from "./components/extension-editor.ts";
|
||||
import { ExtensionInputComponent } from "./components/extension-input.ts";
|
||||
import { ExtensionSelectorComponent } from "./components/extension-selector.ts";
|
||||
import { FooterComponent } from "./components/footer.ts";
|
||||
import { FooterComponent, formatTokens } from "./components/footer.ts";
|
||||
import { formatKeyText, keyDisplayText, keyHint, keyText, rawKeyHint } from "./components/keybinding-hints.ts";
|
||||
import { LoginDialogComponent } from "./components/login-dialog.ts";
|
||||
import { ModelSelectorComponent } from "./components/model-selector.ts";
|
||||
|
|
@ -2961,6 +2968,7 @@ export class InteractiveMode {
|
|||
for (const [, component] of this.pendingTools.entries()) {
|
||||
component.setArgsComplete();
|
||||
}
|
||||
this.maybeShowCacheMissNotice(this.streamingMessage);
|
||||
}
|
||||
this.streamingComponent = undefined;
|
||||
this.streamingMessage = undefined;
|
||||
|
|
@ -3277,6 +3285,11 @@ export class InteractiveMode {
|
|||
): void {
|
||||
this.pendingTools.clear();
|
||||
const renderedPendingTools = new Map<string, ToolExecutionComponent>();
|
||||
// Cache-miss notices are not persisted; re-derive them from the full entry
|
||||
// list and re-inject them after the assistant messages that paid for them.
|
||||
const cacheMisses = this.settingsManager.getShowCacheMissNotices()
|
||||
? collectCacheMisses(this.sessionManager.getEntries(), this.session.modelRegistry)
|
||||
: new Map<AssistantMessage, CacheMiss>();
|
||||
|
||||
if (options.updateFooter) {
|
||||
this.footer.invalidate();
|
||||
|
|
@ -3328,6 +3341,10 @@ export class InteractiveMode {
|
|||
}
|
||||
}
|
||||
}
|
||||
if (message.stopReason !== "aborted" && message.stopReason !== "error") {
|
||||
const miss = cacheMisses.get(message);
|
||||
if (miss) this.addCacheMissNotice(miss);
|
||||
}
|
||||
} else if (message.role === "toolResult") {
|
||||
// Match tool results to pending tool components
|
||||
const component = renderedPendingTools.get(message.toolCallId);
|
||||
|
|
@ -3366,6 +3383,35 @@ export class InteractiveMode {
|
|||
this.renderSessionItems(items, options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Show a transcript notice when a completed assistant message paid for a
|
||||
* significant cache miss. Only states observable facts: the miss itself,
|
||||
* a model switch, or an idle gap past the cache TTL.
|
||||
*/
|
||||
private maybeShowCacheMissNotice(message: AssistantMessage): void {
|
||||
if (!this.settingsManager.getShowCacheMissNotices()) return;
|
||||
|
||||
// Entries don't contain `message` yet: message_end fires before persistence.
|
||||
const miss = detectCacheMiss(this.sessionManager.getEntries(), message, this.session.modelRegistry);
|
||||
if (miss) this.addCacheMissNotice(miss);
|
||||
}
|
||||
|
||||
private addCacheMissNotice(miss: CacheMiss): void {
|
||||
if (miss.missedTokens < 20_000 && miss.missedCost < 0.1) return;
|
||||
|
||||
const cost = miss.missedCost >= 0.01 ? ` (~$${miss.missedCost.toFixed(2)})` : "";
|
||||
const reBilled = `${formatTokens(miss.missedTokens)} tokens re-billed${cost}`;
|
||||
let label = "Cache miss";
|
||||
if (miss.modelChanged) {
|
||||
label = "Cache miss after model switch";
|
||||
} else if (miss.idleMs >= CACHE_TTL_MS) {
|
||||
label = `Cache miss after ${Math.round(miss.idleMs / 60_000)}m idle`;
|
||||
}
|
||||
const text = theme.fg("warning", `${label}: ${reBilled}`);
|
||||
this.chatContainer.addChild(new Spacer(1));
|
||||
this.chatContainer.addChild(new Text(text, 1, 0));
|
||||
}
|
||||
|
||||
renderInitialMessages(): void {
|
||||
const entries = this.sessionManager.buildContextEntries();
|
||||
this.renderSessionEntries(entries, {
|
||||
|
|
@ -4088,6 +4134,7 @@ export class InteractiveMode {
|
|||
doubleEscapeAction: this.settingsManager.getDoubleEscapeAction(),
|
||||
treeFilterMode: this.settingsManager.getTreeFilterMode(),
|
||||
showHardwareCursor: this.settingsManager.getShowHardwareCursor(),
|
||||
showCacheMissNotices: this.settingsManager.getShowCacheMissNotices(),
|
||||
defaultProjectTrust: this.settingsManager.getDefaultProjectTrust(),
|
||||
editorPaddingX: this.settingsManager.getEditorPaddingX(),
|
||||
outputPad: this.settingsManager.getOutputPad(),
|
||||
|
|
@ -4164,6 +4211,10 @@ export class InteractiveMode {
|
|||
this.chatContainer.clear();
|
||||
this.rebuildChatFromMessages();
|
||||
},
|
||||
onShowCacheMissNoticesChange: (shown) => {
|
||||
this.settingsManager.setShowCacheMissNotices(shown);
|
||||
this.rebuildChatFromMessages();
|
||||
},
|
||||
onCollapseChangelogChange: (collapsed) => {
|
||||
this.settingsManager.setCollapseChangelog(collapsed);
|
||||
},
|
||||
|
|
@ -5561,6 +5612,26 @@ export class InteractiveMode {
|
|||
private handleSessionCommand(): void {
|
||||
const stats = this.session.getSessionStats();
|
||||
const sessionName = this.sessionManager.getSessionName();
|
||||
const entries = this.sessionManager.getEntries();
|
||||
const cacheWaste = computeCacheWaste(entries, this.session.modelRegistry);
|
||||
|
||||
// Cost/token totals per provider/model actually used (e.g. OpenRouter `auto`
|
||||
// resolves to a concrete responseModel), sorted by cost descending.
|
||||
const perModelMap = new Map<string, { key: string; cost: number; tokens: number }>();
|
||||
for (const entry of entries) {
|
||||
if (entry.type !== "message" || entry.message.role !== "assistant") continue;
|
||||
const message = entry.message;
|
||||
const usage = message.usage;
|
||||
const key = `${message.provider}/${message.responseModel ?? message.model}`;
|
||||
let bucket = perModelMap.get(key);
|
||||
if (!bucket) {
|
||||
bucket = { key, cost: 0, tokens: 0 };
|
||||
perModelMap.set(key, bucket);
|
||||
}
|
||||
bucket.cost += usage.cost.total;
|
||||
bucket.tokens += usage.input + usage.output + usage.cacheRead + usage.cacheWrite;
|
||||
}
|
||||
const perModel = Array.from(perModelMap.values()).sort((a, b) => b.cost - a.cost);
|
||||
|
||||
let info = `${theme.bold("Session Info")}\n\n`;
|
||||
if (sessionName) {
|
||||
|
|
@ -5569,25 +5640,44 @@ export class InteractiveMode {
|
|||
info += `${theme.fg("dim", "File:")} ${stats.sessionFile ?? "In-memory"}\n`;
|
||||
info += `${theme.fg("dim", "ID:")} ${stats.sessionId}\n\n`;
|
||||
info += `${theme.bold("Messages")}\n`;
|
||||
info += `${theme.fg("dim", "Total:")} ${stats.totalMessages}\n`;
|
||||
info += `${theme.fg("dim", "User:")} ${stats.userMessages}\n`;
|
||||
info += `${theme.fg("dim", "Assistant:")} ${stats.assistantMessages}\n`;
|
||||
info += `${theme.fg("dim", "Tool Calls:")} ${stats.toolCalls}\n`;
|
||||
info += `${theme.fg("dim", "Tool Results:")} ${stats.toolResults}\n`;
|
||||
info += `${theme.fg("dim", "Total:")} ${stats.totalMessages}\n\n`;
|
||||
info += `${theme.fg("dim", "Tools:")} ${stats.toolCalls} calls, ${stats.toolResults} results\n\n`;
|
||||
info += `${theme.bold("Tokens")}\n`;
|
||||
info += `${theme.fg("dim", "Input:")} ${stats.tokens.input.toLocaleString()}\n`;
|
||||
// "Input" is the full prompt volume. With cache activity, split it into
|
||||
// cached (served from cache) vs uncached (everything else) - the only
|
||||
// provider-independent split. Cache writes, where reported, are a detail
|
||||
// of the uncached portion.
|
||||
const { input, cacheRead, cacheWrite } = stats.tokens;
|
||||
const promptTokens = input + cacheRead + cacheWrite;
|
||||
info += `${theme.fg("dim", "Input:")} ${promptTokens.toLocaleString()}\n`;
|
||||
if (promptTokens > 0 && (cacheRead > 0 || cacheWrite > 0)) {
|
||||
const hitRate = theme.fg("dim", `(${((cacheRead / promptTokens) * 100).toFixed(1)}%)`);
|
||||
info += ` ${theme.fg("dim", "Cached:")} ${cacheRead.toLocaleString()} ${hitRate}\n`;
|
||||
const written =
|
||||
cacheWrite > 0 ? ` ${theme.fg("dim", `(${cacheWrite.toLocaleString()} written to cache)`)}` : "";
|
||||
info += ` ${theme.fg("dim", "Uncached:")} ${(input + cacheWrite).toLocaleString()}${written}\n`;
|
||||
}
|
||||
info += `${theme.fg("dim", "Output:")} ${stats.tokens.output.toLocaleString()}\n`;
|
||||
if (stats.tokens.cacheRead > 0) {
|
||||
info += `${theme.fg("dim", "Cache Read:")} ${stats.tokens.cacheRead.toLocaleString()}\n`;
|
||||
}
|
||||
if (stats.tokens.cacheWrite > 0) {
|
||||
info += `${theme.fg("dim", "Cache Write:")} ${stats.tokens.cacheWrite.toLocaleString()}\n`;
|
||||
}
|
||||
info += `${theme.fg("dim", "Total:")} ${stats.tokens.total.toLocaleString()}\n`;
|
||||
|
||||
if (stats.cost > 0) {
|
||||
if (stats.cost > 0 || cacheWaste.missedTokens > 0) {
|
||||
info += `\n${theme.bold("Cost")}\n`;
|
||||
info += `${theme.fg("dim", "Total:")} ${stats.cost.toFixed(4)}`;
|
||||
info += `${theme.fg("dim", "Total:")} $${stats.cost.toFixed(3)}`;
|
||||
if (perModel.length > 1) {
|
||||
for (const entry of perModel) {
|
||||
info += `\n ${theme.fg("dim", `${entry.key}:`)} $${entry.cost.toFixed(3)} ${theme.fg("dim", `(${formatTokens(entry.tokens)} tokens)`)}`;
|
||||
}
|
||||
}
|
||||
if (cacheWaste.missedTokens > 0) {
|
||||
const missLabel = cacheWaste.missCount === 1 ? "1 miss" : `${cacheWaste.missCount} misses`;
|
||||
const detail = `${cacheWaste.missedTokens.toLocaleString()} tokens, ${missLabel}`;
|
||||
info +=
|
||||
cacheWaste.missedCost >= 0.0001
|
||||
? `\n${theme.fg("dim", "Cache Re-billed:")} $${cacheWaste.missedCost.toFixed(3)} ${theme.fg("dim", `(${detail})`)}`
|
||||
: `\n${theme.fg("dim", "Cache Re-billed:")} ${detail}`;
|
||||
}
|
||||
}
|
||||
|
||||
this.chatContainer.addChild(new Spacer(1));
|
||||
|
|
|
|||
|
|
@ -109,7 +109,8 @@ describe("AgentSession.getSessionStats", () => {
|
|||
syncAgentMessages(session, sessionManager);
|
||||
|
||||
const stats = session.getSessionStats();
|
||||
expect(stats.tokens.input).toBe(195_000);
|
||||
// Totals cover ALL entries, including history compacted away (180k + 195k).
|
||||
expect(stats.tokens.input).toBe(375_000);
|
||||
expect(stats.contextUsage).toBeDefined();
|
||||
expect(stats.contextUsage?.tokens).toBeNull();
|
||||
expect(stats.contextUsage?.percent).toBeNull();
|
||||
|
|
@ -132,7 +133,8 @@ describe("AgentSession.getSessionStats", () => {
|
|||
syncAgentMessages(session, sessionManager);
|
||||
|
||||
const stats = session.getSessionStats();
|
||||
expect(stats.tokens.input).toBe(220_000);
|
||||
// Totals cover ALL entries, including history compacted away (180k + 195k + 25k).
|
||||
expect(stats.tokens.input).toBe(400_000);
|
||||
expect(stats.contextUsage).toBeDefined();
|
||||
expect(stats.contextUsage?.tokens).toBe(25_000);
|
||||
expect(stats.contextUsage?.percent).toBe((25_000 / model.contextWindow) * 100);
|
||||
|
|
|
|||
143
packages/coding-agent/test/cache-stats.test.ts
Normal file
143
packages/coding-agent/test/cache-stats.test.ts
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
import type { AssistantMessage } from "@earendil-works/pi-ai";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
collectCacheMisses,
|
||||
computeCacheWaste,
|
||||
detectCacheMiss,
|
||||
type ModelPriceSource,
|
||||
} from "../src/core/cache-stats.ts";
|
||||
import type { SessionEntry } from "../src/core/session-manager.ts";
|
||||
|
||||
const zeroCost = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 };
|
||||
|
||||
const models: ModelPriceSource = {
|
||||
// $/million tokens; used as cache-read price fallback on full-miss turns
|
||||
find: () => ({ cost: { cacheRead: 0.3 } }),
|
||||
};
|
||||
|
||||
function assistant(options: {
|
||||
input?: number;
|
||||
cacheRead?: number;
|
||||
cacheWrite?: number;
|
||||
cost?: Partial<typeof zeroCost>;
|
||||
model?: string;
|
||||
timestamp?: number;
|
||||
}): AssistantMessage {
|
||||
return {
|
||||
role: "assistant",
|
||||
content: [],
|
||||
api: "anthropic-messages",
|
||||
provider: "test",
|
||||
model: options.model ?? "test-model",
|
||||
usage: {
|
||||
input: options.input ?? 0,
|
||||
output: 10,
|
||||
cacheRead: options.cacheRead ?? 0,
|
||||
cacheWrite: options.cacheWrite ?? 0,
|
||||
totalTokens: 0,
|
||||
cost: { ...zeroCost, ...options.cost },
|
||||
},
|
||||
stopReason: "stop",
|
||||
timestamp: options.timestamp ?? 0,
|
||||
} as AssistantMessage;
|
||||
}
|
||||
|
||||
function entry(message: AssistantMessage): SessionEntry {
|
||||
return { type: "message", id: "x", parentId: null, timestamp: "", message } as SessionEntry;
|
||||
}
|
||||
|
||||
// Turn 1: fresh 100k cache write at $3.75/M
|
||||
const turn1 = assistant({ cacheWrite: 100_000, cost: { cacheWrite: 0.375 }, timestamp: 0 });
|
||||
// Turn 2: healthy, everything read back at $0.30/M
|
||||
const turn2 = assistant({
|
||||
cacheRead: 100_000,
|
||||
cacheWrite: 5_000,
|
||||
cost: { cacheRead: 0.03, cacheWrite: 0.019 },
|
||||
timestamp: 60_000,
|
||||
});
|
||||
|
||||
describe("computeCacheWaste", () => {
|
||||
it("accumulates missed tokens and cost across turns", () => {
|
||||
// Turn 3: full miss, previous 105k prompt re-billed at $3.75/M write
|
||||
const turn3 = assistant({ cacheWrite: 110_000, cost: { cacheWrite: 0.4125 }, timestamp: 120_000 });
|
||||
const totals = computeCacheWaste([entry(turn1), entry(turn2), entry(turn3)], models);
|
||||
expect(totals.missedTokens).toBe(105_000);
|
||||
// 105k at ($3.75 - $0.30)/M
|
||||
expect(totals.missedCost).toBeCloseTo(0.36225, 5);
|
||||
});
|
||||
|
||||
it("counts nothing for healthy sessions", () => {
|
||||
const totals = computeCacheWaste([entry(turn1), entry(turn2)], models);
|
||||
expect(totals.missedTokens).toBe(0);
|
||||
expect(totals.missedCost).toBe(0);
|
||||
});
|
||||
|
||||
it("skips the turn after a compaction reset", () => {
|
||||
const reset = { type: "compaction", id: "c", parentId: null, timestamp: "" } as SessionEntry;
|
||||
const afterReset = assistant({ cacheWrite: 20_000, cost: { cacheWrite: 0.075 } });
|
||||
const totals = computeCacheWaste([entry(turn1), reset, entry(afterReset)], models);
|
||||
expect(totals.missedTokens).toBe(0);
|
||||
});
|
||||
|
||||
it("counts misses caused by model switches", () => {
|
||||
const otherModel = assistant({ cacheWrite: 100_000, cost: { cacheWrite: 0.375 }, model: "other-model" });
|
||||
const totals = computeCacheWaste([entry(turn1), entry(otherModel)], models);
|
||||
expect(totals.missedTokens).toBe(100_000);
|
||||
expect(totals.missCount).toBe(1);
|
||||
});
|
||||
|
||||
it("skips providers that report no cache activity", () => {
|
||||
const a = assistant({ input: 100_000 });
|
||||
const b = assistant({ input: 110_000 });
|
||||
const totals = computeCacheWaste([entry(a), entry(b)], models);
|
||||
expect(totals.missedTokens).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("collectCacheMisses", () => {
|
||||
it("maps counted misses to their assistant messages by reference", () => {
|
||||
const missTurn = assistant({ cacheWrite: 110_000, cost: { cacheWrite: 0.4125 }, timestamp: 120_000 });
|
||||
const misses = collectCacheMisses([entry(turn1), entry(turn2), entry(missTurn)], models);
|
||||
expect(misses.size).toBe(1);
|
||||
expect(misses.get(missTurn)?.missedTokens).toBe(105_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectCacheMiss", () => {
|
||||
it("detects a miss on a just-completed message with idle time", () => {
|
||||
const missMessage = assistant({ cacheWrite: 110_000, cost: { cacheWrite: 0.4125 }, timestamp: 600_000 });
|
||||
const miss = detectCacheMiss([entry(turn1), entry(turn2)], missMessage, models);
|
||||
expect(miss).toBeDefined();
|
||||
expect(miss?.missedTokens).toBe(105_000);
|
||||
expect(miss?.missedCost).toBeCloseTo(0.36225, 5);
|
||||
// 600s - 60s since the previous request
|
||||
expect(miss?.idleMs).toBe(540_000);
|
||||
expect(miss?.modelChanged).toBe(false);
|
||||
});
|
||||
|
||||
it("flags model switches on detected misses", () => {
|
||||
const otherModel = assistant({
|
||||
cacheWrite: 110_000,
|
||||
cost: { cacheWrite: 0.4125 },
|
||||
model: "other-model",
|
||||
timestamp: 120_000,
|
||||
});
|
||||
const miss = detectCacheMiss([entry(turn1), entry(turn2)], otherModel, models);
|
||||
expect(miss?.missedTokens).toBe(105_000);
|
||||
expect(miss?.modelChanged).toBe(true);
|
||||
});
|
||||
|
||||
it("returns undefined for healthy turns", () => {
|
||||
const healthy = assistant({
|
||||
cacheRead: 105_000,
|
||||
cacheWrite: 2_000,
|
||||
cost: { cacheRead: 0.0315, cacheWrite: 0.0075 },
|
||||
timestamp: 120_000,
|
||||
});
|
||||
expect(detectCacheMiss([entry(turn1), entry(turn2)], healthy, models)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns undefined for the first turn of a session", () => {
|
||||
expect(detectCacheMiss([], turn1, models)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
|
@ -42,8 +42,8 @@ type RenderSessionContextThis = {
|
|||
getShowImages(): boolean;
|
||||
getImageWidthCells(): number;
|
||||
};
|
||||
sessionManager: { getCwd(): string };
|
||||
session: { retryAttempt: number };
|
||||
sessionManager: { getCwd(): string; getEntries(): SessionEntry[] };
|
||||
session: { retryAttempt: number; modelRegistry: { find(provider: string, modelId: string): undefined } };
|
||||
toolOutputExpanded: boolean;
|
||||
isInitialized: boolean;
|
||||
updateEditorBorderColor(): void;
|
||||
|
|
@ -71,8 +71,8 @@ function createFakeInteractiveModeThis(): RenderSessionContextThis {
|
|||
getShowImages: () => false,
|
||||
getImageWidthCells: () => 60,
|
||||
},
|
||||
sessionManager: { getCwd: () => process.cwd() },
|
||||
session: { retryAttempt: 0 },
|
||||
sessionManager: { getCwd: () => process.cwd(), getEntries: () => [] },
|
||||
session: { retryAttempt: 0, modelRegistry: { find: () => undefined } },
|
||||
toolOutputExpanded: false,
|
||||
isInitialized: true,
|
||||
updateEditorBorderColor: vi.fn(),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue