From 26bfc4e4c4d39e127d34d512b7824eef515ee58d Mon Sep 17 00:00:00 2001 From: Resham Joshi <65915470+iamtoruk@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:48:37 +0100 Subject: [PATCH] feat(optimize): report fast-mode spend on background agents and long-context premiums (#1858) * fix(optimize): price findings from the calls they point at Every finding's dollar figure was its token estimate times one blended rate: 70% of the period's spend over all input-side tokens. That rate is close to a cache-read price, so it misprices anything that is not a cache read. Schema findings already weight cache reads at 0.1, so their reads were discounted twice; findings whose tokens include output were priced as input. A finding can now carry savingsUSD, computed from the calls it names. Every surface reads it through findingUSD: the CLI and TUI optimize views, the JSON report (and so the desktop Optimize page), the menubar payload and MCP get_savings, and the --apply list. classTotals sums it, scaled to the apply-able share for fix findings. Findings without it keep tokensSaved times the blended rate, and any token whose session or model has no price still falls back to that rate. Measured now: - low-worth-sessions: each session's real cost times its recovery fraction - retry-heavy-capabilities: the retry turns' real cost times the fraction - redundant-rereads, build-folder-reads: the reads at the input rate of their session's most-called model - warmup-heavy: the excess cache writes at the session model's write rate - mcp-low-coverage, mcp-project-scope: each call's charged schema tokens at that call model's cache-write or cache-read rate - unused-mcp, mcp-deferral-off, mcp-alwaysload-hygiene, mcp-defer-threshold: the per-session schema at the input rate of the sessions charged Still on the blended rate, because no calls can be named: read-edit-ratio (reads that never happened), context-heavy-sessions, cost-outliers, low-cache-hit-sessions, claude-md-too-long, bash-output-cap, recurring-context and the unused agents, skills and commands. * feat(optimize): report fast-mode spend on background agents and long-context premiums Two findings for premium-tier spend the parser already prices, so their dollars come from the calls themselves. fast-mode-background (nudge) sums, over fast-mode calls inside subagent sessions, the call's cost minus the same tokens at standard speed. It fires at $1 or more in the window. Some background runs need the speed, so it stays a habit check. long-context-premium (keep) sums, over calls whose prompt crossed the model's long-context tier, the tiered cost minus the same tokens at base rates. Fast calls are priced at standard on both sides so the fast premium is never counted twice. tokensSaved is the prompt tokens above the threshold; the fix names each model's threshold from its tier. It fires at $1 or more and has no apply action. Both rate high above $20, medium above $5, else low. --- src/optimize.ts | 129 ++++++++++++++++++++++++++++++++- tests/optimize-apply.test.ts | 2 + tests/optimize-premium.test.ts | 124 +++++++++++++++++++++++++++++++ 3 files changed, 253 insertions(+), 2 deletions(-) create mode 100644 tests/optimize-premium.test.ts diff --git a/src/optimize.ts b/src/optimize.ts index e4d66563..2255a2c8 100644 --- a/src/optimize.ts +++ b/src/optimize.ts @@ -11,8 +11,8 @@ import { EDIT_TOOLS } from './classifier.js' import { readSessionLines, readSessionFileSync } from './fs-utils.js' import { discoverAllSessions, providerDisplayName } from './providers/index.js' import { parseJsonlLine, shouldSkipLine } from './parser.js' -import type { DateRange, ProjectSummary, SessionSummary } from './types.js' -import { cacheWriteCostPerToken, getModelCosts, type ModelCosts } from './models.js' +import type { DateRange, ParsedApiCall, ProjectSummary, SessionSummary } from './types.js' +import { billableOutputTokens, cacheWriteCostPerToken, calculateCost, getModelCosts, getShortModelName, pricingModelAt, type ModelCosts } from './models.js' import { formatCost } from './currency.js' import { formatTokens } from './format.js' import { recommendModelDefault, type ModelDefaultRecommendation } from './act/model-defaults.js' @@ -118,6 +118,9 @@ const LOW_CACHE_HIT_MIN_INPUT_TOKENS = CONTEXT_BLOAT_MIN_INPUT_TOKENS const LOW_CACHE_HIT_MIN_CALLS = 5 const LOW_CACHE_HIT_PREVIEW = 5 const WORTH_IT_MIN_COST_USD = 2 +const PREMIUM_MIN_USD = 1 +const PREMIUM_HIGH_USD = 20 +const PREMIUM_MEDIUM_USD = 5 const WORTH_IT_NO_EDIT_MIN_COST_USD = 3 // A zero-edit session's full cost is an upper bound on recoverable waste, not // a point estimate, so use a bounded fraction for honest savings numbers. @@ -373,6 +376,8 @@ export type FindingId = | 'unused-skills' | 'unused-commands' | 'recurring-context' + | 'fast-mode-background' + | 'long-context-premium' /// How a finding is meant to be acted on: /// - `fix` CodeBurn can write the change itself (`codeburn optimize --apply`) @@ -412,6 +417,8 @@ export const FINDING_CLASS: Record = { 'unused-skills': 'fix', 'unused-commands': 'fix', 'recurring-context': 'nudge', + 'fast-mode-background': 'nudge', // some runs pay for speed on purpose + 'long-context-premium': 'keep', // the context may be needed } /// Ids whose plan is built from the `apply` payload: without it the plan @@ -456,6 +463,8 @@ export const FINDING_BASIS: Record = { // system prompt, tool schemas and CLAUDE.md in with the pasted block, so // nothing counted isolates the block. Its size is modelled from its bytes. 'recurring-context': 'estimated', // block chars x BASH_TOKENS_PER_CHAR x repeats + 'fast-mode-background': 'measured', // priced calls, fast minus standard + 'long-context-premium': 'measured', // priced calls, tier minus base rates } /// Scope label for a setting that lives in ~/.zshrc / ~/.bashrc. The MCP @@ -3985,6 +3994,120 @@ export function detectLowCacheHitSessions(projects: ProjectSummary[]): WasteFind } } +/// The call's tokens priced again at `speed`; `provider` undefined drops the +/// long-context tier, as `tieredCostsFor` applies it only to tiered providers. +function repricedCallUSD(call: ParsedApiCall, speed: ParsedApiCall['speed'], provider: string | undefined): number { + const u = call.usage + return calculateCost( + pricingModelAt(call.model, call.timestamp), u.inputTokens, + billableOutputTokens(call.provider, u.outputTokens, u.reasoningTokens), + u.cacheCreationInputTokens, u.cacheReadInputTokens, u.webSearchRequests, + speed, call.cacheCreationOneHourTokens ?? 0, provider, + ) +} + +function premiumImpact(usd: number): Impact { + return usd > PREMIUM_HIGH_USD ? 'high' : usd > PREMIUM_MEDIUM_USD ? 'medium' : 'low' +} + +function plural(n: number, word: string): string { + return `${n} ${word}${n === 1 ? '' : 's'}` +} + +export function detectFastModeBackground(projects: ProjectSummary[]): WasteFinding | null { + let calls = 0 + let outputTokens = 0 + let premiumUSD = 0 + const models = new Set() + for (const project of projects) { + for (const session of project.sessions) { + if (session.isSidechain !== true) continue + for (const turn of session.turns) { + for (const call of turn.assistantCalls) { + if (call.speed !== 'fast' || call.supplementaryAccounting) continue + const premium = call.costUSD - repricedCallUSD(call, 'standard', call.provider) + if (premium <= 0) continue + calls++ + outputTokens += call.usage.outputTokens + premiumUSD += premium + models.add(getShortModelName(call.model)) + } + } + } + } + if (premiumUSD < PREMIUM_MIN_USD) return null + + return { + id: 'fast-mode-background', + title: `Fast mode cost ${formatCost(premiumUSD)} extra on background agents`, + explanation: `${plural(calls, 'fast-mode call')} ran inside subagent sessions on ${[...models].sort().join(', ')}, where nobody waits for the reply. At standard speed the same tokens would have cost ${formatCost(premiumUSD)} less. Some background runs need the speed, so this is a habit check, not a rule.`, + impact: premiumImpact(premiumUSD), + tokensSaved: outputTokens, + savingsUSD: premiumUSD, + fix: { + type: 'paste', + destination: 'manual', + label: 'Keep fast mode for interactive work:', + text: 'Use fast mode only for interactive work. Turn it off before long subagent or background runs, where nobody waits for the speed.', + }, + } +} + +export function detectLongContextPremium(projects: ProjectSummary[]): WasteFinding | null { + let totalCalls = 0 + let tokensAbove = 0 + let premiumUSD = 0 + const sessions = new Set() + const byModel = new Map() + for (const project of projects) { + for (const session of project.sessions) { + for (const turn of session.turns) { + for (const call of turn.assistantCalls) { + if (call.supplementaryAccounting) continue + totalCalls++ + const threshold = getModelCosts(pricingModelAt(call.model, call.timestamp))?.longContextTier?.thresholdTokens + if (threshold === undefined) continue + // Priced at the call's speed minus fast, so fast-mode-background owns the fast premium. + const speed = call.speed === 'fast' ? 'standard' : call.speed + const premium = repricedCallUSD(call, speed, call.provider) - repricedCallUSD(call, speed, undefined) + if (premium <= 0) continue + const u = call.usage + const prompt = u.inputTokens + u.cacheReadInputTokens + Math.max(u.cacheCreationInputTokens, call.cacheCreationOneHourTokens ?? 0) + tokensAbove += Math.max(0, prompt - threshold) + premiumUSD += premium + sessions.add(`${project.projectPath}\0${session.sessionId}`) + const name = getShortModelName(call.model) + const entry = byModel.get(name) ?? { threshold, calls: 0 } + entry.calls++ + byModel.set(name, entry) + } + } + } + } + if (premiumUSD < PREMIUM_MIN_USD) return null + + const crossed = [...byModel.values()].reduce((sum, m) => sum + m.calls, 0) + const models = [...byModel.entries()].sort((a, b) => b[1].calls - a[1].calls || a[0].localeCompare(b[0])) + const list = models.map(([name, m]) => `${name} above ${formatTokens(m.threshold)}: ${plural(m.calls, 'call')}`).join('; ') + const share = ((crossed / totalCalls) * 100).toFixed(1) + const limits = models.map(([name, m]) => `${formatTokens(m.threshold)} on ${name}`).join(', ') + + return { + id: 'long-context-premium', + title: `${plural(crossed, 'call')} crossed the long-context price line, ${formatCost(premiumUSD)} extra`, + explanation: `${crossed} of ${totalCalls} calls (${share}%) in ${plural(sessions.size, 'session')} sent more prompt tokens than the model's long-context threshold (${list}), so each of those requests billed at the higher tier. The context may have been needed; this is what it cost.`, + impact: premiumImpact(premiumUSD), + tokensSaved: tokensAbove, + savingsUSD: premiumUSD, + fix: { + type: 'paste', + destination: 'manual', + label: 'Stay under the long-context price line:', + text: `Compact or start a fresh session before the prompt passes ${limits}.`, + }, + } +} + function findYoungProjectFirstSessionIds(projects: ProjectSummary[]): Set { const firstSessionIds = new Set() @@ -4247,6 +4370,8 @@ export async function scanAndDetect( () => detectContextBloat(behavioralProjects, lowWorthSessionIds, provider), () => detectSessionOutliers(behavioralProjects, outlierExclusions, provider), () => detectLowCacheHitSessions(behavioralProjects), + () => detectFastModeBackground(projects), + () => detectLongContextPremium(projects), claudeOnly(() => detectBloatedClaudeMd(projectCwds)), claudeOnly(() => detectBashBloat()), claudeOnly(() => detectRecurringContext(openers)), diff --git a/tests/optimize-apply.test.ts b/tests/optimize-apply.test.ts index 701705fa..b3a9c6af 100644 --- a/tests/optimize-apply.test.ts +++ b/tests/optimize-apply.test.ts @@ -963,6 +963,8 @@ describe('finding class', () => { 'unused-skills': makeFinding('unused-skills', CMD_FIX, { kind: 'archive', names: ['ghost'] }), 'unused-commands': makeFinding('unused-commands', CMD_FIX, { kind: 'archive', names: ['ghost'] }), 'recurring-context': makeFinding('recurring-context', PROMPT_FIX), + 'fast-mode-background': makeFinding('fast-mode-background', PROMPT_FIX), + 'long-context-premium': makeFinding('long-context-premium', PROMPT_FIX), } const planCtx: PlanContext = { homeDir: fx.home, cwd: fx.project, shell: '/bin/zsh', claudeVersion: () => '2.1.130' } diff --git a/tests/optimize-premium.test.ts b/tests/optimize-premium.test.ts new file mode 100644 index 00000000..05adc510 --- /dev/null +++ b/tests/optimize-premium.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it } from 'vitest' + +import { detectFastModeBackground, detectLongContextPremium, findingClass } from '../src/optimize.js' +import { calculateCost } from '../src/models.js' +import type { ParsedApiCall, ProjectSummary, SessionSummary } from '../src/types.js' + +type Tokens = { input?: number; output?: number; cacheRead?: number } + +function apiCall(model: string, speed: ParsedApiCall['speed'], t: Tokens, i: number): ParsedApiCall { + const input = t.input ?? 0 + const output = t.output ?? 0 + const cacheRead = t.cacheRead ?? 0 + return { + provider: 'claude', + model, + usage: { + inputTokens: input, + outputTokens: output, + cacheCreationInputTokens: 0, + cacheReadInputTokens: cacheRead, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + }, + costUSD: calculateCost(model, input, output, 0, cacheRead, 0, speed, 0, 'claude'), + tools: [], + mcpTools: [], + skills: [], + subagentTypes: [], + hasAgentSpawn: false, + hasPlanMode: false, + speed, + timestamp: `2026-10-01T10:${String(i).padStart(2, '0')}:00Z`, + bashCommands: [], + deduplicationKey: `call-${i}`, + } +} + +function session(id: string, calls: ParsedApiCall[], isSidechain = false): SessionSummary { + return { + sessionId: id, + project: 'app', + isSidechain, + firstTimestamp: '2026-10-01T10:00:00Z', + lastTimestamp: '2026-10-01T11:00:00Z', + totalCostUSD: calls.reduce((s, c) => s + c.costUSD, 0), + apiCalls: calls.length, + turns: calls.map(c => ({ userMessage: 'go', assistantCalls: [c], timestamp: c.timestamp, sessionId: id, category: 'coding', retries: 0, hasEdits: false })), + } as unknown as SessionSummary +} + +function projects(...sessions: SessionSummary[]): ProjectSummary[] { + return [{ project: 'app', projectPath: '/tmp/app', sessions } as unknown as ProjectSummary] +} + +describe('detectFastModeBackground', () => { + it('reports the fast premium of subagent calls', () => { + // opus-4-8 fast is 2x: 100k output x $25/M = $2.50 extra. + const finding = detectFastModeBackground(projects(session('agent-1', [apiCall('claude-opus-4-8', 'fast', { output: 100_000 }, 0)], true))) + expect(finding).not.toBeNull() + expect(finding!.id).toBe('fast-mode-background') + expect(findingClass(finding!)).toBe('nudge') + expect(finding!.savingsUSD).toBeCloseTo(2.5, 10) + expect(finding!.tokensSaved).toBe(100_000) + expect(finding!.impact).toBe('low') + expect(finding!.title).toContain('$2.50') + expect(finding!.explanation).toContain('1 fast-mode call ') + expect(finding!.explanation).toContain('Opus 4.8') + }) + + it('rates a premium above $20 high', () => { + const calls = Array.from({ length: 10 }, (_, i) => apiCall('claude-opus-4-8', 'fast', { output: 100_000 }, i)) + expect(detectFastModeBackground(projects(session('agent-1', calls, true)))!.impact).toBe('high') + }) + + it('ignores main-thread fast calls and premiums under $1', () => { + expect(detectFastModeBackground(projects(session('main', [apiCall('claude-opus-4-8', 'fast', { output: 100_000 }, 0)])))).toBeNull() + expect(detectFastModeBackground(projects(session('agent-1', [apiCall('claude-opus-4-8', 'fast', { output: 800 }, 0)], true)))).toBeNull() + }) + + it('ignores a model without a fast multiplier', () => { + expect(detectFastModeBackground(projects(session('agent-1', [apiCall('claude-opus-4-6', 'fast', { output: 100_000 }, 0)], true)))).toBeNull() + }) +}) + +describe('detectLongContextPremium', () => { + // sonnet-4-5 above 200k: 10k input + 290k cache read + 1k output costs + // $0.06 + $0.174 + $0.0225 = $0.2565 against $0.03 + $0.087 + $0.015 = $0.132. + const over = (i: number) => apiCall('claude-sonnet-4-5', 'standard', { input: 10_000, cacheRead: 290_000, output: 1_000 }, i) + + it('reports calls priced above the tier', () => { + const finding = detectLongContextPremium(projects( + session('s1', Array.from({ length: 10 }, (_, i) => over(i))), + session('s2', [apiCall('claude-sonnet-4-5', 'standard', { input: 150_000, output: 1_000 }, 20)]), + )) + expect(finding).not.toBeNull() + expect(finding!.id).toBe('long-context-premium') + expect(findingClass(finding!)).toBe('keep') + expect(finding!.apply).toBeUndefined() + expect(finding!.savingsUSD).toBeCloseTo(10 * 0.1245, 10) + expect(finding!.tokensSaved).toBe(10 * 100_000) + expect(finding!.impact).toBe('low') + expect(finding!.explanation).toContain('10 of 11 calls (90.9%) in 1 session ') + expect(finding!.explanation).toContain('Sonnet 4.5 above 200.0K: 10 calls') + expect(finding!.fix.type === 'paste' && finding!.fix.text).toContain('200.0K on Sonnet 4.5') + }) + + it('prices only the tier on a fast call, leaving the fast premium to its own finding', () => { + // gpt-5.5 above 272k at standard: $0.10 + $0.29 + $0.045 = $0.435 against $0.05 + $0.145 + $0.03 = $0.225. + const fast = Array.from({ length: 10 }, (_, i) => apiCall('gpt-5.5', 'fast', { input: 10_000, cacheRead: 290_000, output: 1_000 }, i)) + expect(detectLongContextPremium(projects(session('s1', fast)))!.savingsUSD).toBeCloseTo(10 * 0.21, 10) + }) + + it('ignores calls under the threshold and premiums under $1', () => { + expect(detectLongContextPremium(projects(session('s1', Array.from({ length: 10 }, (_, i) => + apiCall('claude-sonnet-4-5', 'standard', { input: 10_000, cacheRead: 190_000, output: 1_000 }, i)))))).toBeNull() + expect(detectLongContextPremium(projects(session('s1', [over(0)])))).toBeNull() + }) + + it('ignores a model without a long-context tier', () => { + expect(detectLongContextPremium(projects(session('s1', Array.from({ length: 10 }, (_, i) => + apiCall('claude-sonnet-4-6', 'standard', { input: 10_000, cacheRead: 290_000, output: 1_000 }, i)))))).toBeNull() + }) +})