diff --git a/src/optimize.ts b/src/optimize.ts index e4d66563..2255a2c8 100644 --- a/src/optimize.ts +++ b/src/optimize.ts @@ -11,8 +11,8 @@ import { EDIT_TOOLS } from './classifier.js' import { readSessionLines, readSessionFileSync } from './fs-utils.js' import { discoverAllSessions, providerDisplayName } from './providers/index.js' import { parseJsonlLine, shouldSkipLine } from './parser.js' -import type { DateRange, ProjectSummary, SessionSummary } from './types.js' -import { cacheWriteCostPerToken, getModelCosts, type ModelCosts } from './models.js' +import type { DateRange, ParsedApiCall, ProjectSummary, SessionSummary } from './types.js' +import { billableOutputTokens, cacheWriteCostPerToken, calculateCost, getModelCosts, getShortModelName, pricingModelAt, type ModelCosts } from './models.js' import { formatCost } from './currency.js' import { formatTokens } from './format.js' import { recommendModelDefault, type ModelDefaultRecommendation } from './act/model-defaults.js' @@ -118,6 +118,9 @@ const LOW_CACHE_HIT_MIN_INPUT_TOKENS = CONTEXT_BLOAT_MIN_INPUT_TOKENS const LOW_CACHE_HIT_MIN_CALLS = 5 const LOW_CACHE_HIT_PREVIEW = 5 const WORTH_IT_MIN_COST_USD = 2 +const PREMIUM_MIN_USD = 1 +const PREMIUM_HIGH_USD = 20 +const PREMIUM_MEDIUM_USD = 5 const WORTH_IT_NO_EDIT_MIN_COST_USD = 3 // A zero-edit session's full cost is an upper bound on recoverable waste, not // a point estimate, so use a bounded fraction for honest savings numbers. @@ -373,6 +376,8 @@ export type FindingId = | 'unused-skills' | 'unused-commands' | 'recurring-context' + | 'fast-mode-background' + | 'long-context-premium' /// How a finding is meant to be acted on: /// - `fix` CodeBurn can write the change itself (`codeburn optimize --apply`) @@ -412,6 +417,8 @@ export const FINDING_CLASS: Record = { 'unused-skills': 'fix', 'unused-commands': 'fix', 'recurring-context': 'nudge', + 'fast-mode-background': 'nudge', // some runs pay for speed on purpose + 'long-context-premium': 'keep', // the context may be needed } /// Ids whose plan is built from the `apply` payload: without it the plan @@ -456,6 +463,8 @@ export const FINDING_BASIS: Record = { // system prompt, tool schemas and CLAUDE.md in with the pasted block, so // nothing counted isolates the block. Its size is modelled from its bytes. 'recurring-context': 'estimated', // block chars x BASH_TOKENS_PER_CHAR x repeats + 'fast-mode-background': 'measured', // priced calls, fast minus standard + 'long-context-premium': 'measured', // priced calls, tier minus base rates } /// Scope label for a setting that lives in ~/.zshrc / ~/.bashrc. The MCP @@ -3985,6 +3994,120 @@ export function detectLowCacheHitSessions(projects: ProjectSummary[]): WasteFind } } +/// The call's tokens priced again at `speed`; `provider` undefined drops the +/// long-context tier, as `tieredCostsFor` applies it only to tiered providers. +function repricedCallUSD(call: ParsedApiCall, speed: ParsedApiCall['speed'], provider: string | undefined): number { + const u = call.usage + return calculateCost( + pricingModelAt(call.model, call.timestamp), u.inputTokens, + billableOutputTokens(call.provider, u.outputTokens, u.reasoningTokens), + u.cacheCreationInputTokens, u.cacheReadInputTokens, u.webSearchRequests, + speed, call.cacheCreationOneHourTokens ?? 0, provider, + ) +} + +function premiumImpact(usd: number): Impact { + return usd > PREMIUM_HIGH_USD ? 'high' : usd > PREMIUM_MEDIUM_USD ? 'medium' : 'low' +} + +function plural(n: number, word: string): string { + return `${n} ${word}${n === 1 ? '' : 's'}` +} + +export function detectFastModeBackground(projects: ProjectSummary[]): WasteFinding | null { + let calls = 0 + let outputTokens = 0 + let premiumUSD = 0 + const models = new Set() + for (const project of projects) { + for (const session of project.sessions) { + if (session.isSidechain !== true) continue + for (const turn of session.turns) { + for (const call of turn.assistantCalls) { + if (call.speed !== 'fast' || call.supplementaryAccounting) continue + const premium = call.costUSD - repricedCallUSD(call, 'standard', call.provider) + if (premium <= 0) continue + calls++ + outputTokens += call.usage.outputTokens + premiumUSD += premium + models.add(getShortModelName(call.model)) + } + } + } + } + if (premiumUSD < PREMIUM_MIN_USD) return null + + return { + id: 'fast-mode-background', + title: `Fast mode cost ${formatCost(premiumUSD)} extra on background agents`, + explanation: `${plural(calls, 'fast-mode call')} ran inside subagent sessions on ${[...models].sort().join(', ')}, where nobody waits for the reply. At standard speed the same tokens would have cost ${formatCost(premiumUSD)} less. Some background runs need the speed, so this is a habit check, not a rule.`, + impact: premiumImpact(premiumUSD), + tokensSaved: outputTokens, + savingsUSD: premiumUSD, + fix: { + type: 'paste', + destination: 'manual', + label: 'Keep fast mode for interactive work:', + text: 'Use fast mode only for interactive work. Turn it off before long subagent or background runs, where nobody waits for the speed.', + }, + } +} + +export function detectLongContextPremium(projects: ProjectSummary[]): WasteFinding | null { + let totalCalls = 0 + let tokensAbove = 0 + let premiumUSD = 0 + const sessions = new Set() + const byModel = new Map() + for (const project of projects) { + for (const session of project.sessions) { + for (const turn of session.turns) { + for (const call of turn.assistantCalls) { + if (call.supplementaryAccounting) continue + totalCalls++ + const threshold = getModelCosts(pricingModelAt(call.model, call.timestamp))?.longContextTier?.thresholdTokens + if (threshold === undefined) continue + // Priced at the call's speed minus fast, so fast-mode-background owns the fast premium. + const speed = call.speed === 'fast' ? 'standard' : call.speed + const premium = repricedCallUSD(call, speed, call.provider) - repricedCallUSD(call, speed, undefined) + if (premium <= 0) continue + const u = call.usage + const prompt = u.inputTokens + u.cacheReadInputTokens + Math.max(u.cacheCreationInputTokens, call.cacheCreationOneHourTokens ?? 0) + tokensAbove += Math.max(0, prompt - threshold) + premiumUSD += premium + sessions.add(`${project.projectPath}\0${session.sessionId}`) + const name = getShortModelName(call.model) + const entry = byModel.get(name) ?? { threshold, calls: 0 } + entry.calls++ + byModel.set(name, entry) + } + } + } + } + if (premiumUSD < PREMIUM_MIN_USD) return null + + const crossed = [...byModel.values()].reduce((sum, m) => sum + m.calls, 0) + const models = [...byModel.entries()].sort((a, b) => b[1].calls - a[1].calls || a[0].localeCompare(b[0])) + const list = models.map(([name, m]) => `${name} above ${formatTokens(m.threshold)}: ${plural(m.calls, 'call')}`).join('; ') + const share = ((crossed / totalCalls) * 100).toFixed(1) + const limits = models.map(([name, m]) => `${formatTokens(m.threshold)} on ${name}`).join(', ') + + return { + id: 'long-context-premium', + title: `${plural(crossed, 'call')} crossed the long-context price line, ${formatCost(premiumUSD)} extra`, + explanation: `${crossed} of ${totalCalls} calls (${share}%) in ${plural(sessions.size, 'session')} sent more prompt tokens than the model's long-context threshold (${list}), so each of those requests billed at the higher tier. The context may have been needed; this is what it cost.`, + impact: premiumImpact(premiumUSD), + tokensSaved: tokensAbove, + savingsUSD: premiumUSD, + fix: { + type: 'paste', + destination: 'manual', + label: 'Stay under the long-context price line:', + text: `Compact or start a fresh session before the prompt passes ${limits}.`, + }, + } +} + function findYoungProjectFirstSessionIds(projects: ProjectSummary[]): Set { const firstSessionIds = new Set() @@ -4247,6 +4370,8 @@ export async function scanAndDetect( () => detectContextBloat(behavioralProjects, lowWorthSessionIds, provider), () => detectSessionOutliers(behavioralProjects, outlierExclusions, provider), () => detectLowCacheHitSessions(behavioralProjects), + () => detectFastModeBackground(projects), + () => detectLongContextPremium(projects), claudeOnly(() => detectBloatedClaudeMd(projectCwds)), claudeOnly(() => detectBashBloat()), claudeOnly(() => detectRecurringContext(openers)), diff --git a/tests/optimize-apply.test.ts b/tests/optimize-apply.test.ts index 701705fa..b3a9c6af 100644 --- a/tests/optimize-apply.test.ts +++ b/tests/optimize-apply.test.ts @@ -963,6 +963,8 @@ describe('finding class', () => { 'unused-skills': makeFinding('unused-skills', CMD_FIX, { kind: 'archive', names: ['ghost'] }), 'unused-commands': makeFinding('unused-commands', CMD_FIX, { kind: 'archive', names: ['ghost'] }), 'recurring-context': makeFinding('recurring-context', PROMPT_FIX), + 'fast-mode-background': makeFinding('fast-mode-background', PROMPT_FIX), + 'long-context-premium': makeFinding('long-context-premium', PROMPT_FIX), } const planCtx: PlanContext = { homeDir: fx.home, cwd: fx.project, shell: '/bin/zsh', claudeVersion: () => '2.1.130' } diff --git a/tests/optimize-premium.test.ts b/tests/optimize-premium.test.ts new file mode 100644 index 00000000..05adc510 --- /dev/null +++ b/tests/optimize-premium.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it } from 'vitest' + +import { detectFastModeBackground, detectLongContextPremium, findingClass } from '../src/optimize.js' +import { calculateCost } from '../src/models.js' +import type { ParsedApiCall, ProjectSummary, SessionSummary } from '../src/types.js' + +type Tokens = { input?: number; output?: number; cacheRead?: number } + +function apiCall(model: string, speed: ParsedApiCall['speed'], t: Tokens, i: number): ParsedApiCall { + const input = t.input ?? 0 + const output = t.output ?? 0 + const cacheRead = t.cacheRead ?? 0 + return { + provider: 'claude', + model, + usage: { + inputTokens: input, + outputTokens: output, + cacheCreationInputTokens: 0, + cacheReadInputTokens: cacheRead, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + }, + costUSD: calculateCost(model, input, output, 0, cacheRead, 0, speed, 0, 'claude'), + tools: [], + mcpTools: [], + skills: [], + subagentTypes: [], + hasAgentSpawn: false, + hasPlanMode: false, + speed, + timestamp: `2026-10-01T10:${String(i).padStart(2, '0')}:00Z`, + bashCommands: [], + deduplicationKey: `call-${i}`, + } +} + +function session(id: string, calls: ParsedApiCall[], isSidechain = false): SessionSummary { + return { + sessionId: id, + project: 'app', + isSidechain, + firstTimestamp: '2026-10-01T10:00:00Z', + lastTimestamp: '2026-10-01T11:00:00Z', + totalCostUSD: calls.reduce((s, c) => s + c.costUSD, 0), + apiCalls: calls.length, + turns: calls.map(c => ({ userMessage: 'go', assistantCalls: [c], timestamp: c.timestamp, sessionId: id, category: 'coding', retries: 0, hasEdits: false })), + } as unknown as SessionSummary +} + +function projects(...sessions: SessionSummary[]): ProjectSummary[] { + return [{ project: 'app', projectPath: '/tmp/app', sessions } as unknown as ProjectSummary] +} + +describe('detectFastModeBackground', () => { + it('reports the fast premium of subagent calls', () => { + // opus-4-8 fast is 2x: 100k output x $25/M = $2.50 extra. + const finding = detectFastModeBackground(projects(session('agent-1', [apiCall('claude-opus-4-8', 'fast', { output: 100_000 }, 0)], true))) + expect(finding).not.toBeNull() + expect(finding!.id).toBe('fast-mode-background') + expect(findingClass(finding!)).toBe('nudge') + expect(finding!.savingsUSD).toBeCloseTo(2.5, 10) + expect(finding!.tokensSaved).toBe(100_000) + expect(finding!.impact).toBe('low') + expect(finding!.title).toContain('$2.50') + expect(finding!.explanation).toContain('1 fast-mode call ') + expect(finding!.explanation).toContain('Opus 4.8') + }) + + it('rates a premium above $20 high', () => { + const calls = Array.from({ length: 10 }, (_, i) => apiCall('claude-opus-4-8', 'fast', { output: 100_000 }, i)) + expect(detectFastModeBackground(projects(session('agent-1', calls, true)))!.impact).toBe('high') + }) + + it('ignores main-thread fast calls and premiums under $1', () => { + expect(detectFastModeBackground(projects(session('main', [apiCall('claude-opus-4-8', 'fast', { output: 100_000 }, 0)])))).toBeNull() + expect(detectFastModeBackground(projects(session('agent-1', [apiCall('claude-opus-4-8', 'fast', { output: 800 }, 0)], true)))).toBeNull() + }) + + it('ignores a model without a fast multiplier', () => { + expect(detectFastModeBackground(projects(session('agent-1', [apiCall('claude-opus-4-6', 'fast', { output: 100_000 }, 0)], true)))).toBeNull() + }) +}) + +describe('detectLongContextPremium', () => { + // sonnet-4-5 above 200k: 10k input + 290k cache read + 1k output costs + // $0.06 + $0.174 + $0.0225 = $0.2565 against $0.03 + $0.087 + $0.015 = $0.132. + const over = (i: number) => apiCall('claude-sonnet-4-5', 'standard', { input: 10_000, cacheRead: 290_000, output: 1_000 }, i) + + it('reports calls priced above the tier', () => { + const finding = detectLongContextPremium(projects( + session('s1', Array.from({ length: 10 }, (_, i) => over(i))), + session('s2', [apiCall('claude-sonnet-4-5', 'standard', { input: 150_000, output: 1_000 }, 20)]), + )) + expect(finding).not.toBeNull() + expect(finding!.id).toBe('long-context-premium') + expect(findingClass(finding!)).toBe('keep') + expect(finding!.apply).toBeUndefined() + expect(finding!.savingsUSD).toBeCloseTo(10 * 0.1245, 10) + expect(finding!.tokensSaved).toBe(10 * 100_000) + expect(finding!.impact).toBe('low') + expect(finding!.explanation).toContain('10 of 11 calls (90.9%) in 1 session ') + expect(finding!.explanation).toContain('Sonnet 4.5 above 200.0K: 10 calls') + expect(finding!.fix.type === 'paste' && finding!.fix.text).toContain('200.0K on Sonnet 4.5') + }) + + it('prices only the tier on a fast call, leaving the fast premium to its own finding', () => { + // gpt-5.5 above 272k at standard: $0.10 + $0.29 + $0.045 = $0.435 against $0.05 + $0.145 + $0.03 = $0.225. + const fast = Array.from({ length: 10 }, (_, i) => apiCall('gpt-5.5', 'fast', { input: 10_000, cacheRead: 290_000, output: 1_000 }, i)) + expect(detectLongContextPremium(projects(session('s1', fast)))!.savingsUSD).toBeCloseTo(10 * 0.21, 10) + }) + + it('ignores calls under the threshold and premiums under $1', () => { + expect(detectLongContextPremium(projects(session('s1', Array.from({ length: 10 }, (_, i) => + apiCall('claude-sonnet-4-5', 'standard', { input: 10_000, cacheRead: 190_000, output: 1_000 }, i)))))).toBeNull() + expect(detectLongContextPremium(projects(session('s1', [over(0)])))).toBeNull() + }) + + it('ignores a model without a long-context tier', () => { + expect(detectLongContextPremium(projects(session('s1', Array.from({ length: 10 }, (_, i) => + apiCall('claude-sonnet-4-6', 'standard', { input: 10_000, cacheRead: 290_000, output: 1_000 }, i)))))).toBeNull() + }) +})