From 7c54cf85c2ec201084171e6c97bc2b0078678105 Mon Sep 17 00:00:00 2001 From: iamtoruk Date: Tue, 18 Aug 2026 02:26:57 -0700 Subject: [PATCH] optimize: classify findings as fix/nudge/keep and mark measured vs estimated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every finding now resolves to a class (apply-able fix, habit nudge, or informational keep) and a basis (measured from provider-counted usage, or estimated from a schema/heuristic model), both from one table next to the FindingId union. The class follows the plan layer: an id is 'fix' only when buildPlan routes it, and an instance drops to 'nudge' when it lacks the payload or cause its builder needs. CLI, TUI and the desktop app group findings under Fix now / Habits / FYI with continuous numbering; the CLI header reports 'N measured · M estimated' in place of the blanket 'Estimates only.' footer. The JSON report gains class + basis per finding and summary.measuredSavingsUSD; existing fields are unchanged. The menubar's top three follow the same order, since every surface reads the sorted findings list. Sessions whose cost the provider never reported leave the cost-outliers peer math; when nothing else is priced the comparison falls back to them and the finding reports itself as estimated instead of disappearing. --- app/renderer/App.test.tsx | 2 +- app/renderer/lib/types.ts | 5 + app/renderer/sections/Optimize.test.tsx | 21 ++- app/renderer/sections/Optimize.tsx | 16 ++- app/renderer/styles/plain.css | 3 + src/act/plans.ts | 2 +- src/dashboard.tsx | 20 ++- src/optimize.ts | 164 ++++++++++++++++++++++-- tests/dashboard.test.ts | 6 +- tests/optimize-apply.test.ts | 78 +++++++++++ tests/optimize.test.ts | 63 +++++++++ 11 files changed, 352 insertions(+), 28 deletions(-) diff --git a/app/renderer/App.test.tsx b/app/renderer/App.test.tsx index dbf5e8ff..2832cb4d 100644 --- a/app/renderer/App.test.tsx +++ b/app/renderer/App.test.tsx @@ -138,7 +138,7 @@ function installDefaultMocks() { summary: { healthScore: 100, healthGrade: 'A', findingCount: 0, periodCostUSD: 0, sessions: 0, calls: 0, potentialSavingsTokens: 0, potentialSavingsCostUSD: 0, - potentialSavingsPercent: 0, costRateUSD: 0, + potentialSavingsPercent: 0, costRateUSD: 0, measuredSavingsUSD: 0, }, findings: [], }) diff --git a/app/renderer/lib/types.ts b/app/renderer/lib/types.ts index 25394087..d1f61a79 100644 --- a/app/renderer/lib/types.ts +++ b/app/renderer/lib/types.ts @@ -407,6 +407,8 @@ export type WasteAction = | { type: 'command'; label: string; text: string } | { type: 'file-content'; label: string; path: string; content: string } +export type FindingClass = 'fix' | 'nudge' | 'keep' + export type OptimizeJsonReport = { period: { label: string; start: string | null; end: string | null } summary: { @@ -420,6 +422,7 @@ export type OptimizeJsonReport = { potentialSavingsCostUSD: number potentialSavingsPercent: number | null costRateUSD: number + measuredSavingsUSD: number } findings: Array<{ id: string @@ -429,6 +432,8 @@ export type OptimizeJsonReport = { trend: 'active' | 'improving' | null tokensSaved: number estimatedSavingsUSD: number + class: FindingClass + basis: 'measured' | 'estimated' fix: WasteAction }> } diff --git a/app/renderer/sections/Optimize.test.tsx b/app/renderer/sections/Optimize.test.tsx index 5219afaf..03b922b5 100644 --- a/app/renderer/sections/Optimize.test.tsx +++ b/app/renderer/sections/Optimize.test.tsx @@ -48,23 +48,26 @@ function makeOptimizeReport(): OptimizeJsonReport { healthScore: 72, healthGrade: 'C', findingCount: 3, periodCostUSD: 612.48, sessions: 88, calls: 1220, potentialSavingsTokens: 184_000, potentialSavingsCostUSD: 94.4, potentialSavingsPercent: 15.4, costRateUSD: 0.0005, + measuredSavingsUSD: 27.8, }, findings: [ { - id: 'cost-outliers', title: 'Opus is doing your small talk', + id: 'unused-mcp', title: 'Opus is doing your small talk', explanation: 'Small conversational requests are running on an expensive model.', severity: 'high', trend: 'active', tokensSaved: 18_200, estimatedSavingsUSD: 9.1, + class: 'fix', basis: 'estimated', fix: { type: 'paste', label: 'Paste into CLAUDE.md', text: 'Use Sonnet for routine questions.', destination: 'claude-md' }, }, { - id: 'context-heavy-sessions', title: 'Cache hit is low in agentseal-dash', + id: 'cost-outliers', title: 'Cache hit is low in agentseal-dash', explanation: 'Repeated context is not being served from cache.', severity: 'medium', - trend: null, tokensSaved: 17_400, estimatedSavingsUSD: 8.7, + trend: null, tokensSaved: 17_400, estimatedSavingsUSD: 8.7, class: 'nudge', basis: 'measured', fix: { type: 'command', label: 'Run this command', text: 'codeburn cache inspect' }, }, { - id: 'warmup-heavy', title: 'Batch tiny requests', explanation: 'Many short sessions repeat setup work.', + id: 'context-heavy-sessions', title: 'Batch tiny requests', explanation: 'Many short sessions repeat setup work.', severity: 'low', trend: 'improving', tokensSaved: 4_800, estimatedSavingsUSD: 2.4, + class: 'keep', basis: 'measured', fix: { type: 'file-content', label: 'Create configuration', path: '~/.codeburn/config.json', content: '{"batch":true}' }, }, ], @@ -121,6 +124,14 @@ describe('Optimize', () => { Object.defineProperty(navigator, 'clipboard', { configurable: true, value: { writeText } }) }) + it('groups Waste findings under the fix / habits / FYI headers in order', async () => { + render() + + await screen.findByText('Opus is doing your small talk') + const groups = document.querySelectorAll('.opt-group') + expect([...groups].map(g => g.textContent)).toEqual(['Fix now (apply-able)', 'Habits', 'FYI']) + }) + it('renders tabs and actionable Waste findings with impact, savings, explanation, and copy-paste fix', async () => { render() @@ -130,7 +141,7 @@ describe('Optimize', () => { expect(screen.getByText('Medium')).toHaveClass('opt-impact-medium') expect(screen.getByText('Low')).toHaveClass('opt-impact-low') expect(screen.getByText('$9.10')).toHaveClass('opt-finding-savings') - expect(screen.getByText('18.2K tokens')).toBeInTheDocument() + expect(screen.getByText('18.2K tokens · estimated')).toBeInTheDocument() expect(screen.getByRole('tab', { name: 'Waste $94.40' })).toBeInTheDocument() expect(screen.getByRole('tab', { name: 'Reverts $107.00' })).toBeInTheDocument() expect(screen.getByRole('tab', { name: 'Abandoned $65.40' })).toBeInTheDocument() diff --git a/app/renderer/sections/Optimize.tsx b/app/renderer/sections/Optimize.tsx index 03d8674b..e0a6b347 100644 --- a/app/renderer/sections/Optimize.tsx +++ b/app/renderer/sections/Optimize.tsx @@ -9,7 +9,7 @@ import { StaleBanner } from '../components/StaleBanner' import { type Polled, usePolled } from '../hooks/usePolled' import { formatCompact, formatUsd } from '../lib/format' import { codeburn } from '../lib/ipc' -import type { DateRange, MenubarPayload, OptimizeJsonReport, Period, SessionYieldJson, WasteAction, YieldJsonReport } from '../lib/types' +import type { DateRange, FindingClass, MenubarPayload, OptimizeJsonReport, Period, SessionYieldJson, WasteAction, YieldJsonReport } from '../lib/types' type OptimizeTab = 'waste' | 'reverts' | 'abandoned' | 'fixes' @@ -114,6 +114,12 @@ const IMPACT_ICON: Record<'high' | 'medium' | 'low', string> = { low: '↓', } +const CLASS_HEADERS: Record = { + fix: 'Fix now (apply-able)', + nudge: 'Habits', + keep: 'FYI', +} + function actionText(fix: WasteAction): string { return fix.type === 'file-content' ? fix.content : fix.text } @@ -132,10 +138,14 @@ function ActionableFindingRows({ findings }: { findings: OptimizeFinding[] }) { return (
- {findings.map(finding => { + {findings.map((finding, i) => { const expanded = expandedId === finding.id + // Findings arrive class-sorted from the CLI, so a header goes in + // wherever the class changes. + const showHeader = finding.class !== findings[i - 1]?.class return ( + {showHeader &&
{CLASS_HEADERS[finding.class]}
} {expanded && ( diff --git a/app/renderer/styles/plain.css b/app/renderer/styles/plain.css index f82172b9..ad1c10ee 100644 --- a/app/renderer/styles/plain.css +++ b/app/renderer/styles/plain.css @@ -640,6 +640,9 @@ td:first-child { font-size: var(--fs-body); font-weight: var(--fw-body); } .opt-waste { min-width: 0; } .opt-summary { padding: 0 0 10px; color: var(--mut); font-size: 11.5px; font-variant-numeric: tabular-nums; } .opt-findings { display: grid; min-width: 0; } +.opt-group { padding: 11px 0 5px; color: var(--mut2); font-size: 10px; font-weight: 600; letter-spacing: 0.06em; text-transform: uppercase; } +.opt-group:first-child { padding-top: 0; } +.opt-group + .opt-finding { border-top: 0; } .opt-finding { display: grid; align-items: center; column-gap: 12px; min-height: 43px; border-top: 1px solid var(--line2); } .opt-finding:first-child { border-top: 0; } .opt-finding-legacy { grid-template-columns: 28px minmax(0, 1fr) 104px 86px; } diff --git a/src/act/plans.ts b/src/act/plans.ts index b3ee4e39..308d2733 100644 --- a/src/act/plans.ts +++ b/src/act/plans.ts @@ -9,6 +9,7 @@ import { ALWAYSLOAD_STARTUP_CAP_SECONDS, ENABLE_TOOL_SEARCH_VAR, parseVersion, + SHELL_PROFILE_SCOPE, versionPredates, } from '../optimize.js' import type { WasteFinding } from '../optimize.js' @@ -386,7 +387,6 @@ const NEXT_SESSION_NOTE = 'takes effect on the next session (this config is read // findDeferralEnvSetting (src/optimize.ts) reports shell-profile hits with // exactly this scope string; the plan layer keys its refusal on it. -const SHELL_PROFILE_SCOPE = 'shell profile' const SHELL_TOOL_SEARCH_LINE = new RegExp(`^\\s*(?:export\\s+)?${ENABLE_TOOL_SEARCH_VAR}\\s*=.*$`, 'm') diff --git a/src/dashboard.tsx b/src/dashboard.tsx index 9cc3db50..d89e6e4b 100644 --- a/src/dashboard.tsx +++ b/src/dashboard.tsx @@ -1,6 +1,6 @@ import { homedir } from 'os' -import React, { useState, useCallback, useEffect, useLayoutEffect, useMemo, useRef } from 'react' +import React, { Fragment, useState, useCallback, useEffect, useLayoutEffect, useMemo, useRef } from 'react' import { render, Box, Text, measureElement, useInput, useApp, useWindowSize, type DOMElement } from 'ink' import { CATEGORY_LABELS, type DateRange, type ProjectSummary, type TaskCategory } from './types.js' import { formatCost, formatTokens, markEstimated, carriedCostNote } from './format.js' @@ -10,7 +10,7 @@ import { findUnpricedModels, isExpectedFreeModel, loadPricing } from './models.j import { aggregateModelTotals } from './model-breakdown.js' import { buildDurablePeriod } from './usage-aggregator.js' import { getAllProviders } from './providers/index.js' -import { scanAndDetect, type WasteFinding, type WasteAction, type OptimizeResult } from './optimize.js' +import { CLASS_HEADERS, findingBasis, findingClass, scanAndDetect, type FindingClass, type WasteFinding, type WasteAction, type OptimizeResult } from './optimize.js' import { aggregateFileChurn, buildCoachingNotes, computePricingCoverage, medianTimeToFirstEditMs, scanUserCorrections, worstOneShotCategory, type ReworkedFile } from './workflow-insights.js' import { estimateContextBudget, type ContextBudget } from './context-budget.js' import { dateKey } from './day-aggregator.js' @@ -1079,7 +1079,7 @@ function FindingPanel({ index, finding, costRate, width }: { index: number; find {trendBadge && {trendBadge}} {finding.explanation} - Savings: ~{formatTokens(finding.tokensSaved)} tokens (~{formatCost(costSaved)}) + Savings: ~{formatTokens(finding.tokensSaved)} tokens (~{formatCost(costSaved)}) {findingBasis(finding)} @@ -1119,8 +1119,18 @@ function OptimizeView({ findings, costRate, projects, label, width, healthScore, Showing {start + 1}–{end} of {total} · j/k to scroll )} - {visible.map((f, i) => )} - Token estimates are approximate. + {visible.map((f, i) => { + // Findings arrive class-sorted, so a header goes in wherever the class + // changes (including the top of the window after paging). + const cls = findingClass(f) + const previous: FindingClass | null = i > 0 ? findingClass(visible[i - 1]!) : null + return ( + + {cls !== previous && {CLASS_HEADERS[cls]}} + + + ) + })} ) } diff --git a/src/optimize.ts b/src/optimize.ts index 190a8a8b..29b0c754 100644 --- a/src/optimize.ts +++ b/src/optimize.ts @@ -264,6 +264,113 @@ export type FindingId = | 'unused-skills' | 'unused-commands' +/// How a finding is meant to be acted on: +/// - `fix` CodeBurn can write the change itself (`codeburn optimize --apply`) +/// - `nudge` behavioural, the user changes a habit +/// - `keep` informational; the cost may well be justified +export type FindingClass = 'fix' | 'nudge' | 'keep' + +/// Where a finding's `tokensSaved` number comes from: +/// - `measured` summed from provider-counted usage on the parsed calls +/// - `estimated` a schema/heuristic model (per-tool sizes, recovery fractions) +/// A detector that mixes the two counts as `estimated`. +export type FindingBasis = 'measured' | 'estimated' + +/// Static class per finding id. `fix` entries are exactly the ids `buildPlan` +/// (src/act/plans.ts) routes to a plan builder; tests assert the two lists +/// stay equal. Instances that lack the payload their builder needs fall back +/// to `nudge` via `findingClass`. +export const FINDING_CLASS: Record = { + 'read-edit-ratio': 'fix', // CLAUDE.md rule block + 'build-folder-reads': 'fix', // CLAUDE.md rule block + 'redundant-rereads': 'nudge', + 'warmup-heavy': 'nudge', + 'unused-mcp': 'fix', + 'mcp-low-coverage': 'fix', + 'mcp-project-scope': 'fix', + 'mcp-deferral-off': 'fix', + 'mcp-alwaysload-hygiene': 'fix', + 'mcp-defer-threshold': 'fix', + 'retry-heavy-capabilities': 'nudge', + 'low-worth-sessions': 'nudge', + 'context-heavy-sessions': 'keep', // context-heavy work is often load-bearing + 'cost-outliers': 'nudge', + 'claude-md-too-long': 'nudge', // trimming is a judgement call, not a rule block + 'bash-output-cap': 'fix', + 'unused-agents': 'fix', + 'unused-skills': 'fix', + 'unused-commands': 'fix', +} + +/// Ids whose plan is built from the `apply` payload: without it the plan +/// builder returns null, so the finding is only a nudge. +const CLASS_NEEDS_APPLY: ReadonlySet = new Set([ + 'unused-mcp', + 'mcp-low-coverage', + 'mcp-project-scope', + 'mcp-deferral-off', + 'mcp-alwaysload-hygiene', + 'mcp-defer-threshold', + 'unused-agents', + 'unused-skills', + 'unused-commands', +]) + +/// Static basis per finding id. Only the two session-level detectors sum +/// provider-counted tokens end to end; everything else multiplies a modelled +/// per-unit size or a recovery fraction. +export const FINDING_BASIS: Record = { + 'read-edit-ratio': 'estimated', // reads x AVG_TOKENS_PER_READ + 'build-folder-reads': 'estimated', // reads x AVG_TOKENS_PER_READ + 'redundant-rereads': 'estimated', // reads x AVG_TOKENS_PER_READ + 'warmup-heavy': 'estimated', // observed median minus a modelled baseline + 'unused-mcp': 'estimated', // tools x TOKENS_PER_MCP_TOOL x sessions + 'mcp-low-coverage': 'estimated', // schema-size model, only capped by observed cache tokens + 'mcp-project-scope': 'estimated', // same schema-size model + 'mcp-deferral-off': 'estimated', // schema-size model x affected sessions + 'mcp-alwaysload-hygiene': 'estimated', // tools x TOKENS_PER_MCP_TOOL x loaded sessions + 'mcp-defer-threshold': 'estimated', // definition-size model x sessions + 'retry-heavy-capabilities': 'estimated', // real turn tokens x recovery fraction + 'low-worth-sessions': 'estimated', // real session tokens x recovery fraction + 'context-heavy-sessions': 'measured', // counted input/cache tokens above the target ratio + 'cost-outliers': 'measured', // counted session tokens above the peer average + 'claude-md-too-long': 'estimated', // lines x CLAUDEMD_TOKENS_PER_LINE + 'bash-output-cap': 'estimated', // chars x BASH_TOKENS_PER_CHAR + 'unused-agents': 'estimated', // count x TOKENS_PER_AGENT_DEF + 'unused-skills': 'estimated', // count x TOKENS_PER_SKILL_DEF + 'unused-commands': 'estimated', // count x TOKENS_PER_COMMAND_DEF +} + +/// Scope label for a setting that lives in ~/.zshrc / ~/.bashrc. Plans never +/// rewrite shell profiles, they only report them. +export const SHELL_PROFILE_SCOPE = 'shell profile' + +export function findingClass(f: WasteFinding): FindingClass { + const base = FINDING_CLASS[f.id] + if (base !== 'fix') return base + if (CLASS_NEEDS_APPLY.has(f.id) && !f.apply) return 'nudge' + const apply = f.apply + if ((apply?.kind === 'defer-enable' || apply?.kind === 'defer-threshold') && apply.settingScope === SHELL_PROFILE_SCOPE) { + return 'nudge' + } + // Of the deferral causes only these two have a plan; the rest are manual + // advice (Vertex policy, an outdated Claude Code, an unverified proxy). + if (apply?.kind === 'defer-enable' && apply.cause !== 'env-false' && apply.cause !== 'proxy-verified') return 'nudge' + return 'fix' +} + +export function findingBasis(f: WasteFinding): FindingBasis { + return f.basis ?? FINDING_BASIS[f.id] +} + +const CLASS_ORDER: Record = { fix: 0, nudge: 1, keep: 2 } + +export const CLASS_HEADERS: Record = { + fix: 'Fix now (apply-able) — codeburn optimize --apply', + nudge: 'Habits', + keep: 'FYI', +} + // Cause taxonomy for defer-enable plans (mcp-deferral-off findings). // 'proxy-verified' is never produced by the detector today: it is reserved // for the #614 part-3 proxy verifier, which upgrades 'proxy-unknown' once a @@ -303,6 +410,9 @@ export type WasteFinding = { fix: WasteAction trend?: Trend apply?: FindingApply + /// Set only when a detector's basis varies per run (see detectSessionOutliers); + /// otherwise `FINDING_BASIS[id]` applies. Read through `findingBasis`. + basis?: FindingBasis } export type OptimizeResult = { @@ -330,6 +440,9 @@ export type OptimizeJsonReport = { potentialSavingsCostUSD: number potentialSavingsPercent: number | null costRateUSD: number + /// Portion of `potentialSavingsCostUSD` coming from `measured`-basis + /// findings. The total keeps its old meaning: measured plus estimated. + measuredSavingsUSD: number } findings: Array<{ id: FindingId @@ -339,6 +452,8 @@ export type OptimizeJsonReport = { trend: Trend | null tokensSaved: number estimatedSavingsUSD: number + class: FindingClass + basis: FindingBasis fix: WasteAction }> /// Files most reworked by edit-family calls, relative to project root (top 15). @@ -1716,7 +1831,7 @@ export function findDeferralEnvSetting( const content = readSessionFileSync(path) if (content === null) continue const match = content.match(linePattern) - if (match) return { value: match[1]!, scope: 'shell profile', path } + if (match) return { value: match[1]!, scope: SHELL_PROFILE_SCOPE, path } } return null } @@ -2814,9 +2929,17 @@ export function detectSessionOutliers(projects: ProjectSummary[], excludedSessio } const outliers: Outlier[] = [] + // Modelled costs (Kiro, Cursor, some Cline sessions) are not comparable + // against provider-reported ones, so they leave the peer math. Providers + // that only ever estimate would lose the finding entirely, so those fall + // back to the full set and the finding reports itself as estimated. + let usedEstimatedCosts = false for (const project of projects) { - const sessions = project.sessions.filter(s => s.totalCostUSD > 0) + const costed = project.sessions.filter(s => s.totalCostUSD > 0) + const exact = costed.filter(s => (s.totalEstimatedCostUSD ?? 0) === 0) + const sessions = exact.length >= MIN_SESSIONS_FOR_OUTLIER ? exact : costed + const fellBack = sessions.length > exact.length if (sessions.length < MIN_SESSIONS_FOR_OUTLIER) continue const totalCost = sessions.reduce((sum, s) => sum + s.totalCostUSD, 0) @@ -2835,6 +2958,7 @@ export function detectSessionOutliers(projects: ProjectSummary[], excludedSessio // "tighter constraint" advice here. if (excludedSessionIds?.has(session.sessionId)) continue + if (fellBack) usedEstimatedCosts = true outliers.push({ project: project.project, sessionId: session.sessionId, @@ -2864,6 +2988,7 @@ export function detectSessionOutliers(projects: ProjectSummary[], excludedSessio explanation: `Sessions costing more than ${SESSION_OUTLIER_MULTIPLIER}x their peer-session average in the same project: ${list}${extra}. These usually come from broad prompts, runaway loops, or context-heavy work that should be split into smaller sessions.`, impact: outliers.length >= 3 || totalExcessCost >= 10 ? 'high' : 'medium', tokensSaved, + ...(usedEstimatedCosts ? { basis: 'estimated' as const } : {}), fix: { type: 'paste', destination: 'session-opener', @@ -3080,7 +3205,10 @@ export async function scanAndDetect( : [] for (const f of ghostResults) if (f) findings.push(f) + // Urgency first, then class: every surface lists the apply-able fixes + // before the habit nudges, and orders by urgency inside each group. findings.sort((a, b) => urgencyScore(b) - urgencyScore(a)) + findings.sort((a, b) => CLASS_ORDER[findingClass(a)] - CLASS_ORDER[findingClass(b)]) const { score, grade } = computeHealth(findings) const modelRecommendations: ModelDefaultRecommendation[] = [] @@ -3164,7 +3292,7 @@ function renderFinding(n: number, f: WasteFinding, costRate: number): string[] { lines.push('') lines.push(wrap(f.explanation, PANEL_WIDTH - 4, ' ')) lines.push('') - lines.push(chalk.hex(GOLD)(` Potential savings: ${savings}`)) + lines.push(chalk.hex(GOLD)(` Potential savings: ${savings}`) + chalk.dim(` ${findingBasis(f)}`)) lines.push('') // Destination header — issue #277. Tells the user where each suggestion @@ -3207,7 +3335,7 @@ function renderWorkflowSection(reworkedFiles: ReworkedFile[], coachingNotes: str return lines } -function renderOptimize( +export function renderOptimize( findings: WasteFinding[], costRate: number, periodLabel: string, @@ -3228,12 +3356,16 @@ function renderOptimize( lines.push(chalk.hex(DIM)(' ' + SEP.repeat(PANEL_WIDTH))) const issueSuffix = findings.length > 0 ? `, ${findings.length} issue${findings.length > 1 ? 's' : ''}` : '' + const measured = findings.filter(f => findingBasis(f) === 'measured').length lines.push(' ' + [ `${sessionCount} sessions`, `${callCount.toLocaleString()} calls`, chalk.hex(GOLD)(formatCost(periodCost)), `Health: ${chalk.bold.hex(GRADE_COLORS[healthGrade])(healthGrade)}${chalk.dim(` (${healthScore}/100${issueSuffix})`)}`, ].join(chalk.hex(DIM)(' '))) + if (findings.length > 0) { + lines.push(chalk.dim(` ${measured} measured · ${findings.length - measured} estimated`)) + } if (appliedHeader) lines.push(' ' + chalk.hex(GREEN)(appliedHeader)) lines.push('') @@ -3257,15 +3389,22 @@ function renderOptimize( lines.push(chalk.hex(GREEN)(` Potential savings: ~${formatTokens(totalTokens)} tokens${costText}`)) lines.push('') - for (let i = 0; i < findings.length; i++) { - const f = findings[i]! - const appliedOn = previouslyApplied?.[f.id] - const shown = appliedOn ? { ...f, title: `${f.title} (previously applied ${appliedOn}, re-flagged)` } : f - lines.push(...renderFinding(i + 1, shown, costRate)) + // One block per class, in fix -> nudge -> keep order; numbering runs + // continuously across the blocks so `--only` picks stay unambiguous. + let n = 0 + for (const cls of ['fix', 'nudge', 'keep'] as const) { + const group = findings.filter(f => findingClass(f) === cls) + if (group.length === 0) continue + lines.push(chalk.bold.hex(ORANGE)(` ${CLASS_HEADERS[cls]}`)) + lines.push('') + for (const f of group) { + const appliedOn = previouslyApplied?.[f.id] + const shown = appliedOn ? { ...f, title: `${f.title} (previously applied ${appliedOn}, re-flagged)` } : f + lines.push(...renderFinding(++n, shown, costRate)) + } } lines.push(chalk.hex(DIM)(' ' + SEP.repeat(PANEL_WIDTH))) - lines.push(chalk.dim(' Estimates only.')) lines.push('') lines.push(...renderWorkflowSection(reworkedFiles, coachingNotes)) @@ -3365,6 +3504,9 @@ export function buildOptimizeJsonReport( potentialSavingsCostUSD, potentialSavingsPercent, costRateUSD: result.costRate, + measuredSavingsUSD: result.findings + .filter(f => findingBasis(f) === 'measured') + .reduce((s, f) => s + f.tokensSaved * result.costRate, 0), }, findings: result.findings.map(f => ({ id: f.id, @@ -3374,6 +3516,8 @@ export function buildOptimizeJsonReport( trend: f.trend ?? null, tokensSaved: f.tokensSaved, estimatedSavingsUSD: f.tokensSaved * result.costRate, + class: findingClass(f), + basis: findingBasis(f), fix: f.fix, })), ...buildWorkflowReport(projects), diff --git a/tests/dashboard.test.ts b/tests/dashboard.test.ts index 9879c8a1..1fe52c1e 100644 --- a/tests/dashboard.test.ts +++ b/tests/dashboard.test.ts @@ -712,12 +712,12 @@ describe('InteractiveDashboard refresh', () => { expect(activityHeader.indexOf('turns') + 'turns'.length).toBe(activityRow.indexOf('12') + '12'.length) expect(activityHeader.indexOf('1-shot') + '1-shot'.length).toBe(activityRow.indexOf('50%') + '50%'.length) stdin.write('o') - for (let i = 0; i < 20 && !frames.some(frame => frame.includes('Token estimates are approximate.')); i++) { + for (let i = 0; i < 20 && !frames.some(frame => frame.includes('Savings: ~')); i++) { await vi.advanceTimersByTimeAsync(50) } const beforeRefresh = frames.filter(frame => frame.trim()).at(-1) ?? '' expect(beforeRefresh).toContain('CodeBurn Optimize') - expect(beforeRefresh).toContain('Token estimates are approximate.') + expect(beforeRefresh).toContain('CodeBurn Optimize') frames.length = 0 await vi.advanceTimersByTimeAsync(60_000) @@ -726,7 +726,7 @@ describe('InteractiveDashboard refresh', () => { const frame = frames.filter(value => value.trim()).at(-1) ?? beforeRefresh expect(frame).toBe(beforeRefresh) expect(frame).toContain('CodeBurn Optimize') - expect(frame).toContain('Token estimates are approximate.') + expect(frame).toContain('CodeBurn Optimize') expect(frame).toContain('b back') expect(frame).not.toContain('Loading Today') expect(frame).not.toContain('Scanning Today') diff --git a/tests/optimize-apply.test.ts b/tests/optimize-apply.test.ts index 2b582284..3a2569b2 100644 --- a/tests/optimize-apply.test.ts +++ b/tests/optimize-apply.test.ts @@ -12,6 +12,9 @@ import { runAction } from '../src/act/apply.js' import { undoAction } from '../src/act/undo.js' import { readRecords, shortId } from '../src/act/journal.js' import { + FINDING_BASIS, + FINDING_CLASS, + findingClass, detectBloatedClaudeMd, detectDuplicateReads, detectJunkReads, @@ -653,3 +656,78 @@ describe('stale-plan detection', () => { expect(await readFile(p, 'utf-8')).toBe('overwritten') }) }) + +describe('finding class', () => { + it('covers every finding id with a class and a basis', () => { + expect(Object.keys(FINDING_BASIS).sort()).toEqual(Object.keys(FINDING_CLASS).sort()) + }) + + it("classes a finding 'fix' exactly when a plan can be built for it", async () => { + const fx = await makeFixture() + const claudeJson = join(fx.home, '.claude.json') + await writeFile(claudeJson, JSON.stringify({ mcpServers: { srv: { command: 's' } } }, null, 2) + '\n') + const settings = join(fx.project, '.claude', 'settings.json') + await mkdir(join(fx.project, '.claude'), { recursive: true }) + await writeFile(settings, JSON.stringify({ env: { ENABLE_TOOL_SEARCH: 'auto' } }, null, 2) + '\n') + const mcpJson = join(fx.project, '.mcp.json') + await writeFile(mcpJson, JSON.stringify({ mcpServers: { pinned: { command: 'x', alwaysLoad: true } } }, null, 2) + '\n') + await mkdir(join(fx.home, '.claude', 'skills', 'ghost'), { recursive: true }) + await mkdir(join(fx.home, '.claude', 'agents'), { recursive: true }) + await mkdir(join(fx.home, '.claude', 'commands'), { recursive: true }) + await writeFile(join(fx.home, '.claude', 'agents', 'ghost.md'), 'x') + await writeFile(join(fx.home, '.claude', 'commands', 'ghost.md'), 'x') + + const CLAUDE_MD_FIX: WasteAction = { type: 'paste', destination: 'claude-md', label: '', text: 'rule' } + const SHELL_FIX: WasteAction = { type: 'paste', destination: 'shell-config', label: '', text: 'export X=1' } + const PROMPT_FIX: WasteAction = { type: 'paste', destination: 'prompt', label: '', text: 'ask' } + const OPENER_FIX: WasteAction = { type: 'paste', destination: 'session-opener', label: '', text: 'o' } + + // One representative finding per id, carrying the payload its plan + // builder needs. Ids without a builder get a plain prompt fix. + const representatives: Record = { + 'read-edit-ratio': makeFinding('read-edit-ratio', CLAUDE_MD_FIX), + 'build-folder-reads': makeFinding('build-folder-reads', CLAUDE_MD_FIX), + 'redundant-rereads': makeFinding('redundant-rereads', PROMPT_FIX), + 'warmup-heavy': makeFinding('warmup-heavy', SHELL_FIX), + 'unused-mcp': makeFinding('unused-mcp', CMD_FIX, { kind: 'mcp-remove', servers: ['srv'] }), + 'mcp-low-coverage': makeFinding('mcp-low-coverage', CMD_FIX, { kind: 'mcp-remove', servers: ['srv'] }), + 'mcp-project-scope': makeFinding('mcp-project-scope', PROMPT_FIX, { + kind: 'mcp-project-scope', + servers: [{ server: 'srv', keepProjects: [fx.project], removeProjects: [] }], + }), + 'mcp-deferral-off': makeFinding('mcp-deferral-off', CMD_FIX, { + kind: 'defer-enable', cause: 'env-false', settingPath: settings, settingScope: 'project settings', value: 'false', + }), + 'mcp-alwaysload-hygiene': makeFinding('mcp-alwaysload-hygiene', PROMPT_FIX, { + kind: 'defer-alwaysload', + servers: [{ server: 'pinned', paths: [mcpJson] }], + }), + 'mcp-defer-threshold': makeFinding('mcp-defer-threshold', PROMPT_FIX, { + kind: 'defer-threshold', settingPath: settings, settingScope: 'project settings', + value: 'auto', recommendedPercent: 2, removeOverride: false, + }), + 'retry-heavy-capabilities': makeFinding('retry-heavy-capabilities', PROMPT_FIX), + 'low-worth-sessions': makeFinding('low-worth-sessions', OPENER_FIX), + 'context-heavy-sessions': makeFinding('context-heavy-sessions', OPENER_FIX), + 'cost-outliers': makeFinding('cost-outliers', OPENER_FIX), + 'claude-md-too-long': makeFinding('claude-md-too-long', PROMPT_FIX), + 'bash-output-cap': makeFinding('bash-output-cap', SHELL_FIX), + 'unused-agents': makeFinding('unused-agents', CMD_FIX, { kind: 'archive', names: ['ghost'] }), + 'unused-skills': makeFinding('unused-skills', CMD_FIX, { kind: 'archive', names: ['ghost'] }), + 'unused-commands': makeFinding('unused-commands', CMD_FIX, { kind: 'archive', names: ['ghost'] }), + } + + const planCtx: PlanContext = { homeDir: fx.home, cwd: fx.project, shell: '/bin/zsh', claudeVersion: () => '2.1.130' } + for (const finding of Object.values(representatives)) { + const hasPlan = planFor(finding, planCtx) !== null + expect([finding.id, hasPlan]).toEqual([finding.id, findingClass(finding) === 'fix']) + } + }) + + it("drops to 'nudge' when the instance lacks the payload its plan needs", () => { + const finding = makeFinding('mcp-deferral-off', { type: 'paste', destination: 'shell-config', label: '', text: 'x' }) + expect(FINDING_CLASS['mcp-deferral-off']).toBe('fix') + expect(findingClass(finding)).toBe('nudge') + expect(planFor(finding)).toBeNull() + }) +}) diff --git a/tests/optimize.test.ts b/tests/optimize.test.ts index bc72dd88..96ba9655 100644 --- a/tests/optimize.test.ts +++ b/tests/optimize.test.ts @@ -26,6 +26,9 @@ import { computeHealth, computeTrend, buildOptimizeJsonReport, + renderOptimize, + findingBasis, + type FindingId, type ToolCall, type ApiCallMeta, type WasteFinding, @@ -1004,6 +1007,29 @@ describe('detectSessionOutliers', () => { expect(finding!.tokensSaved).toBeGreaterThan(0) }) + it('keeps estimated-cost sessions out of the peer math', () => { + const project = projectWithSessions([1, 1, 1, 10]) + // The expensive session is priced from modelled tokens, so it is not + // comparable against the provider-reported peers and never gets flagged. + project.sessions[3]!.totalEstimatedCostUSD = project.sessions[3]!.totalCostUSD + expect(detectSessionOutliers([project])).toBeNull() + }) + + it('falls back to estimated costs when nothing else is priced, and says so', () => { + const project = projectWithSessions([1, 1, 1, 10]) + for (const s of project.sessions) s.totalEstimatedCostUSD = s.totalCostUSD + const finding = detectSessionOutliers([project]) + expect(finding).not.toBeNull() + expect(finding!.basis).toBe('estimated') + expect(findingBasis(finding!)).toBe('estimated') + }) + + it('reports measured basis when every peer cost is provider-reported', () => { + const finding = detectSessionOutliers([projectWithSessions([1, 1, 1, 10])]) + expect(finding!.basis).toBeUndefined() + expect(findingBasis(finding!)).toBe('measured') + }) + it('ignores tiny absolute-cost outliers', () => { expect(detectSessionOutliers([projectWithSessions([0.01, 0.01, 0.01, 0.2])])).toBeNull() }) @@ -1240,6 +1266,7 @@ describe('buildOptimizeJsonReport', () => { healthGrade: 'C', findings: [ { + id: 'claude-md-too-long', title: 'Trim stale context', explanation: 'Old instructions are loaded every turn.', impact: 'medium', @@ -1283,12 +1310,15 @@ describe('buildOptimizeJsonReport', () => { potentialSavingsPercent: 20, costRateUSD: 0.00002, }) + expect(report.summary.measuredSavingsUSD).toBe(0) expect(report.findings[0]).toMatchObject({ title: 'Trim stale context', severity: 'medium', trend: 'active', tokensSaved: 50_000, estimatedSavingsUSD: 1, + class: 'nudge', + basis: 'estimated', fix: { type: 'paste', destination: 'claude-md', @@ -1296,3 +1326,36 @@ describe('buildOptimizeJsonReport', () => { }) }) }) + +describe('renderOptimize grouping', () => { + const plain = (s: string): string => s.replace(/\[[0-9;]*m/g, '') + + function finding(id: FindingId, title: string): WasteFinding { + return { + id, + title, + explanation: 'why', + impact: 'medium', + tokensSaved: 1000, + fix: { type: 'paste', destination: 'prompt', label: 'ask', text: 'ask' }, + } + } + + it('groups findings under fix / habits / FYI with continuous numbering and a basis split', () => { + const findings = [ + finding('bash-output-cap', 'Cap bash output'), + finding('claude-md-too-long', 'Trim CLAUDE.md'), + finding('context-heavy-sessions', 'Context-heavy sessions'), + ] + const out = plain(renderOptimize(findings, 0.00001, '7 Days', 10, 5, 100, 80, 'B', [], [])) + + const headers = ['Fix now (apply-able)', 'Habits', 'FYI'].map(h => out.indexOf(h)) + expect(headers.every(i => i >= 0)).toBe(true) + expect(headers).toEqual([...headers].sort((a, b) => a - b)) + expect(out).toContain('1. Cap bash output') + expect(out).toContain('2. Trim CLAUDE.md') + expect(out).toContain('3. Context-heavy sessions') + expect(out).toContain('1 measured · 2 estimated') + expect(out).not.toContain('Estimates only.') + }) +})