From 52eb9fdb8d31d622b3fe0eeb2adab4c9fdd2904f Mon Sep 17 00:00:00 2001 From: iamtoruk Date: Tue, 18 Aug 2026 03:15:07 -0700 Subject: [PATCH 1/3] optimize: re-measure applied fixes on every run, with --auto-revert Every still-applied journal entry now comes back with a verdict on the next optimize run: worked (>=70% of its window-scaled estimate realized), partial, no-effect (printed with its undo command), or measuring while it is younger than the 3-day window. The verdicts come off the rows act report already computes, so there is one reconciliation, not two; the AppliedFix type and its formatter live in act/types.ts so the optimize renderer can use them without importing report.ts back into optimize.ts. --auto-revert undoes the no-effect entries through the same code path as codeburn act undo. It never touches partial or measuring entries, and never a claude-md-rule - those land in whatever directory the user happened to be in, the same reason --yes skips them. --apply now names when the re-measure happens, and --format json carries appliedFixes[] (add-only). --- src/act/optimize-apply.ts | 6 ++ src/act/report.ts | 71 ++++++++++++++++++- src/act/types.ts | 60 ++++++++++++++++ src/main.ts | 45 +++++++----- src/optimize.ts | 42 +++++++++++- tests/act-report.test.ts | 128 +++++++++++++++++++++++++++++++++++ tests/optimize-apply.test.ts | 8 +++ tests/optimize.test.ts | 55 +++++++++++++++ 8 files changed, 391 insertions(+), 24 deletions(-) diff --git a/src/act/optimize-apply.ts b/src/act/optimize-apply.ts index b72af509..da970ca8 100644 --- a/src/act/optimize-apply.ts +++ b/src/act/optimize-apply.ts @@ -7,6 +7,7 @@ import { formatCost } from '../currency.js' import { formatTokens } from '../format.js' import { runAction } from './apply.js' import { shortId } from './journal.js' +import { REPORT_MIN_AGE_DAYS } from './types.js' import { planFindings, type FindingPlan, type PlanContext } from './plans.js' export type ApplyOptions = { @@ -176,9 +177,11 @@ export async function runOptimizeApply( } catch { /* baseline is optional; apply proceeds without it */ } print() + let applied = 0 for (const fp of selected) { try { const record = await runAction(fp.plan!, opts.actionsDir) + applied++ print(` Applied ${chalk.bold(shortId(record.id))} ${record.description}`) print(chalk.dim(` Undo anytime: codeburn act undo ${shortId(record.id)}`)) } catch (e) { @@ -186,5 +189,8 @@ export async function runOptimizeApply( process.exitCode = 1 } } + if (applied > 0) { + print(chalk.dim(` CodeBurn will re-measure these on your next optimize run after ${REPORT_MIN_AGE_DAYS} days.`)) + } print() } diff --git a/src/act/report.ts b/src/act/report.ts index 30c88120..533ae259 100644 --- a/src/act/report.ts +++ b/src/act/report.ts @@ -1,7 +1,8 @@ import { existsSync } from 'fs' import { dirname } from 'node:path' import type { DateRange, ProjectSummary, SessionSummary } from '../types.js' -import type { ActionBaseline, ActionKind, ActionRecord } from './types.js' +import type { ActionBaseline, ActionKind, ActionRecord, AppliedFix, AppliedVerdict } from './types.js' +import { REPORT_MIN_AGE_DAYS, VERDICT_WORKED_RATIO } from './types.js' import type { FindingPlan } from './plans.js' import { AVG_TOKENS_PER_READ, @@ -20,7 +21,8 @@ import { } from '../optimize.js' import { parseAllSessions } from '../parser.js' import { computeYield, type YieldSummary } from '../yield.js' -import { defaultActionsDir, readRecords } from './journal.js' +import { defaultActionsDir, readRecords, shortId } from './journal.js' +import { undoAction } from './undo.js' import { renderTable } from '../text-table.js' import { formatTokens } from '../format.js' import { formatCost } from '../currency.js' @@ -28,7 +30,6 @@ import { formatCost } from '../currency.js' const DAY_MS = 24 * 60 * 60 * 1000 const WINDOW_CAP_DAYS = 30 const BASELINE_WINDOW_DAYS = 14 -const REPORT_MIN_AGE_DAYS = 3 const MIN_POST_WINDOW_SESSIONS = 20 const VOLUME_SHIFT_FACTOR = 2 @@ -59,6 +60,8 @@ const ARCHIVE_DEF_TOKENS: Partial> = { // 'pending' means the applied change has not taken effect in any post-apply // session yet (e.g. deferral before a client restart) - distinct from // 'reverted', which asserts the user undid it. +export { REPORT_MIN_AGE_DAYS } + export type RealizedStatus = 'measured' | 'reverted' | 'not-measurable' | 'pending' export type ActReportRow = { @@ -102,6 +105,8 @@ export type ActReport = { // findingId -> earliest apply date of an active applied action; drives the // optimize "(previously applied ..., re-flagged)" title suffix. appliedByFinding: Record + // One entry per active applied action, including ones too young to measure. + appliedFixes: AppliedFix[] } export type ActReportOptions = { @@ -470,6 +475,63 @@ function isSaneRecord(r: ActionRecord): boolean { return typeof r.at === 'string' && typeof r.status === 'string' && !Number.isNaN(new Date(r.at).getTime()) } +// Turn the measured rows plus the still-young entries into one verdict per +// active applied action. No second reconciliation: everything measurable comes +// straight off the row `act report` already computed. +function buildAppliedFixes(active: ActionRecord[], rows: ActReportRow[], now: Date): AppliedFix[] { + const byId = new Map(rows.map(r => [r.id, r])) + return active.map(rec => { + const row = byId.get(rec.id) + const base = { + id: rec.id, + kind: rec.kind, + findingId: rec.findingId ?? null, + appliedAt: rec.at, + ageDays: ageDays(rec.at, now), + undoCommand: `codeburn act undo ${shortId(rec.id)}`, + } + // No row means too young to measure; a row that is not a measured token + // row (not-measurable, not yet in effect, reverted by the user, or a + // correlation-only kind) has no reduction to judge either. + if (!row) return { ...base, verdict: 'pending' as const, estimatedTokens: rec.baseline?.estimatedTokens ?? 0, realizedTokens: 0, note: '' } + if (row.status !== 'measured' || !isTokenKind(row.kind)) { + return { ...base, verdict: 'pending' as const, estimatedTokens: row.estimatedForWindow, realizedTokens: 0, note: row.note } + } + const estimatedTokens = row.estimatedForWindow + const realizedTokens = row.realizedTokens + const verdict: AppliedVerdict = realizedTokens <= 0 + ? 'no-effect' + : estimatedTokens <= 0 || realizedTokens >= estimatedTokens * VERDICT_WORKED_RATIO ? 'worked' : 'partial' + return { ...base, verdict, estimatedTokens, realizedTokens, note: row.note } + }) +} + +// --auto-revert: undo the fixes that measured no reduction at all. CLAUDE.md +// rules are never undone unattended, matching the --yes guardrail - the file +// belongs to whatever project the user happened to be in. +export async function autoRevertNoEffect( + fixes: AppliedFix[], opts: { actionsDir?: string } = {}, +): Promise<{ lines: string[]; revertedIds: Set }> { + const lines: string[] = [] + const revertedIds = new Set() + for (const fix of fixes) { + if (fix.verdict !== 'no-effect') continue + const label = fix.findingId ?? fix.kind + if (fix.kind === 'claude-md-rule') { + lines.push(`Not auto-reverted: ${label} edits a CLAUDE.md. Revert: ${fix.undoCommand}`) + continue + } + try { + const record = await undoAction({ id: fix.id }, { actionsDir: opts.actionsDir }) + revertedIds.add(fix.id) + lines.push(`Reverted ${shortId(record.id)}: ${record.description}`) + } catch (err) { + lines.push(`Could not revert ${label}: ${err instanceof Error ? err.message : String(err)}`) + } + } + return { lines, revertedIds } +} + export async function computeActReport(opts: ActReportOptions = {}): Promise { const now = opts.now ?? new Date() const rawRecords = await readRecords(opts.actionsDir ?? defaultActionsDir()) @@ -497,6 +559,7 @@ export async function computeActReport(opts: ActReportOptions = {}): Promise ageDays(r.at, now) > REPORT_MIN_AGE_DAYS) @@ -550,6 +613,7 @@ export async function computeActReport(opts: ActReportOptions = {}): Promise = { + worked: '\u2713', + partial: '~', + 'no-effect': '\u2717', + pending: '\u2026', +} + +export function appliedFixGlyph(fix: AppliedFix): string { + return VERDICT_GLYPH[fix.verdict] +} + +// One plain line per applied fix: what it estimated, what it measured, and for +// a fix that did nothing, how to put it back. +export function formatAppliedFix(fix: AppliedFix): string { + const age = Math.max(0, Math.floor(fix.ageDays)) + const head = `${fix.findingId ?? fix.kind} (${age}d ago)` + if (fix.verdict === 'pending') { + const why = fix.note || (age <= REPORT_MIN_AGE_DAYS + ? `measuring, check back after ${REPORT_MIN_AGE_DAYS} days` + : 'measuring') + return `${head}: ${why}` + } + const pair = `est. ${formatTokens(fix.estimatedTokens)} -> measured ${formatTokens(fix.realizedTokens)}` + if (fix.verdict === 'worked') return `${head}: ${pair}` + if (fix.verdict === 'partial') { + const under = Math.round((1 - fix.realizedTokens / fix.estimatedTokens) * 100) + return `${head}: ${pair} (-${under}% vs estimate)` + } + return `${head}: ${pair} - did not help. Revert: ${fix.undoCommand}` +} diff --git a/src/main.ts b/src/main.ts index a8af9bba..e5246df5 100644 --- a/src/main.ts +++ b/src/main.ts @@ -11,6 +11,7 @@ import { renderStatusBar } from './format.js' import { toDateString } from './daily-cache.js' import { dateKey } from './day-aggregator.js' import { CATEGORY_LABELS, type DateRange, type ProjectSummary, type TaskCategory } from './types.js' +import type { AppliedFix } from './act/types.js' import { aggregateModelEfficiency } from './model-efficiency.js' import { buildPeriodData, buildMenubarPayloadForRange, buildDurablePeriod, type DurablePeriod } from './usage-aggregator.js' import { renderDashboard } from './dashboard.js' @@ -1806,6 +1807,7 @@ program .option('--yes', 'With --apply: apply every appliable fix without prompting') .option('--dry-run', 'With --apply: print the plan and exit without changing anything') .option('--only ', 'With --apply: restrict to a comma-separated list of finding ids') + .option('--auto-revert', 'Undo applied fixes that measured no reduction (never CLAUDE.md rules)') .action(async (opts) => { assertProvider(opts.provider, 'optimize') const format = opts.json ? 'json' : opts.format @@ -1835,24 +1837,31 @@ program return } assertFormat(format, ['text', 'json'], 'optimize') - if (format === 'text') { - // Surface realized savings from applied actions. Best effort: optimize - // must never fail because of journal contents, so any error just drops - // the header. computeActReport returns fast without scanning when the - // journal has no eligible applied actions, so users who never opted in - // see identical output. - let appliedHeader: string | undefined - let previouslyApplied: Record | undefined - try { - const { computeActReport, buildOptimizeAppliedHeader } = await import('./act/report.js') - const applied = await computeActReport() - appliedHeader = buildOptimizeAppliedHeader(applied) ?? undefined - previouslyApplied = applied.appliedByFinding - } catch { /* the header is optional; never block the findings */ } - await runOptimize(projects, label, range, { format, appliedHeader, previouslyApplied, provider: opts.provider }) - } else { - await runOptimize(projects, label, range, { format, provider: opts.provider }) - } + // Surface realized savings from applied actions, and re-measure every one + // of them. Best effort: optimize must never fail because of journal + // contents, so any error just drops the extras. computeActReport returns + // fast without scanning when the journal has no applied actions, so users + // who never opted in see identical output. + let appliedHeader: string | undefined + let previouslyApplied: Record | undefined + let appliedFixes: AppliedFix[] | undefined + try { + const { computeActReport, buildOptimizeAppliedHeader, autoRevertNoEffect } = await import('./act/report.js') + const applied = await computeActReport() + appliedHeader = buildOptimizeAppliedHeader(applied) ?? undefined + previouslyApplied = applied.appliedByFinding + appliedFixes = applied.appliedFixes + if (opts.autoRevert) { + const { lines, revertedIds } = await autoRevertNoEffect(appliedFixes) + appliedFixes = appliedFixes.filter(f => !revertedIds.has(f.id)) + // JSON output must stay parseable, so the revert log goes to stderr there. + for (const line of lines) { + if (format === 'json') process.stderr.write(` ${line}\n`) + else console.log(` ${line}`) + } + } + } catch { /* the applied section is optional; never block the findings */ } + await runOptimize(projects, label, range, { format, appliedHeader, previouslyApplied, appliedFixes, provider: opts.provider }) }) program diff --git a/src/optimize.ts b/src/optimize.ts index 4b88a33d..719764ea 100644 --- a/src/optimize.ts +++ b/src/optimize.ts @@ -12,6 +12,7 @@ import type { DateRange, ProjectSummary, SessionSummary } from './types.js' import { formatCost } from './currency.js' import { formatTokens } from './format.js' import { recommendModelDefault, type ModelDefaultRecommendation } from './act/model-defaults.js' +import { appliedFixGlyph, formatAppliedFix, type AppliedFix } from './act/types.js' import { aggregateFileChurn, buildCoachingNotes, scanUserCorrections, medianTimeToFirstEditMs, worstOneShotCategory, type ReworkedFile } from './workflow-insights.js' // ============================================================================ @@ -491,6 +492,8 @@ export type OptimizeJsonReport = { /// 1-3 templated one-liners keyed on the strongest workflow signals. coachingNotes: string[] modelRecommendations?: Array + /// One entry per still-applied fix, re-measured on every run (see act/report.ts). + appliedFixes: Array> } export type ToolCall = { @@ -3365,6 +3368,25 @@ function renderWorkflowSection(reworkedFiles: ReworkedFile[], coachingNotes: str return lines } +const APPLIED_FIX_COLORS: Record = { + worked: GREEN, + partial: GOLD, + 'no-effect': RED, + pending: DIM, +} + +// Closes the loop after --apply: every still-applied fix gets its measured +// verdict back here, on every run. +function renderAppliedFixes(appliedFixes: AppliedFix[]): string[] { + if (appliedFixes.length === 0) return [] + const lines = [chalk.bold.hex(ORANGE)(' Applied fixes'), ''] + for (const fix of appliedFixes) { + lines.push(chalk.hex(APPLIED_FIX_COLORS[fix.verdict])(` ${appliedFixGlyph(fix)} ${formatAppliedFix(fix)}`)) + } + lines.push('') + return lines +} + export function renderOptimize( findings: WasteFinding[], costRate: number, @@ -3379,6 +3401,7 @@ export function renderOptimize( appliedHeader?: string, previouslyApplied?: Record, modelRecommendations?: ModelDefaultRecommendation[], + appliedFixes: AppliedFix[] = [], ): string { const lines: string[] = [] lines.push('') @@ -3406,6 +3429,7 @@ export function renderOptimize( lines.push(chalk.dim(' token waste: junk directory reads, duplicate file reads, unused')) lines.push(chalk.dim(' agents/skills/MCP servers, bloated CLAUDE.md, and more.')) lines.push('') + lines.push(...renderAppliedFixes(appliedFixes)) lines.push(...renderWorkflowSection(reworkedFiles, coachingNotes)) return lines.join('\n') } @@ -3441,6 +3465,7 @@ export function renderOptimize( lines.push(chalk.hex(DIM)(' ' + SEP.repeat(PANEL_WIDTH))) lines.push('') + lines.push(...renderAppliedFixes(appliedFixes)) lines.push(...renderWorkflowSection(reworkedFiles, coachingNotes)) if (modelRecommendations && modelRecommendations.length > 0) { @@ -3478,7 +3503,7 @@ export async function runOptimize( projects: ProjectSummary[], periodLabel: string, dateRange?: DateRange, - opts: { format?: 'text' | 'json'; appliedHeader?: string; previouslyApplied?: Record; provider?: string } = {}, + opts: { format?: 'text' | 'json'; appliedHeader?: string; previouslyApplied?: Record; appliedFixes?: AppliedFix[]; provider?: string } = {}, ): Promise { const format = opts.format ?? 'text' if (projects.length === 0 && format === 'text') { @@ -3497,12 +3522,12 @@ export async function runOptimize( const callCount = projects.reduce((s, p) => s + p.totalApiCalls, 0) if (format === 'json') { - console.log(JSON.stringify(buildOptimizeJsonReport(projects, periodLabel, result, dateRange), null, 2)) + console.log(JSON.stringify(buildOptimizeJsonReport(projects, periodLabel, result, dateRange, opts.appliedFixes), null, 2)) return } const { topReworkedFiles, coachingNotes } = buildWorkflowReport(projects) - const output = renderOptimize(findings, costRate, periodLabel, periodCost, sessions.length, callCount, healthScore, healthGrade, topReworkedFiles, coachingNotes, opts.appliedHeader, opts.previouslyApplied, result.modelRecommendations) + const output = renderOptimize(findings, costRate, periodLabel, periodCost, sessions.length, callCount, healthScore, healthGrade, topReworkedFiles, coachingNotes, opts.appliedHeader, opts.previouslyApplied, result.modelRecommendations, opts.appliedFixes) console.log(output) } @@ -3511,6 +3536,7 @@ export function buildOptimizeJsonReport( periodLabel: string, result: OptimizeResult, dateRange?: DateRange, + appliedFixes: AppliedFix[] = [], ): OptimizeJsonReport { const sessions = projects.flatMap(p => p.sessions) const periodCostUSD = projects.reduce((s, p) => s + p.totalCostUSD, 0) @@ -3557,5 +3583,15 @@ export function buildOptimizeJsonReport( })), ...buildWorkflowReport(projects), modelRecommendations: result.modelRecommendations, + appliedFixes: appliedFixes.map(f => ({ + id: f.id, + kind: f.kind, + findingId: f.findingId, + appliedAt: f.appliedAt, + verdict: f.verdict, + estimatedTokens: f.estimatedTokens, + realizedTokens: f.realizedTokens, + undoCommand: f.undoCommand, + })), } } diff --git a/tests/act-report.test.ts b/tests/act-report.test.ts index 80834d0d..b5913350 100644 --- a/tests/act-report.test.ts +++ b/tests/act-report.test.ts @@ -5,12 +5,14 @@ import { join } from 'node:path' import { journalPath } from '../src/act/journal.js' import { + autoRevertNoEffect, buildActReportJson, buildOptimizeAppliedHeader, captureBaseline, computeActReport, renderActReport, } from '../src/act/report.js' +import { formatAppliedFix, REPORT_MIN_AGE_DAYS } from '../src/act/types.js' import type { ActionRecord } from '../src/act/types.js' import type { WasteFinding } from '../src/optimize.js' import type { ClassifiedTurn, ProjectSummary } from '../src/types.js' @@ -762,3 +764,129 @@ describe('defer baseline capture', () => { expect(b).toBeUndefined() }) }) + +describe('applied-fix verdicts', () => { + const fixOf = async (records: ActionRecord[], projects: ProjectSummary[]) => { + const actionsDir = await writeJournal(records) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load(projects) }) + return { report, fixes: report.appliedFixes, actionsDir } + } + + it('calls a fix that realized its whole window estimate "worked"', async () => { + const { fixes } = await fixOf([mcpRecord()], [projectOf(sessionsAt(20, daysAgo(5)))]) + expect(fixes).toHaveLength(1) + expect(fixes[0]!.verdict).toBe('worked') + expect(fixes[0]!.estimatedTokens).toBe(40_000) + expect(fixes[0]!.realizedTokens).toBe(40_000) + expect(fixes[0]!.undoCommand).toBe('codeburn act undo a1') + }) + + it('holds the worked/partial boundary at the 70% ratio', async () => { + const rec = mcpRecord({ kind: 'mcp-project-scope' }) + // 14 of 20 sessions saved = exactly 70% of the window estimate. + const at70 = await fixOf( + [rec], + [projectOf([...sessionsAt(14, daysAgo(5)), ...sessionsAt(6, daysAgo(4), { mcpInventory: ['mcp__brave-search__search'] })])], + ) + expect(at70.fixes[0]!.verdict).toBe('worked') + + const below = await fixOf( + [rec], + [projectOf([...sessionsAt(13, daysAgo(5)), ...sessionsAt(7, daysAgo(4), { mcpInventory: ['mcp__brave-search__search'] })])], + ) + expect(below.fixes[0]!.verdict).toBe('partial') + expect(formatAppliedFix(below.fixes[0]!)).toContain('-35% vs estimate') + }) + + it('calls a fix that realized nothing "no-effect" and offers the undo', async () => { + const rec = mcpRecord({ kind: 'mcp-project-scope' }) + const stillLoading = sessionsAt(20, daysAgo(5), { mcpInventory: ['mcp__brave-search__search'] }) + const { fixes } = await fixOf([rec], [projectOf(stillLoading)]) + expect(fixes[0]!.verdict).toBe('no-effect') + expect(fixes[0]!.realizedTokens).toBe(0) + expect(formatAppliedFix(fixes[0]!)).toContain('did not help. Revert: codeburn act undo a1') + }) + + it('treats an estimate of zero as worked only when something was realized', async () => { + const zeroEstimate = { windowDays: 14, capturedAt: daysAgo(10), estimatedTokens: 0, sessions: 0, metrics: {} } + const { fixes } = await fixOf( + [mcpRecord({ baseline: { ...zeroEstimate, metrics: { 'brave-search': 2000 } } })], + [projectOf(sessionsAt(20, daysAgo(5)))], + ) + expect(fixes[0]!.estimatedTokens).toBe(40_000) + expect(fixes[0]!.verdict).toBe('worked') + }) + + it('leaves entries younger than the measurement window pending', async () => { + const { fixes } = await fixOf([mcpRecord({ at: daysAgo(1) })], [projectOf(sessionsAt(20, daysAgo(1)))]) + expect(fixes[0]!.verdict).toBe('pending') + expect(formatAppliedFix(fixes[0]!)).toBe(`unused-mcp (1d ago): measuring, check back after ${REPORT_MIN_AGE_DAYS} days`) + }) + + it('keeps a user-reverted entry out of the verdicts and carries its note', async () => { + const back = sessionsAt(20, daysAgo(5), { mcpInventory: ['mcp__brave-search__search'] }) + const { fixes } = await fixOf([mcpRecord()], [projectOf(back)]) + expect(fixes[0]!.verdict).toBe('pending') + expect(fixes[0]!.note).toMatch(/reverted by user/) + }) + + it('drops undone journal entries entirely', async () => { + const rec = mcpRecord() + const { fixes } = await fixOf( + [rec, { ...rec, status: 'undone', undoneAt: daysAgo(2) }], + [projectOf(sessionsAt(20, daysAgo(5)))], + ) + expect(fixes).toHaveLength(0) + }) + + it('never judges a correlation-only kind, so --auto-revert can never touch it', async () => { + const { fixes } = await fixOf([modelDefaultRecord()], [modelProject('app', '/tmp/app', 'candidate-model', 20, 19)]) + expect(fixes[0]!.verdict).toBe('pending') + }) +}) + +describe('autoRevertNoEffect', () => { + const noEffect = (over: Partial = {}) => ({ + ...mcpRecord({ kind: 'mcp-project-scope', ...over }), + }) + + it('undoes the no-effect entries and leaves the rest alone', async () => { + const worked = noEffect({ id: 'keep1', findingId: 'kept' }) + const actionsDir = await writeJournal([noEffect(), worked]) + const stillLoading = sessionsAt(20, daysAgo(5), { mcpInventory: ['mcp__brave-search__search'] }) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(stillLoading)]) }) + // Both are no-effect here; pin that only the ones we hand over get undone. + const target = report.appliedFixes.filter(f => f.id === 'a1') + const { lines, revertedIds } = await autoRevertNoEffect(target, { actionsDir }) + + expect([...revertedIds]).toEqual(['a1']) + expect(lines).toEqual(['Reverted a1: Remove an MCP server from config']) + const after = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(stillLoading)]) }) + expect(after.appliedFixes.map(f => f.id)).toEqual(['keep1']) + }) + + it('never auto-reverts a CLAUDE.md rule, it prints the undo command instead', async () => { + const rec = mcpRecord({ + id: 'cm1', + kind: 'claude-md-rule', + findingId: 'read-edit-ratio', + baseline: { windowDays: 14, capturedAt: daysAgo(10), estimatedTokens: 10_000, sessions: 20, metrics: { reads: 10, edits: 10 } }, + }) + const actionsDir = await writeJournal([rec]) + const sessions = sessionsAt(20, daysAgo(5), { toolBreakdown: { Edit: { calls: 10, tokens: 0 }, Read: { calls: 10, tokens: 0 } } as never }) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessions)]) }) + + expect(report.appliedFixes[0]!.verdict).toBe('no-effect') + const { lines, revertedIds } = await autoRevertNoEffect(report.appliedFixes, { actionsDir }) + expect(revertedIds.size).toBe(0) + expect(lines).toEqual(['Not auto-reverted: read-edit-ratio edits a CLAUDE.md. Revert: codeburn act undo cm1']) + }) + + it('ignores partial and pending entries', async () => { + const actionsDir = await writeJournal([mcpRecord({ at: daysAgo(1) })]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(1)))]) }) + const { lines, revertedIds } = await autoRevertNoEffect(report.appliedFixes, { actionsDir }) + expect(lines).toEqual([]) + expect(revertedIds.size).toBe(0) + }) +}) diff --git a/tests/optimize-apply.test.ts b/tests/optimize-apply.test.ts index 3a2569b2..2a69dbba 100644 --- a/tests/optimize-apply.test.ts +++ b/tests/optimize-apply.test.ts @@ -433,11 +433,19 @@ describe('runOptimizeApply end-to-end', () => { expect(out).toContain(`Applied ${shortId(rec.id)}`) expect(out).toContain(`Undo anytime: codeburn act undo ${shortId(rec.id)}`) } + expect(out).toContain('CodeBurn will re-measure these on your next optimize run after 3 days.') expect(JSON.parse(await readFile(join(fx.home, '.claude.json'), 'utf-8')).mcpServers).toEqual({}) expect(existsSync(join(fx.home, '.claude', 'skills', '.archived', 'foo'))).toBe(true) expect(existsSync(join(fx.home, '.zshrc'))).toBe(true) }) + it('does not promise a re-measure when nothing was applied', async () => { + const { fx, findings } = await threeFindingFixture() + const io = makeIo('q\n') + await runOptimizeApply([], undefined, applyOpts(fx, io, { findings })) + expect(io.stdout()).not.toContain('re-measure') + }) + it('interactive pick "2" applies only the second plan', async () => { const { fx, findings } = await threeFindingFixture() const io = makeIo('2\n') diff --git a/tests/optimize.test.ts b/tests/optimize.test.ts index c2ec97d9..e2179407 100644 --- a/tests/optimize.test.ts +++ b/tests/optimize.test.ts @@ -35,6 +35,7 @@ import { type OptimizeResult, } from '../src/optimize.js' import type { ProjectSummary } from '../src/types.js' +import type { AppliedFix } from '../src/act/types.js' function call(name: string, input: Record, sessionId = 's1', project = 'p1'): ToolCall { return { name, input, sessionId, project } @@ -1320,6 +1321,7 @@ describe('buildOptimizeJsonReport', () => { expect(classes.reduce((s, c) => s + c.tokensSaved, 0)).toBe(report.summary.potentialSavingsTokens) expect(classes.reduce((s, c) => s + c.savingsUSD, 0)).toBeCloseTo(report.summary.potentialSavingsCostUSD, 10) expect(classes.reduce((s, c) => s + c.count, 0)).toBe(report.summary.findingCount) + expect(report.appliedFixes).toEqual([]) expect(report.findings[0]).toMatchObject({ title: 'Trim stale context', severity: 'medium', @@ -1374,3 +1376,56 @@ describe('renderOptimize grouping', () => { expect(out).not.toContain('Estimates only.') }) }) + +describe('renderOptimize applied-fixes section', () => { + const plain = (s: string): string => s.replace(/\[[0-9;]*m/g, '') + + function fixture(over: Partial): AppliedFix { + return { + id: 'abcdef12', + kind: 'mcp-remove', + findingId: 'unused-mcp', + appliedAt: '2026-05-01T00:00:00.000Z', + ageDays: 4, + verdict: 'worked', + estimatedTokens: 300_000, + realizedTokens: 280_000, + note: '', + undoCommand: 'codeburn act undo abcdef12', + ...over, + } + } + + const findings: WasteFinding[] = [{ + id: 'bash-output-cap', + title: 'Cap bash output', + explanation: 'why', + impact: 'medium', + tokensSaved: 1000, + fix: { type: 'paste', destination: 'prompt', label: 'ask', text: 'ask' }, + }] + + const render = (appliedFixes: AppliedFix[], f = findings): string => + plain(renderOptimize(f, 0.00001, '7 Days', 10, 5, 100, 80, 'B', [], [], undefined, undefined, undefined, appliedFixes)) + + it('renders one line per verdict with its own glyph', () => { + const out = render([ + fixture({}), + fixture({ id: 'b', findingId: 'mcp-defer-threshold', verdict: 'partial', ageDays: 3, estimatedTokens: 600_000, realizedTokens: 420_000 }), + fixture({ id: 'c', findingId: 'bash-output-cap', verdict: 'no-effect', ageDays: 5, estimatedTokens: 41_000, realizedTokens: 0, undoCommand: 'codeburn act undo cccccccc' }), + fixture({ id: 'd', findingId: 'mcp-remove-linear', verdict: 'pending', ageDays: 1 }), + ]) + + expect(out).toContain('Applied fixes') + expect(out).toContain('\u2713 unused-mcp (4d ago): est. 300.0K -> measured 280.0K') + expect(out).toContain('~ mcp-defer-threshold (3d ago): est. 600.0K -> measured 420.0K (-30% vs estimate)') + expect(out).toContain('\u2717 bash-output-cap (5d ago): est. 41.0K -> measured 0 - did not help. Revert: codeburn act undo cccccccc') + expect(out).toContain('\u2026 mcp-remove-linear (1d ago): measuring, check back after 3 days') + }) + + it('shows the section on a clean setup too, and omits it when nothing is applied', () => { + expect(render([fixture({})], [])).toContain('Applied fixes') + expect(render([])).not.toContain('Applied fixes') + expect(render([], [])).not.toContain('Applied fixes') + }) +}) From 393628481143c75ef37bbda5e3f6e9279615158f Mon Sep 17 00:00:00 2001 From: iamtoruk Date: Tue, 18 Aug 2026 03:15:12 -0700 Subject: [PATCH 2/3] optimize: show the applied-fix verdicts in the TUI and desktop app Same section, compact: one line per still-applied fix with the verdict glyph, and the undo command for the ones that measured nothing. The app reads appliedFixes[] off the optimize JSON, tolerating its absence from an older CLI. --- app/renderer/lib/types.ts | 11 +++++++ app/renderer/sections/Optimize.test.tsx | 28 ++++++++++++++++ app/renderer/sections/Optimize.tsx | 44 +++++++++++++++++++++++++ app/renderer/styles/plain.css | 9 +++++ src/dashboard.tsx | 29 ++++++++++++++-- 5 files changed, 119 insertions(+), 2 deletions(-) diff --git a/app/renderer/lib/types.ts b/app/renderer/lib/types.ts index 8cc1fd0a..f1531ad8 100644 --- a/app/renderer/lib/types.ts +++ b/app/renderer/lib/types.ts @@ -437,6 +437,17 @@ export type OptimizeJsonReport = { basis: 'measured' | 'estimated' fix: WasteAction }> + /** Still-applied fixes, re-measured on every run. Absent on older CLIs. */ + appliedFixes?: Array<{ + id: string + kind: string + findingId: string | null + appliedAt: string + verdict: 'worked' | 'partial' | 'no-effect' | 'pending' + estimatedTokens: number + realizedTokens: number + undoCommand: string + }> } // ————— T1b: src/sharing/* (defined by the shared contract) ————— diff --git a/app/renderer/sections/Optimize.test.tsx b/app/renderer/sections/Optimize.test.tsx index a01ed0e7..443eec08 100644 --- a/app/renderer/sections/Optimize.test.tsx +++ b/app/renderer/sections/Optimize.test.tsx @@ -129,6 +129,34 @@ describe('Optimize', () => { Object.defineProperty(navigator, 'clipboard', { configurable: true, value: { writeText } }) }) + it('lists applied fixes with a glyph per verdict and the undo hint', async () => { + const report = makeOptimizeReport() + report.appliedFixes = [ + { id: 'a1', kind: 'archive-skill', findingId: 'unused-skills', appliedAt: '2026-07-06T00:00:00.000Z', verdict: 'worked', estimatedTokens: 300_000, realizedTokens: 280_000, undoCommand: 'codeburn act undo a1' }, + { id: 'b2', kind: 'defer-threshold', findingId: 'mcp-defer-threshold', appliedAt: '2026-07-07T00:00:00.000Z', verdict: 'partial', estimatedTokens: 600_000, realizedTokens: 420_000, undoCommand: 'codeburn act undo b2' }, + { id: 'c3', kind: 'shell-config', findingId: 'bash-output-cap', appliedAt: '2026-07-05T00:00:00.000Z', verdict: 'no-effect', estimatedTokens: 41_000, realizedTokens: 0, undoCommand: 'codeburn act undo c3' }, + { id: 'd4', kind: 'mcp-remove', findingId: null, appliedAt: '2026-07-09T00:00:00.000Z', verdict: 'pending', estimatedTokens: 0, realizedTokens: 0, undoCommand: 'codeburn act undo d4' }, + ] + getOptimizeReport.mockResolvedValue(report) + render() + + await screen.findByText('Applied fixes') + const rows = [...document.querySelectorAll('.opt-applied-row')] + expect(rows.map(r => r.className.split(' ')[1])).toEqual([ + 'opt-applied-worked', 'opt-applied-partial', 'opt-applied-no-effect', 'opt-applied-pending', + ]) + expect(rows[0]!.textContent).toContain('unused-skills') + expect(rows[0]!.textContent).toContain('est. 300K \u2192 280K') + expect(rows[3]!.textContent).toContain('mcp-remove') + expect(screen.getByText('codeburn act undo c3')).toBeTruthy() + }) + + it('omits the applied-fixes list when nothing is applied', async () => { + render() + await screen.findByText('Opus is doing your small talk') + expect(document.querySelector('.opt-applied')).toBeNull() + }) + it('groups Waste findings under the fix / habits / FYI headers in order', async () => { render() diff --git a/app/renderer/sections/Optimize.tsx b/app/renderer/sections/Optimize.tsx index 0d6ea911..ac9350c2 100644 --- a/app/renderer/sections/Optimize.tsx +++ b/app/renderer/sections/Optimize.tsx @@ -102,6 +102,50 @@ function WasteRows({ report }: { report: Polled }) { {report.data.summary.findingCount.toLocaleString('en-US')} findings · {formatUsd(report.data.summary.potentialSavingsCostUSD)} potential · health {report.data.summary.healthScore}/100 + + + ) +} + +type AppliedFix = NonNullable[number] + +const VERDICT_GLYPH: Record = { + worked: '\u2713', + partial: '~', + 'no-effect': '\u2717', + pending: '\u2026', +} + +const VERDICT_LABEL: Record = { + worked: 'worked', + partial: 'under estimate', + 'no-effect': 'did not help', + pending: 'measuring', +} + +// Closes the loop after `optimize --apply`: what each applied fix actually +// measured, and for the ones that did nothing, how to put them back. +function AppliedFixRows({ fixes }: { fixes: AppliedFix[] }) { + if (!fixes.length) return null + + return ( +
+
Applied fixes
+ {fixes.map(fix => ( +
+ + {fix.findingId ?? fix.kind} + {VERDICT_LABEL[fix.verdict]} + + {fix.verdict === 'pending' + ? '\u2014' + : `est. ${formatCompact(fix.estimatedTokens)} \u2192 ${formatCompact(fix.realizedTokens)}`} + +
+ ))} + {fixes.some(fix => fix.verdict === 'no-effect') && ( +
Revert one that did not help: {fixes.find(fix => fix.verdict === 'no-effect')!.undoCommand}
+ )}
) } diff --git a/app/renderer/styles/plain.css b/app/renderer/styles/plain.css index ad1c10ee..6d7a86c4 100644 --- a/app/renderer/styles/plain.css +++ b/app/renderer/styles/plain.css @@ -671,6 +671,15 @@ td:first-child { font-size: var(--fs-body); font-weight: var(--fw-body); } .opt-fix-code { max-width: 100%; overflow-x: auto; margin: 0; padding: 10px 11px; border: 1px solid var(--line); border-radius: 6px; background: var(--phead); color: var(--ink); font-family: var(--mono); font-size: 11px; line-height: 1.5; white-space: pre; } .opt-fix-command .opt-fix-code code::before { content: '$ '; color: var(--mut2); user-select: none; } .opt-copy { flex: 0 0 auto; padding: 4px 9px; border: 1px solid var(--line); border-radius: 6px; background: var(--panel); color: var(--mut); font: inherit; font-size: 10.5px; cursor: pointer; } +.opt-applied { padding-top: 12px; } +.opt-applied-row { display: grid; grid-template-columns: 16px minmax(0, 1fr) 110px 140px; align-items: center; column-gap: 12px; min-height: 34px; border-top: 1px solid var(--line2); } +.opt-applied-glyph { color: var(--mut2); font-family: var(--mono); font-size: 12px; } +.opt-applied-verdict { color: var(--mut); font-size: 10.5px; } +.opt-applied-worked .opt-applied-glyph, .opt-applied-worked .opt-applied-verdict { color: var(--ok); } +.opt-applied-partial .opt-applied-glyph, .opt-applied-partial .opt-applied-verdict { color: var(--warn); } +.opt-applied-no-effect .opt-applied-glyph, .opt-applied-no-effect .opt-applied-verdict { color: var(--bad); } +.opt-applied-hint { padding: 9px 0 0; } +.opt-applied-hint code { font-family: var(--mono); } .opt-copy:hover, .opt-copy:focus-visible { border-color: var(--accent); color: var(--ink); outline: none; } .ov-analytics-row { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 12px; align-items: stretch; } .ov-analytics-row > :only-child { grid-column: 1 / -1; } diff --git a/src/dashboard.tsx b/src/dashboard.tsx index 68ff3b1a..3a844784 100644 --- a/src/dashboard.tsx +++ b/src/dashboard.tsx @@ -11,6 +11,7 @@ import { aggregateModelTotals } from './model-breakdown.js' import { buildDurablePeriod } from './usage-aggregator.js' import { getAllProviders } from './providers/index.js' import { classHeaderLine, classTotals, findingBasis, findingClass, scanAndDetect, type FindingClass, type WasteFinding, type WasteAction, type OptimizeResult } from './optimize.js' +import { appliedFixGlyph, formatAppliedFix, type AppliedFix } from './act/types.js' import { aggregateFileChurn, buildCoachingNotes, computePricingCoverage, medianTimeToFirstEditMs, scanUserCorrections, worstOneShotCategory, type ReworkedFile } from './workflow-insights.js' import { estimateContextBudget, type ContextBudget } from './context-budget.js' import { dateKey } from './day-aggregator.js' @@ -1094,7 +1095,14 @@ const GRADE_COLORS: Record = { A: '#5BF5A0', B: '#5BF5A0', C: GO // off the alt-buffer top and the user couldn't see the StatusBar at all. const FINDINGS_WINDOW_SIZE = 3 -function OptimizeView({ findings, costRate, projects, label, width, healthScore, healthGrade, cursor }: { findings: WasteFinding[]; costRate: number; projects: ProjectSummary[]; label: string; width: number; healthScore: number; healthGrade: string; cursor: number }) { +const APPLIED_FIX_COLORS: Record = { + worked: '#5BF5A0', + partial: GOLD, + 'no-effect': '#F55B5B', + pending: DIM, +} + +function OptimizeView({ findings, costRate, projects, label, width, healthScore, healthGrade, cursor, appliedFixes = [] }: { findings: WasteFinding[]; costRate: number; projects: ProjectSummary[]; label: string; width: number; healthScore: number; healthGrade: string; cursor: number; appliedFixes?: AppliedFix[] }) { const periodCost = projects.reduce((s, p) => s + p.totalCostUSD, 0) const totalTokens = findings.reduce((s, f) => s + f.tokensSaved, 0) const totalCost = totalTokens * costRate @@ -1132,6 +1140,16 @@ function OptimizeView({ findings, costRate, projects, label, width, healthScore, ) })} + {appliedFixes.length > 0 && ( + + Applied fixes + {appliedFixes.map(fix => ( + + {appliedFixGlyph(fix)} {formatAppliedFix(fix)} + + ))} + + )} ) } @@ -1313,6 +1331,7 @@ export function InteractiveDashboard({ initialProjects, initialDailyHistoryProje const [detectedProviders, setDetectedProviders] = useState([]) const [view, setView] = useState('dashboard') const [optimizeResult, setOptimizeResult] = useState(null) + const [appliedFixes, setAppliedFixes] = useState([]) const [optimizeLoading, setOptimizeLoading] = useState(false) const [projectBudgets, setProjectBudgets] = useState>(new Map()) const [planUsages, setPlanUsages] = useState(initialPlanUsages ?? []) @@ -1473,6 +1492,12 @@ export function InteractiveDashboard({ initialProjects, initialDailyHistoryProje try { const result = await scanAndDetect(projects, currentRange(), activeProvider) if (reloadGenerationRef.current === generation) setOptimizeResult(result) + // Best effort: a bad journal never keeps the findings off screen. + try { + const { computeActReport } = await import('./act/report.js') + const applied = await computeActReport() + if (reloadGenerationRef.current === generation) setAppliedFixes(applied.appliedFixes) + } catch { /* the applied section is optional */ } } catch (error) { console.error(error) } finally { @@ -1637,7 +1662,7 @@ export function InteractiveDashboard({ initialProjects, initialDailyHistoryProje {view === 'compare' ? setView('dashboard')} /> : view === 'optimize' && optimizeResult - ? + ? : } {coachingNote && ( From f8bc9b2594d6ba4374bd482dece4cc735512888d Mon Sep 17 00:00:00 2001 From: iamtoruk Date: Tue, 18 Aug 2026 03:15:12 -0700 Subject: [PATCH 3/3] docs: cover what happens after optimize --apply docs/optimize.md gains an "After you apply" section (the four verdicts, --auto-revert, what is never auto-reverted); README and CHANGELOG follow. --- CHANGELOG.md | 1 + README.md | 3 ++- docs/optimize.md | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a53889d8..0773af1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Added +- **Applied fixes get re-measured on every `optimize` run, and told plainly whether they worked.** After `codeburn optimize --apply`, every still-applied fix comes back in an `Applied fixes` section on subsequent `codeburn optimize` runs, carrying the verdict `act report` already computes from the same reconciliation: `worked` (at least 70% of its window-scaled estimate realized), `partial` (something, but under that), `no-effect` (no measured reduction, printed with the exact `codeburn act undo ` that puts it back), or `measuring` for anything younger than the 3-day measurement window. The numbers are measured — provider-counted usage over the post-apply window — not re-estimated. `--apply` now says when the re-measure will happen, `--format json` gains `appliedFixes[]` (add-only), and the same section appears in the dashboard TUI and the desktop app. New `codeburn optimize --auto-revert` undoes the fixes that measured no reduction at all through the same code path as `codeburn act undo`; it never touches `partial` or still-measuring fixes, and never auto-reverts a `CLAUDE.md` rule (it prints the undo command instead), matching the `--yes` guardrail. - **Optimize findings say what to do with them and where their number came from.** Every finding now carries a class and a basis, and every surface groups by it: `Fix now (apply-able)` for findings `codeburn optimize --apply` can write itself, `Habits` for the behavioural ones, `FYI` for informational ones whose cost may be justified. A finding only counts as apply-able when a plan can actually be built for that instance, so an `mcp-deferral-off` caused by Vertex policy or a shell-profile override is grouped as a habit rather than promising a fix that does not exist. Alongside it, each finding is marked `measured` (summed from provider-counted usage) or `estimated` (a schema-size or recovery-fraction model), with the split reported in the header as `N measured · M estimated` in place of the blanket "Estimates only." footer. Sessions whose cost the provider never reported are kept out of the `cost-outliers` peer comparison, and a provider that only ever estimates gets the finding marked `estimated` rather than dropped. `--format json` gains `class` and `basis` per finding plus `summary.measuredSavingsUSD` (existing fields unchanged), and the new `docs/optimize.md` covers what is scanned, exactly what `--apply` may write, and how to read the health grade. ### Added (CLI) diff --git a/README.md b/README.md index 47a1166b..18449284 100644 --- a/README.md +++ b/README.md @@ -186,11 +186,12 @@ codeburn optimize --apply --yes # apply every appliable fix without prompt codeburn act list # every change CodeBurn has made codeburn act undo --last # roll the most recent change back codeburn act report # realized vs estimated savings +codeburn optimize --auto-revert # undo the applied fixes that measured no reduction ``` `codeburn optimize` finds the waste; `--apply` fixes the config-class findings for you: settings values, environment variables, archiving unused agents and skills. Every change is backed up and journaled before it lands. `codeburn act list` shows the history and `codeburn act undo ` restores the original files (it refuses if the files changed since being applied, unless you pass `--force`). -The loop closes on honesty: once an applied fix is at least 3 days old, `codeburn act report` compares its estimated savings against what your sessions actually did, and later `codeburn optimize` runs show that realized figure in the header. Estimates get checked against reality, not just claimed. +The loop closes on honesty: once an applied fix is at least 3 days old, `codeburn act report` compares its estimated savings against what your sessions actually did, and every later `codeburn optimize` run lists it under `Applied fixes` with a plain verdict — worked, under its estimate, or did not help, with the undo command for that last case. `--auto-revert` undoes the ones that did nothing (never `CLAUDE.md` rules). Estimates get checked against reality, not just claimed. ## Guard your budget diff --git a/docs/optimize.md b/docs/optimize.md index 87e59d5b..a4f9ac21 100644 --- a/docs/optimize.md +++ b/docs/optimize.md @@ -67,6 +67,40 @@ or name it explicitly: codeburn optimize --apply --only read-edit-ratio ``` +## After you apply + +Applying a fix is a claim, so CodeBurn checks it. Every `codeburn optimize` run re-measures the +fixes still in place and prints them under `Applied fixes`, one line each: + +| Line | Verdict | Meaning | +|---|---|---| +| `✓ unused-skills (7d ago): est. 300.0K -> measured 280.0K` | worked | at least 70% of the estimate showed up in your sessions | +| `~ mcp-defer-threshold (5d ago): est. 600.0K -> measured 420.0K (-30% vs estimate)` | partial | it helped, but under its estimate | +| `✗ bash-output-cap (6d ago): est. 41.0K -> measured 0 - did not help. Revert: codeburn act undo 3f2a1c04` | no-effect | no measured reduction at all | +| `… mcp-remove (1d ago): measuring, check back after 3 days` | measuring | too young, or the change has not taken effect in a session yet | + +The estimate shown is the at-apply estimate scaled to the measured window, so the two numbers are +comparable. Both come from the same reconciliation `codeburn act report` prints — there is one set of +numbers, not two — and they are **measured**: provider-counted usage over the post-apply window. +Anything that cannot be measured (no baseline captured, a fix you reverted by hand, a +correlation-only kind like `guard-install`) stays on the `measuring` line with the reason, never a +claimed saving. + +`--format json` carries the same list as `appliedFixes[]`, and the section appears in the dashboard +TUI and the desktop app. + +### `--auto-revert` + +```bash +codeburn optimize --auto-revert +``` + +Off by default. It undoes exactly the fixes whose verdict is `no-effect`, through the same code path +as `codeburn act undo` (backups restored, drift check applied, the revert journaled). It never +touches a `partial` or still-measuring fix, and it never auto-reverts a `claude-md-rule` — those land +in whatever project directory you were in, the same reason `--yes` skips them, so it prints the undo +command and leaves the file alone. + ## measured vs estimated Each finding also carries a `basis`, printed next to its savings and summarised in the header as