From ad471f3d8c4bb67136f0d66225d5e043acf67acb Mon Sep 17 00:00:00 2001 From: AgentSeal Date: Fri, 3 Jul 2026 13:04:37 +0200 Subject: [PATCH] feat(act): realized savings measurement (codeburn act report) (#606) Capture a trailing-14-day before-baseline when a fix is applied and re-measure it against the post-apply window so optimize can show realized numbers next to estimates. - ActionBaseline (windowDays, capturedAt, estimatedTokens, sessions, metrics) persisted by runAction; captured in the optimize --apply flow and at guard install time. - codeburn act report [--json]: applied, not-undone actions older than 3 days, re-running the detectors over apply-date-to-now (capped 30 days). Per-kind realized deltas: MCP/archive tokens-per-session times saved sessions with reverted-by-user detection; read-edit deficit reduction; guard yield split labeled correlation. Bash cap is marked not measurable (result sizes are not retained). Low confidence under 20 post-window sessions or past a 2x volume shift. Realized numbers rounded down, estimate kept visible. - optimize gains one header line only when a measured action exists, and appends "(previously applied , re-flagged)" to re-triggered findings. No change for users with no applied actions. Reuses scanAndDetect helpers over a date-bounded range; exports the token constants and read/edit tool sets rather than duplicating the math. --- src/act/cli.ts | 19 ++ src/act/optimize-apply.ts | 9 + src/act/report.ts | 537 ++++++++++++++++++++++++++++++++++++++ src/act/types.ts | 25 +- src/guard/cli.ts | 8 + src/main.ts | 15 +- src/optimize.ts | 32 ++- tests/act-report.test.ts | 303 +++++++++++++++++++++ 8 files changed, 932 insertions(+), 16 deletions(-) create mode 100644 src/act/report.ts create mode 100644 tests/act-report.test.ts diff --git a/src/act/cli.ts b/src/act/cli.ts index 2481525..15bf2f7 100644 --- a/src/act/cli.ts +++ b/src/act/cli.ts @@ -2,6 +2,7 @@ import type { Command } from 'commander' import { renderTable } from '../text-table.js' import { defaultActionsDir, readRecords, shortId } from './journal.js' import { DriftError, undoAction } from './undo.js' +import { buildActReportJson, computeActReport, renderActReport } from './report.js' function formatWhen(at: string): string { return at.replace('T', ' ').slice(0, 16) @@ -63,4 +64,22 @@ export function registerActCommands(program: Command): void { process.exitCode = 1 } }) + + act + .command('report') + .description('Realized vs estimated savings for applied actions older than 3 days') + .option('--json', 'Output the realized report as JSON') + .action(async (opts: { json?: boolean }) => { + try { + const report = await computeActReport() + if (opts.json) { + console.log(JSON.stringify(buildActReportJson(report), null, 2)) + return + } + console.log(renderActReport(report)) + } catch (err) { + console.error(err instanceof Error ? err.message : String(err)) + process.exitCode = 1 + } + }) } diff --git a/src/act/optimize-apply.ts b/src/act/optimize-apply.ts index 462f04f..5b90685 100644 --- a/src/act/optimize-apply.ts +++ b/src/act/optimize-apply.ts @@ -162,6 +162,15 @@ export async function runOptimizeApply( return } + // Stamp a trailing-14-day before-baseline onto each plan so runAction + // persists it and `act report` can measure realized savings later. Best + // effort: a scan failure leaves the baseline absent (reported "not + // measurable"), never blocking the apply. + try { + const { captureBaselinesForPlans } = await import('./report.js') + await captureBaselinesForPlans(selected) + } catch { /* baseline is optional; apply proceeds without it */ } + print() for (const fp of selected) { try { diff --git a/src/act/report.ts b/src/act/report.ts new file mode 100644 index 0000000..400c010 --- /dev/null +++ b/src/act/report.ts @@ -0,0 +1,537 @@ +import { existsSync } from 'fs' +import type { DateRange, ProjectSummary, SessionSummary } from '../types.js' +import type { ActionBaseline, ActionKind, ActionRecord } from './types.js' +import type { FindingPlan } from './plans.js' +import { + AVG_TOKENS_PER_READ, + HEALTHY_READ_EDIT_RATIO, + TOKENS_PER_MCP_TOOL, + TOOLS_PER_MCP_SERVER, + TOKENS_PER_SKILL_DEF, + TOKENS_PER_AGENT_DEF, + TOKENS_PER_COMMAND_DEF, + READ_TOOL_NAMES, + EDIT_TOOL_NAMES, + aggregateMcpCoverage, + computeInputCostRate, + type McpServerCoverage, + type WasteFinding, +} from '../optimize.js' +import { parseAllSessions } from '../parser.js' +import { computeYield, type YieldSummary } from '../yield.js' +import { defaultActionsDir, readRecords } from './journal.js' +import { renderTable } from '../text-table.js' +import { formatTokens } from '../format.js' +import { formatCost } from '../currency.js' + +const DAY_MS = 24 * 60 * 60 * 1000 +const WINDOW_CAP_DAYS = 30 +const BASELINE_WINDOW_DAYS = 14 +const REPORT_MIN_AGE_DAYS = 3 +const MIN_POST_WINDOW_SESSIONS = 20 +const VOLUME_SHIFT_FACTOR = 2 + +// Encode the epic's honest-accounting rules where they are seen: each kind +// measures only its own metric, guard is correlation, and realized figures are +// rounded down with the estimate kept visible. +const HONEST_FOOTER = + 'Each fix measures only its own metric; effects are never attributed across signals. ' + + 'Guard rows are correlation, not attribution. Realized numbers are rounded down; the estimate stays visible.' + +const MCP_KINDS = new Set(['mcp-remove', 'mcp-project-scope']) +const ARCHIVE_DEF_TOKENS: Partial> = { + 'archive-skill': TOKENS_PER_SKILL_DEF, + 'archive-agent': TOKENS_PER_AGENT_DEF, + 'archive-command': TOKENS_PER_COMMAND_DEF, +} + +export type RealizedStatus = 'measured' | 'reverted' | 'not-measurable' + +export type ActReportRow = { + id: string + appliedAt: string + date: string + kind: ActionKind + description: string + estimatedTokens: number + realizedTokens: number + status: RealizedStatus + confidence: 'low' | 'normal' + note: string + // guard-install only: yield split then vs now, labeled correlation. + correlation?: { + abandonedPctThen: number + abandonedPctNow: number + avgSessionCostThenUSD: number + avgSessionCostNowUSD: number + } +} + +export type ActReport = { + generatedAt: string + windowCapDays: number + costRate: number + rows: ActReportRow[] + totalRealizedTokens: number + totalRealizedCostUSD: number + measuredCount: number + activeCount: number + observedDays: number + // findingId -> earliest apply date of an active applied action; drives the + // optimize "(previously applied ..., re-flagged)" title suffix. + appliedByFinding: Record +} + +export type ActReportOptions = { + actionsDir?: string + now?: Date + cwd?: string + loadProjects?: (range: DateRange) => Promise + computeYield?: (range: DateRange) => Promise +} + +// --------------------------------------------------------------------------- +// Shared measurement helpers +// --------------------------------------------------------------------------- + +function ageDays(iso: string, now: Date): number { + return (now.getTime() - new Date(iso).getTime()) / DAY_MS +} + +function allSessions(projects: ProjectSummary[]): SessionSummary[] { + return projects.flatMap(p => p.sessions) +} + +function sessionsInWindow(projects: ProjectSummary[], start: Date, end: Date): SessionSummary[] { + const out: SessionSummary[] = [] + for (const p of projects) { + for (const s of p.sessions) { + if (!s.firstTimestamp) continue + const t = new Date(s.firstTimestamp).getTime() + if (t >= start.getTime() && t <= end.getTime()) out.push(s) + } + } + return out +} + +function countToolCalls(sessions: SessionSummary[], names: ReadonlySet): number { + let n = 0 + for (const s of sessions) { + for (const [tool, data] of Object.entries(s.toolBreakdown)) { + if (names.has(tool)) n += data.calls + } + } + return n +} + +function countBashCalls(sessions: SessionSummary[]): number { + let n = 0 + for (const s of sessions) { + for (const data of Object.values(s.bashBreakdown)) n += data.calls + } + return n +} + +function sessionLoadsAny(s: SessionSummary, servers: string[]): boolean { + for (const fqn of s.mcpInventory ?? []) { + const seg = fqn.split('__')[1] + if (seg && servers.includes(seg)) return true + } + for (const server of Object.keys(s.mcpBreakdown)) { + if (servers.includes(server)) return true + } + return false +} + +function countSessionsLoading(projects: ProjectSummary[], servers: string[]): number { + return allSessions(projects).filter(s => sessionLoadsAny(s, servers)).length +} + +// A kind whose realized effect is a token saving (everything except guard, +// which is a dollars/yield correlation, and out-of-scope kinds). +function isTokenKind(kind: ActionKind): boolean { + return kind !== 'guard-install' && kind !== 'guard-uninstall' && kind !== 'model-default' +} + +function confidenceFor(afterSessions: number, baseline: ActionBaseline, afterStart: Date, now: Date): 'low' | 'normal' { + if (afterSessions < MIN_POST_WINDOW_SESSIONS) return 'low' + if (baseline.sessions > 0 && baseline.windowDays > 0) { + const afterDays = Math.max((now.getTime() - afterStart.getTime()) / DAY_MS, 1) + const shift = (afterSessions / afterDays) / (baseline.sessions / baseline.windowDays) + if (shift > VOLUME_SHIFT_FACTOR || shift < 1 / VOLUME_SHIFT_FACTOR) return 'low' + } + return 'normal' +} + +// --------------------------------------------------------------------------- +// Per-kind realized deltas +// --------------------------------------------------------------------------- + +function mcpRow( + base: ActReportRow, rec: ActionRecord, sessions: SessionSummary[], + baseline: ActionBaseline, afterStart: Date, now: Date, +): ActReportRow { + const servers = Object.keys(baseline.metrics) + const perSessionTokens = Object.values(baseline.metrics).reduce((a, b) => a + b, 0) + if (servers.length === 0 || perSessionTokens === 0) return { ...base, note: 'not measurable: empty baseline' } + const stillLoading = sessions.filter(s => sessionLoadsAny(s, servers)).length + const confidence = confidenceFor(sessions.length, baseline, afterStart, now) + if (rec.kind === 'mcp-remove' && stillLoading > 0) { + return { + ...base, + status: 'reverted', + confidence, + note: `reverted by user: ${servers.join(', ')} loaded again in ${stillLoading} post-apply session${stillLoading === 1 ? '' : 's'}`, + } + } + const savedSessions = Math.max(0, sessions.length - stillLoading) + return { ...base, status: 'measured', realizedTokens: Math.floor(perSessionTokens * savedSessions), confidence } +} + +function archiveRow( + base: ActReportRow, rec: ActionRecord, sessions: SessionSummary[], + baseline: ActionBaseline, afterStart: Date, now: Date, +): ActReportRow { + const perSessionTokens = Object.values(baseline.metrics).reduce((a, b) => a + b, 0) + if (perSessionTokens === 0) return { ...base, note: 'not measurable: empty baseline' } + const confidence = confidenceFor(sessions.length, baseline, afterStart, now) + const restored = rec.changes.some(c => c.op === 'move' && existsSync(c.path)) + if (restored) { + return { ...base, status: 'reverted', confidence, note: 'reverted by user: an archived item was moved back into place' } + } + return { ...base, status: 'measured', realizedTokens: Math.floor(perSessionTokens * sessions.length), confidence } +} + +function readEditRow( + base: ActReportRow, sessions: SessionSummary[], + baseline: ActionBaseline, afterStart: Date, now: Date, +): ActReportRow { + const editsNow = countToolCalls(sessions, EDIT_TOOL_NAMES) + const readsNow = countToolCalls(sessions, READ_TOOL_NAMES) + const editsThen = baseline.metrics['edits'] ?? 0 + const readsThen = baseline.metrics['reads'] ?? 0 + if (editsThen <= 0 || editsNow <= 0) return { ...base, note: 'not measurable: not enough edit activity to compare' } + const ratioThen = readsThen / editsThen + const ratioNow = readsNow / editsNow + // Detector estimate math: reads short of HEALTHY_READ_EDIT_RATIO per edit are + // the retry-prone deficit. Credit only the reduction in that deficit, scaled + // by current edits; a worsened ratio claims nothing. + const deficitThen = Math.max(HEALTHY_READ_EDIT_RATIO - ratioThen, 0) + const deficitNow = Math.max(HEALTHY_READ_EDIT_RATIO - ratioNow, 0) + const realized = Math.floor(Math.max(0, deficitThen - deficitNow) * editsNow * AVG_TOKENS_PER_READ) + return { + ...base, + status: 'measured', + realizedTokens: realized, + confidence: confidenceFor(sessions.length, baseline, afterStart, now), + note: `read:edit ${ratioThen.toFixed(1)}:1 -> ${ratioNow.toFixed(1)}:1`, + } +} + +async function guardRow( + base: ActReportRow, afterStart: Date, now: Date, + baseline: ActionBaseline, opts: ActReportOptions, +): Promise { + const abandonedThen = baseline.metrics['abandonedPct'] + const avgThen = baseline.metrics['avgSessionCostUSD'] + if (abandonedThen === undefined || avgThen === undefined) { + return { ...base, note: 'not measurable: no yield baseline captured at install time' } + } + const yieldFn = opts.computeYield ?? ((range: DateRange) => computeYield(range, opts.cwd ?? process.cwd())) + let summary: YieldSummary + try { + summary = await yieldFn({ start: afterStart, end: now }) + } catch { + return { ...base, note: 'not measurable: yield could not be computed for the post-apply window' } + } + const abandonedNow = summary.total.cost > 0 ? Math.round((summary.abandoned.cost / summary.total.cost) * 100) : 0 + const avgNow = summary.total.sessions > 0 ? summary.total.cost / summary.total.sessions : 0 + return { + ...base, + status: 'measured', + confidence: summary.total.sessions < MIN_POST_WINDOW_SESSIONS ? 'low' : 'normal', + note: 'correlation, not attribution', + correlation: { + abandonedPctThen: abandonedThen, + abandonedPctNow: abandonedNow, + avgSessionCostThenUSD: avgThen, + avgSessionCostNowUSD: avgNow, + }, + } +} + +async function computeRow(rec: ActionRecord, sessions: SessionSummary[], afterStart: Date, now: Date, opts: ActReportOptions): Promise { + const base: ActReportRow = { + id: rec.id, + appliedAt: rec.at, + date: rec.at.slice(0, 10), + kind: rec.kind, + description: rec.description, + estimatedTokens: rec.baseline?.estimatedTokens ?? 0, + realizedTokens: 0, + status: 'not-measurable', + confidence: 'normal', + note: '', + } + const baseline = rec.baseline + if (!baseline) return { ...base, note: 'not measurable: no baseline captured at apply time' } + + if (MCP_KINDS.has(rec.kind)) return mcpRow(base, rec, sessions, baseline, afterStart, now) + if (rec.kind in ARCHIVE_DEF_TOKENS) return archiveRow(base, rec, sessions, baseline, afterStart, now) + if (rec.kind === 'claude-md-rule') return readEditRow(base, sessions, baseline, afterStart, now) + if (rec.kind === 'shell-config') return { ...base, note: 'not measurable: bash result token sizes are not retained in the summary' } + if (rec.kind === 'guard-install') return guardRow(base, afterStart, now, baseline, opts) + return { ...base, note: 'not measurable: kind is not tracked by act report' } +} + +// --------------------------------------------------------------------------- +// Report +// --------------------------------------------------------------------------- + +export async function computeActReport(opts: ActReportOptions = {}): Promise { + const now = opts.now ?? new Date() + const records = await readRecords(opts.actionsDir ?? defaultActionsDir()) + const active = records.filter(r => r.status === 'applied') + + const appliedByFinding: Record = {} + for (const r of active) { + if (!r.findingId) continue + const date = r.at.slice(0, 10) + const prev = appliedByFinding[r.findingId] + if (!prev || date < prev) appliedByFinding[r.findingId] = date + } + + const empty: ActReport = { + generatedAt: now.toISOString(), + windowCapDays: WINDOW_CAP_DAYS, + costRate: 0, + rows: [], + totalRealizedTokens: 0, + totalRealizedCostUSD: 0, + measuredCount: 0, + activeCount: active.length, + observedDays: 0, + appliedByFinding, + } + + const eligible = active.filter(r => ageDays(r.at, now) > REPORT_MIN_AGE_DAYS) + if (eligible.length === 0) return empty + + const windowStart = new Date(now.getTime() - WINDOW_CAP_DAYS * DAY_MS) + const loadProjects = opts.loadProjects ?? ((range: DateRange) => parseAllSessions(range, 'claude')) + const projects = await loadProjects({ start: windowStart, end: now }) + const costRate = computeInputCostRate(projects) + + const rows: ActReportRow[] = [] + for (const rec of eligible) { + const afterStart = new Date(Math.max(new Date(rec.at).getTime(), windowStart.getTime())) + rows.push(await computeRow(rec, sessionsInWindow(projects, afterStart, now), afterStart, now, opts)) + } + + const measuredRows = rows.filter(r => r.status === 'measured' && isTokenKind(r.kind)) + const totalRealizedTokens = measuredRows.reduce((s, r) => s + r.realizedTokens, 0) + const observedDays = Math.min( + WINDOW_CAP_DAYS, + measuredRows.reduce((mx, r) => Math.max(mx, Math.ceil(ageDays(r.appliedAt, now))), 0), + ) + + return { + generatedAt: now.toISOString(), + windowCapDays: WINDOW_CAP_DAYS, + costRate, + rows, + totalRealizedTokens, + totalRealizedCostUSD: totalRealizedTokens * costRate, + measuredCount: measuredRows.length, + activeCount: active.length, + observedDays, + appliedByFinding, + } +} + +export function buildOptimizeAppliedHeader(report: ActReport): string | null { + if (report.measuredCount === 0) return null + const cost = report.costRate > 0 ? ` (~${formatCost(report.totalRealizedCostUSD)})` : '' + const days = report.observedDays + return `Applied fixes: ${report.activeCount} active, realized ~${formatTokens(report.totalRealizedTokens)} tokens${cost} over ${days} day${days === 1 ? '' : 's'}. Details: codeburn act report` +} + +function realizedCell(r: ActReportRow): string { + if (r.status === 'reverted') return 'reverted' + if (r.status === 'not-measurable') return 'not measurable' + if (r.correlation) return `abandoned ${r.correlation.abandonedPctThen}% -> ${r.correlation.abandonedPctNow}% (corr.)` + return formatTokens(r.realizedTokens) +} + +export function renderActReport(report: ActReport): string { + if (report.rows.length === 0) { + const lines = ['', ' No applied actions to measure yet.'] + if (report.activeCount > 0) { + lines.push(` ${report.activeCount} action${report.activeCount === 1 ? '' : 's'} applied; measurement starts after ${REPORT_MIN_AGE_DAYS} days.`) + } else { + lines.push(' Apply fixes with codeburn optimize --apply, then check back after a few days.') + } + lines.push('') + return lines.join('\n') + } + + const rows = report.rows.map(r => [ + r.date, + r.description, + r.estimatedTokens > 0 ? formatTokens(r.estimatedTokens) : '-', + realizedCell(r), + r.status === 'measured' && isTokenKind(r.kind) ? r.confidence : '-', + ]) + const totalCost = report.costRate > 0 ? ` (~${formatCost(report.totalRealizedCostUSD)})` : '' + rows.push(['', 'Total realized', '', `${formatTokens(report.totalRealizedTokens)}${totalCost}`, '']) + + const table = renderTable( + [{ header: 'Applied' }, { header: 'Action' }, { header: 'Estimated', right: true }, { header: 'Realized', right: true }, { header: 'Confidence' }], + rows, + { boldRows: new Set([rows.length - 1]) }, + ) + + const details: string[] = [] + for (const r of report.rows) { + if (r.status === 'measured' && isTokenKind(r.kind)) continue + if (r.note) details.push(` ${r.date} ${r.kind}: ${r.note}`) + if (r.correlation) { + details.push(` avg session cost ${formatCost(r.correlation.avgSessionCostThenUSD)} -> ${formatCost(r.correlation.avgSessionCostNowUSD)}`) + } + } + + return ['', table, ...(details.length > 0 ? ['', ...details] : []), '', ' ' + HONEST_FOOTER, ''].join('\n') +} + +export function buildActReportJson(report: ActReport): unknown { + return { + generatedAt: report.generatedAt, + windowCapDays: report.windowCapDays, + actions: report.rows.map(r => { + const tokenMeasured = r.status === 'measured' && isTokenKind(r.kind) + return { + id: r.id, + date: r.date, + kind: r.kind, + description: r.description, + estimatedTokens: r.estimatedTokens, + realizedTokens: tokenMeasured ? r.realizedTokens : null, + status: r.status, + confidence: tokenMeasured ? r.confidence : null, + note: r.note, + ...(r.correlation ? { correlation: r.correlation } : {}), + } + }), + totals: { + realizedTokens: report.totalRealizedTokens, + realizedCostUSD: report.totalRealizedCostUSD, + measuredActions: report.measuredCount, + activeActions: report.activeCount, + observedDays: report.observedDays, + }, + footer: HONEST_FOOTER, + } +} + +// --------------------------------------------------------------------------- +// Baseline capture (apply time) +// --------------------------------------------------------------------------- + +type CaptureCtx = { + projects: ProjectSummary[] + coverage: McpServerCoverage[] + windowDays: number + now: Date +} + +function mcpServersFromApply(finding: WasteFinding): string[] { + if (finding.apply?.kind === 'mcp-remove') return finding.apply.servers + if (finding.apply?.kind === 'mcp-project-scope') return finding.apply.servers.map(s => s.server) + return [] +} + +function needsConfigBaseline(kind: ActionKind): boolean { + return MCP_KINDS.has(kind) || kind in ARCHIVE_DEF_TOKENS || kind === 'claude-md-rule' || kind === 'shell-config' +} + +export function captureBaseline(finding: WasteFinding, kind: ActionKind, ctx: CaptureCtx): ActionBaseline | undefined { + const common = { + windowDays: ctx.windowDays, + capturedAt: ctx.now.toISOString(), + estimatedTokens: Math.max(0, Math.round(finding.tokensSaved)), + } + + if (MCP_KINDS.has(kind)) { + const servers = mcpServersFromApply(finding) + if (servers.length === 0) return undefined + const covByServer = new Map(ctx.coverage.map(c => [c.server, c])) + const metrics: Record = {} + for (const server of servers) { + const cov = covByServer.get(server) + const tools = cov && cov.toolsAvailable > 0 ? cov.toolsAvailable : TOOLS_PER_MCP_SERVER + metrics[server] = tools * TOKENS_PER_MCP_TOOL + } + return { ...common, sessions: countSessionsLoading(ctx.projects, servers), metrics } + } + + const defTokens = ARCHIVE_DEF_TOKENS[kind] + if (defTokens !== undefined) { + const names = finding.apply?.kind === 'archive' ? finding.apply.names : [] + if (names.length === 0) return undefined + const metrics: Record = {} + for (const name of names) metrics[name] = defTokens + return { ...common, sessions: allSessions(ctx.projects).length, metrics } + } + + const sessions = allSessions(ctx.projects) + if (kind === 'claude-md-rule') { + return { ...common, sessions: sessions.length, metrics: { reads: countToolCalls(sessions, READ_TOOL_NAMES), edits: countToolCalls(sessions, EDIT_TOOL_NAMES) } } + } + if (kind === 'shell-config') { + return { ...common, sessions: sessions.length, metrics: { calls: countBashCalls(sessions) } } + } + return undefined +} + +// Scan the trailing 14 days once and stamp a baseline onto every appliable +// plan that carries one, so runAction persists it for `act report` to diff. +export async function captureBaselinesForPlans( + plans: FindingPlan[], + opts: { now?: Date; loadProjects?: (range: DateRange) => Promise } = {}, +): Promise { + const applicable = plans.filter(fp => fp.plan && needsConfigBaseline(fp.plan.kind)) + if (applicable.length === 0) return + const now = opts.now ?? new Date() + const start = new Date(now.getTime() - BASELINE_WINDOW_DAYS * DAY_MS) + const loadProjects = opts.loadProjects ?? ((range: DateRange) => parseAllSessions(range, 'claude')) + const projects = await loadProjects({ start, end: now }) + const ctx: CaptureCtx = { projects, coverage: aggregateMcpCoverage(projects), windowDays: BASELINE_WINDOW_DAYS, now } + for (const fp of applicable) { + const baseline = captureBaseline(fp.finding, fp.plan!.kind, ctx) + if (baseline) fp.plan!.baseline = baseline + } +} + +export async function captureGuardBaseline( + opts: { now?: Date; cwd?: string; computeYield?: (range: DateRange) => Promise } = {}, +): Promise { + const now = opts.now ?? new Date() + const range = { start: new Date(now.getTime() - BASELINE_WINDOW_DAYS * DAY_MS), end: now } + const yieldFn = opts.computeYield ?? ((r: DateRange) => computeYield(r, opts.cwd ?? process.cwd())) + let summary: YieldSummary + try { + summary = await yieldFn(range) + } catch { + return undefined + } + return { + windowDays: BASELINE_WINDOW_DAYS, + capturedAt: now.toISOString(), + estimatedTokens: 0, + sessions: summary.total.sessions, + metrics: { + abandonedPct: summary.total.cost > 0 ? Math.round((summary.abandoned.cost / summary.total.cost) * 100) : 0, + avgSessionCostUSD: summary.total.sessions > 0 ? summary.total.cost / summary.total.sessions : 0, + }, + } +} diff --git a/src/act/types.ts b/src/act/types.ts index 4e8eff9..1f0baf6 100644 --- a/src/act/types.ts +++ b/src/act/types.ts @@ -14,6 +14,27 @@ export type FileChange = { afterHash: string // sha256 of the post-apply bytes, checked for drift on undo } +// Before/after measurement captured when an action is applied, diffed against +// the post-apply window by `act report`. `metrics` holds the kind-specific +// numbers: +// mcp-remove / mcp-project-scope: server name -> schema tokens per session +// archive-skill|agent|command: item name -> definition tokens per session +// claude-md-rule (read/edit rule): { reads, edits } +// shell-config (bash cap): { calls } +// guard-install: { abandonedPct, avgSessionCostUSD } +// estimatedTokens is the finding's estimate at apply time (0 for guard, which +// is a correlation signal, not a token estimate). sessions is the affected-scope +// session count over the window, kept out of `metrics` so it can never collide +// with a server literally named "sessions"; it feeds only the volume-shift +// confidence check. +export type ActionBaseline = { + windowDays: number + capturedAt: string + estimatedTokens: number + sessions: number + metrics: Record +} + export type ActionRecord = { id: string // crypto.randomUUID() at: string // ISO timestamp @@ -23,7 +44,7 @@ export type ActionRecord = { changes: FileChange[] status: 'applied' | 'undone' undoneAt?: string - baseline?: Record + baseline?: ActionBaseline } // expectedHash: sha256 of the raw on-disk bytes the plan's content was @@ -41,5 +62,5 @@ export type ActionPlan = { description: string findingId?: string | null changes: PlannedChange[] - baseline?: Record + baseline?: ActionBaseline } diff --git a/src/guard/cli.ts b/src/guard/cli.ts index 0cc3ca7..4ac49ff 100644 --- a/src/guard/cli.ts +++ b/src/guard/cli.ts @@ -41,6 +41,14 @@ async function doInstall(scope: Scope, statusline: boolean): Promise { const built = buildInstall(path, { statusline }) for (const note of built.notes) console.log(chalk.yellow(` ! ${note}`)) if (built.plan) { + // Capture a trailing-14-day yield baseline so `act report` can correlate + // the guard install against later yield. Best effort; a failure just + // leaves the guard row "not measurable". + try { + const { captureGuardBaseline } = await import('../act/report.js') + const baseline = await captureGuardBaseline({ cwd: process.cwd() }) + if (baseline) built.plan.baseline = baseline + } catch { /* baseline is optional */ } const record = await runAction(built.plan) console.log(` Installed ${chalk.bold(shortId(record.id))} ${built.plan.description}`) console.log(chalk.dim(` Undo anytime: codeburn act undo ${shortId(record.id)}`)) diff --git a/src/main.ts b/src/main.ts index 758d244..634fb56 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1350,7 +1350,20 @@ program return } assertFormat(format, ['text', 'json'], 'optimize') - await runOptimize(projects, label, range, { format }) + if (format === 'text') { + // Surface realized savings from applied actions. computeActReport returns + // fast without scanning when the journal has no eligible applied actions, + // so users who never opted in see identical output. + const { computeActReport, buildOptimizeAppliedHeader } = await import('./act/report.js') + const applied = await computeActReport() + await runOptimize(projects, label, range, { + format, + appliedHeader: buildOptimizeAppliedHeader(applied) ?? undefined, + previouslyApplied: applied.appliedByFinding, + }) + } else { + await runOptimize(projects, label, range, { format }) + } }) program diff --git a/src/optimize.ts b/src/optimize.ts index 3610a15..8c59e0e 100644 --- a/src/optimize.ts +++ b/src/optimize.ts @@ -26,12 +26,12 @@ const RED = '#F55B5B' // Token estimation constants // ============================================================================ -const AVG_TOKENS_PER_READ = 600 -const TOKENS_PER_MCP_TOOL = 400 -const TOOLS_PER_MCP_SERVER = 5 -const TOKENS_PER_AGENT_DEF = 80 -const TOKENS_PER_SKILL_DEF = 80 -const TOKENS_PER_COMMAND_DEF = 60 +export const AVG_TOKENS_PER_READ = 600 +export const TOKENS_PER_MCP_TOOL = 400 +export const TOOLS_PER_MCP_SERVER = 5 +export const TOKENS_PER_AGENT_DEF = 80 +export const TOKENS_PER_SKILL_DEF = 80 +export const TOKENS_PER_COMMAND_DEF = 60 const CLAUDEMD_TOKENS_PER_LINE = 13 const BASH_TOKENS_PER_CHAR = 0.25 @@ -48,7 +48,7 @@ const MIN_DUPLICATE_READS_TO_FLAG = 5 const DUPLICATE_READS_HIGH_THRESHOLD = 30 const DUPLICATE_READS_MEDIUM_THRESHOLD = 10 const MIN_EDITS_FOR_RATIO = 10 -const HEALTHY_READ_EDIT_RATIO = 4 +export const HEALTHY_READ_EDIT_RATIO = 4 const LOW_RATIO_HIGH_THRESHOLD = 2 const LOW_RATIO_MEDIUM_THRESHOLD = 3 const MIN_API_CALLS_FOR_CACHE = 10 @@ -1605,8 +1605,8 @@ export function detectBloatedClaudeMd(projectCwds: Set): WasteFinding | } } -const READ_TOOL_NAMES = new Set(['Read', 'Grep', 'Glob', 'FileReadTool', 'GrepTool', 'GlobTool']) -const EDIT_TOOL_NAMES = new Set(['Edit', 'Write', 'FileEditTool', 'FileWriteTool', 'NotebookEdit']) +export const READ_TOOL_NAMES = new Set(['Read', 'Grep', 'Glob', 'FileReadTool', 'GrepTool', 'GlobTool']) +export const EDIT_TOOL_NAMES = new Set(['Edit', 'Write', 'FileEditTool', 'FileWriteTool', 'NotebookEdit']) export function detectLowReadEditRatio(calls: ToolCall[]): WasteFinding | null { let reads = 0 @@ -2343,7 +2343,7 @@ function sessionTrend( const INPUT_COST_RATIO = 0.7 const DEFAULT_COST_PER_TOKEN = 0 -function computeInputCostRate(projects: ProjectSummary[]): number { +export function computeInputCostRate(projects: ProjectSummary[]): number { const sessions = projects.flatMap(p => p.sessions) const totalCost = sessions.reduce((s, sess) => s + sess.totalCostUSD, 0) const totalTokens = sessions.reduce((s, sess) => @@ -2526,6 +2526,8 @@ function renderOptimize( callCount: number, healthScore: number, healthGrade: HealthGrade, + appliedHeader?: string, + previouslyApplied?: Record, ): string { const lines: string[] = [] lines.push('') @@ -2539,6 +2541,7 @@ function renderOptimize( chalk.hex(GOLD)(formatCost(periodCost)), `Health: ${chalk.bold.hex(GRADE_COLORS[healthGrade])(healthGrade)}${chalk.dim(` (${healthScore}/100${issueSuffix})`)}`, ].join(chalk.hex(DIM)(' '))) + if (appliedHeader) lines.push(' ' + chalk.hex(GREEN)(appliedHeader)) lines.push('') if (findings.length === 0) { @@ -2561,7 +2564,10 @@ function renderOptimize( lines.push('') for (let i = 0; i < findings.length; i++) { - lines.push(...renderFinding(i + 1, findings[i], costRate)) + const f = findings[i]! + const appliedOn = previouslyApplied?.[f.id] + const shown = appliedOn ? { ...f, title: `${f.title} (previously applied ${appliedOn}, re-flagged)` } : f + lines.push(...renderFinding(i + 1, shown, costRate)) } lines.push(chalk.hex(DIM)(' ' + SEP.repeat(PANEL_WIDTH))) @@ -2574,7 +2580,7 @@ export async function runOptimize( projects: ProjectSummary[], periodLabel: string, dateRange?: DateRange, - opts: { format?: 'text' | 'json' } = {}, + opts: { format?: 'text' | 'json'; appliedHeader?: string; previouslyApplied?: Record } = {}, ): Promise { const format = opts.format ?? 'text' if (projects.length === 0 && format === 'text') { @@ -2597,7 +2603,7 @@ export async function runOptimize( return } - const output = renderOptimize(findings, costRate, periodLabel, periodCost, sessions.length, callCount, healthScore, healthGrade) + const output = renderOptimize(findings, costRate, periodLabel, periodCost, sessions.length, callCount, healthScore, healthGrade, opts.appliedHeader, opts.previouslyApplied) console.log(output) } diff --git a/tests/act-report.test.ts b/tests/act-report.test.ts new file mode 100644 index 0000000..76a59fb --- /dev/null +++ b/tests/act-report.test.ts @@ -0,0 +1,303 @@ +import { afterAll, describe, expect, it } from 'vitest' +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { journalPath } from '../src/act/journal.js' +import { + buildActReportJson, + buildOptimizeAppliedHeader, + computeActReport, + renderActReport, +} from '../src/act/report.js' +import type { ActionRecord } from '../src/act/types.js' +import type { ProjectSummary } from '../src/types.js' + +type Session = ProjectSummary['sessions'][number] + +const roots: string[] = [] +afterAll(async () => { for (const r of roots) await rm(r, { recursive: true, force: true }) }) + +const NOW = new Date('2026-07-01T00:00:00Z') +function daysAgo(n: number): string { + return new Date(NOW.getTime() - n * 86_400_000).toISOString() +} + +async function writeJournal(records: ActionRecord[]): Promise { + const root = await mkdtemp(join(tmpdir(), 'codeburn-act-report-')) + roots.push(root) + const actionsDir = join(root, 'actions') + await mkdir(actionsDir, { recursive: true }) + await writeFile(journalPath(actionsDir), records.map(r => JSON.stringify(r)).join('\n') + (records.length ? '\n' : '')) + return actionsDir +} + +function makeSession(id: string, firstTimestamp: string, over: Partial = {}): Session { + return { + sessionId: id, + project: 'app', + firstTimestamp, + lastTimestamp: firstTimestamp, + totalCostUSD: 1, + totalSavingsUSD: 0, + totalInputTokens: 1000, + totalOutputTokens: 0, + totalReasoningTokens: 0, + totalCacheReadTokens: 0, + totalCacheWriteTokens: 0, + apiCalls: 1, + turns: [], + modelBreakdown: {}, + toolBreakdown: {}, + mcpBreakdown: {}, + bashBreakdown: {}, + categoryBreakdown: {} as Session['categoryBreakdown'], + skillBreakdown: {}, + subagentBreakdown: {}, + ...over, + } +} + +function projectOf(sessions: Session[]): ProjectSummary { + return { + project: 'app', + projectPath: '/tmp/app', + sessions, + totalCostUSD: sessions.reduce((s, x) => s + x.totalCostUSD, 0), + totalSavingsUSD: 0, + totalApiCalls: sessions.length, + totalProxiedCostUSD: 0, + } +} + +function sessionsAt(count: number, ts: string, over: Partial = {}): Session[] { + return Array.from({ length: count }, (_, i) => makeSession(`s${i}`, ts, over)) +} + +function mcpRecord(over: Partial = {}): ActionRecord { + const at = daysAgo(10) + return { + id: 'a1', + at, + kind: 'mcp-remove', + findingId: 'unused-mcp', + description: 'Remove an MCP server from config', + changes: [], + status: 'applied', + baseline: { windowDays: 14, capturedAt: at, estimatedTokens: 56_000, sessions: 28, metrics: { 'brave-search': 2000 } }, + ...over, + } +} + +const load = (projects: ProjectSummary[]) => async () => projects + +describe('mcp realized delta', () => { + it('multiplies baseline tokens-per-session by post-window sessions (exact)', async () => { + const actionsDir = await writeJournal([mcpRecord()]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(5)))]) }) + + expect(report.rows).toHaveLength(1) + const row = report.rows[0]! + expect(row.status).toBe('measured') + expect(row.realizedTokens).toBe(40_000) // 2000 tokens/session * 20 sessions + expect(row.estimatedTokens).toBe(56_000) + expect(row.confidence).toBe('normal') + expect(report.totalRealizedTokens).toBe(40_000) + }) + + it('reports "reverted" with zero savings when the server reappears in the window', async () => { + const actionsDir = await writeJournal([mcpRecord()]) + const back = sessionsAt(20, daysAgo(5), { mcpInventory: ['mcp__brave-search__search'] }) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(back)]) }) + + const row = report.rows[0]! + expect(row.status).toBe('reverted') + expect(row.realizedTokens).toBe(0) + expect(row.note).toMatch(/reverted by user/) + expect(report.totalRealizedTokens).toBe(0) + }) +}) + +describe('confidence markers', () => { + it('marks low when fewer than 20 post-window sessions', async () => { + const actionsDir = await writeJournal([mcpRecord()]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(10, daysAgo(5)))]) }) + + const row = report.rows[0]! + expect(row.status).toBe('measured') + expect(row.realizedTokens).toBe(20_000) // 2000 * 10 + expect(row.confidence).toBe('low') + }) + + it('marks low when volume shifts more than 2x versus baseline', async () => { + // 25 post-window sessions (>= 20, so not the count rule) over 10 days is + // 2.5/day against a 1/day baseline (14 sessions / 14 days) -> 2.5x shift. + const rec = mcpRecord({ baseline: { windowDays: 14, capturedAt: daysAgo(10), estimatedTokens: 56_000, sessions: 14, metrics: { 'brave-search': 2000 } } }) + const actionsDir = await writeJournal([rec]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(25, daysAgo(5)))]) }) + + const row = report.rows[0]! + expect(row.status).toBe('measured') + expect(row.confidence).toBe('low') + }) + + it('stays normal when volume is comparable to baseline', async () => { + const actionsDir = await writeJournal([mcpRecord()]) // baseline 28/14 = 2/day + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(5)))]) }) // 20/10 = 2/day + expect(report.rows[0]!.confidence).toBe('normal') + }) +}) + +describe('eligibility', () => { + it('excludes undone actions and actions younger than 3 days', async () => { + const records = [ + mcpRecord({ id: 'old', at: daysAgo(10) }), + mcpRecord({ id: 'young', at: daysAgo(1) }), + mcpRecord({ id: 'undone', at: daysAgo(20), status: 'undone' }), + ] + const actionsDir = await writeJournal(records) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(5)))]) }) + + expect(report.rows).toHaveLength(1) + expect(report.rows[0]!.id).toBe('old') + expect(report.activeCount).toBe(2) // old + young are applied; undone is not + }) +}) + +describe('read-edit realized delta', () => { + it('credits the reduction in the read deficit using the detector estimate math', async () => { + // Baseline ratio 1:1 (deficit 3 reads/edit). After window: 120 reads / 40 + // edits = 3:1 (deficit 1). Credit (3 - 1) * 40 edits * 600 = 48000. + const at = daysAgo(10) + const rec: ActionRecord = { + id: 'r1', at, kind: 'claude-md-rule', findingId: 'read-edit-ratio', + description: 'Add the read-edit-ratio rule block', changes: [], status: 'applied', + baseline: { windowDays: 14, capturedAt: at, estimatedTokens: 12_000, sessions: 28, metrics: { reads: 10, edits: 10 } }, + } + const actionsDir = await writeJournal([rec]) + const session = makeSession('s0', daysAgo(5), { toolBreakdown: { Read: { calls: 120 }, Edit: { calls: 40 } } }) + const filler = sessionsAt(19, daysAgo(4)) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf([session, ...filler])]) }) + + const row = report.rows[0]! + expect(row.status).toBe('measured') + expect(row.realizedTokens).toBe(48_000) + expect(row.note).toMatch(/1\.0:1 -> 3\.0:1/) + }) +}) + +describe('archive realized delta', () => { + it('measures per-item definition tokens times sessions and detects un-archive', async () => { + const at = daysAgo(10) + const kept = join(tmpdir(), 'codeburn-act-report-absent-skill-xyz') // absent -> not reverted + const base = { windowDays: 14, capturedAt: at, estimatedTokens: 160, sessions: 28, metrics: { 'skill-a': 80, 'skill-b': 80 } } + const rec: ActionRecord = { + id: 'ar1', at, kind: 'archive-skill', findingId: 'unused-skills', + description: 'Archive 2 unused skills', status: 'applied', + changes: [{ path: kept, backup: null, op: 'move', movedTo: kept + '.archived', afterHash: '' }], + baseline: base, + } + const actionsDir = await writeJournal([rec]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(5)))]) }) + expect(report.rows[0]!.status).toBe('measured') + expect(report.rows[0]!.realizedTokens).toBe(3200) // 160 tokens/session * 20 + + // Now the original path exists again -> reverted, zero savings. + const restoredPath = join(actionsDir, 'restored-skill') + await writeFile(restoredPath, 'x') + const rec2: ActionRecord = { ...rec, changes: [{ path: restoredPath, backup: null, op: 'move', movedTo: restoredPath + '.archived', afterHash: '' }] } + const dir2 = await writeJournal([rec2]) + const report2 = await computeActReport({ actionsDir: dir2, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(5)))]) }) + expect(report2.rows[0]!.status).toBe('reverted') + expect(report2.rows[0]!.realizedTokens).toBe(0) + }) +}) + +describe('unmeasured kinds', () => { + it('marks bash cap not measurable but keeps the estimate visible', async () => { + const at = daysAgo(10) + const rec: ActionRecord = { + id: 'b1', at, kind: 'shell-config', findingId: 'bash-output-cap', + description: 'Set the bash output cap', changes: [], status: 'applied', + baseline: { windowDays: 14, capturedAt: at, estimatedTokens: 3750, sessions: 28, metrics: { calls: 200 } }, + } + const actionsDir = await writeJournal([rec]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(5)))]) }) + expect(report.rows[0]!.status).toBe('not-measurable') + expect(report.rows[0]!.estimatedTokens).toBe(3750) + expect(report.measuredCount).toBe(0) + }) +}) + +describe('optimize header', () => { + it('appears only when a measured token action exists', async () => { + const actionsDir = await writeJournal([mcpRecord()]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(5)))]) }) + const header = buildOptimizeAppliedHeader(report) + expect(header).toMatch(/^Applied fixes: 1 active, realized ~40\.0K tokens.*over 10 days\. Details: codeburn act report$/) + }) + + it('returns null and never scans when the journal has no eligible actions', async () => { + const emptyDir = await writeJournal([]) + const report = await computeActReport({ + actionsDir: emptyDir, now: NOW, + loadProjects: async () => { throw new Error('should not scan for an empty journal') }, + }) + expect(report.rows).toHaveLength(0) + expect(report.activeCount).toBe(0) + expect(buildOptimizeAppliedHeader(report)).toBeNull() + }) + + it('records the earliest apply date per finding for re-flagging', async () => { + const records = [ + mcpRecord({ id: 'x1', at: daysAgo(9), findingId: 'unused-mcp' }), + mcpRecord({ id: 'x2', at: daysAgo(4), findingId: 'unused-mcp' }), + ] + const actionsDir = await writeJournal(records) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(3)))]) }) + expect(report.appliedByFinding['unused-mcp']).toBe(daysAgo(9).slice(0, 10)) + }) +}) + +describe('json + render shape', () => { + it('mirrors the rows and totals in --json', async () => { + const actionsDir = await writeJournal([mcpRecord()]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(5)))]) }) + const json = buildActReportJson(report) as { + actions: Array> + totals: Record + footer: string + windowCapDays: number + } + + expect(Array.isArray(json.actions)).toBe(true) + expect(json.actions[0]).toMatchObject({ + kind: 'mcp-remove', + estimatedTokens: 56_000, + realizedTokens: 40_000, + status: 'measured', + confidence: 'normal', + }) + expect(json.totals).toMatchObject({ realizedTokens: 40_000, measuredActions: 1, activeActions: 1 }) + expect(json.windowCapDays).toBe(30) + expect(typeof json.footer).toBe('string') + expect(json.footer).toMatch(/correlation/) + }) + + it('renders an empty state without a table when nothing is measurable', async () => { + const emptyDir = await writeJournal([]) + const report = await computeActReport({ actionsDir: emptyDir, now: NOW, loadProjects: async () => [] }) + const out = renderActReport(report) + expect(out).toMatch(/No applied actions to measure yet/) + expect(out).not.toMatch(/Total realized/) + }) + + it('renders a table with a total row when measurements exist', async () => { + const actionsDir = await writeJournal([mcpRecord()]) + const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(5)))]) }) + const out = renderActReport(report) + expect(out).toMatch(/Total realized/) + expect(out).toMatch(/40\.0K/) + expect(out).toMatch(/measures only its own metric/) + }) +})