mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-08-20 14:04:25 +00:00
optimize: re-measure applied fixes on every run, with --auto-revert
Every still-applied journal entry now comes back with a verdict on the next optimize run: worked (>=70% of its window-scaled estimate realized), partial, no-effect (printed with its undo command), or measuring while it is younger than the 3-day window. The verdicts come off the rows act report already computes, so there is one reconciliation, not two; the AppliedFix type and its formatter live in act/types.ts so the optimize renderer can use them without importing report.ts back into optimize.ts. --auto-revert undoes the no-effect entries through the same code path as codeburn act undo. It never touches partial or measuring entries, and never a claude-md-rule - those land in whatever directory the user happened to be in, the same reason --yes skips them. --apply now names when the re-measure happens, and --format json carries appliedFixes[] (add-only).
This commit is contained in:
parent
117d66f7f7
commit
52eb9fdb8d
8 changed files with 391 additions and 24 deletions
|
|
@ -7,6 +7,7 @@ import { formatCost } from '../currency.js'
|
|||
import { formatTokens } from '../format.js'
|
||||
import { runAction } from './apply.js'
|
||||
import { shortId } from './journal.js'
|
||||
import { REPORT_MIN_AGE_DAYS } from './types.js'
|
||||
import { planFindings, type FindingPlan, type PlanContext } from './plans.js'
|
||||
|
||||
export type ApplyOptions = {
|
||||
|
|
@ -176,9 +177,11 @@ export async function runOptimizeApply(
|
|||
} catch { /* baseline is optional; apply proceeds without it */ }
|
||||
|
||||
print()
|
||||
let applied = 0
|
||||
for (const fp of selected) {
|
||||
try {
|
||||
const record = await runAction(fp.plan!, opts.actionsDir)
|
||||
applied++
|
||||
print(` Applied ${chalk.bold(shortId(record.id))} ${record.description}`)
|
||||
print(chalk.dim(` Undo anytime: codeburn act undo ${shortId(record.id)}`))
|
||||
} catch (e) {
|
||||
|
|
@ -186,5 +189,8 @@ export async function runOptimizeApply(
|
|||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
if (applied > 0) {
|
||||
print(chalk.dim(` CodeBurn will re-measure these on your next optimize run after ${REPORT_MIN_AGE_DAYS} days.`))
|
||||
}
|
||||
print()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import { existsSync } from 'fs'
|
||||
import { dirname } from 'node:path'
|
||||
import type { DateRange, ProjectSummary, SessionSummary } from '../types.js'
|
||||
import type { ActionBaseline, ActionKind, ActionRecord } from './types.js'
|
||||
import type { ActionBaseline, ActionKind, ActionRecord, AppliedFix, AppliedVerdict } from './types.js'
|
||||
import { REPORT_MIN_AGE_DAYS, VERDICT_WORKED_RATIO } from './types.js'
|
||||
import type { FindingPlan } from './plans.js'
|
||||
import {
|
||||
AVG_TOKENS_PER_READ,
|
||||
|
|
@ -20,7 +21,8 @@ import {
|
|||
} from '../optimize.js'
|
||||
import { parseAllSessions } from '../parser.js'
|
||||
import { computeYield, type YieldSummary } from '../yield.js'
|
||||
import { defaultActionsDir, readRecords } from './journal.js'
|
||||
import { defaultActionsDir, readRecords, shortId } from './journal.js'
|
||||
import { undoAction } from './undo.js'
|
||||
import { renderTable } from '../text-table.js'
|
||||
import { formatTokens } from '../format.js'
|
||||
import { formatCost } from '../currency.js'
|
||||
|
|
@ -28,7 +30,6 @@ import { formatCost } from '../currency.js'
|
|||
const DAY_MS = 24 * 60 * 60 * 1000
|
||||
const WINDOW_CAP_DAYS = 30
|
||||
const BASELINE_WINDOW_DAYS = 14
|
||||
const REPORT_MIN_AGE_DAYS = 3
|
||||
const MIN_POST_WINDOW_SESSIONS = 20
|
||||
const VOLUME_SHIFT_FACTOR = 2
|
||||
|
||||
|
|
@ -59,6 +60,8 @@ const ARCHIVE_DEF_TOKENS: Partial<Record<ActionKind, number>> = {
|
|||
// 'pending' means the applied change has not taken effect in any post-apply
|
||||
// session yet (e.g. deferral before a client restart) - distinct from
|
||||
// 'reverted', which asserts the user undid it.
|
||||
export { REPORT_MIN_AGE_DAYS }
|
||||
|
||||
export type RealizedStatus = 'measured' | 'reverted' | 'not-measurable' | 'pending'
|
||||
|
||||
export type ActReportRow = {
|
||||
|
|
@ -102,6 +105,8 @@ export type ActReport = {
|
|||
// findingId -> earliest apply date of an active applied action; drives the
|
||||
// optimize "(previously applied ..., re-flagged)" title suffix.
|
||||
appliedByFinding: Record<string, string>
|
||||
// One entry per active applied action, including ones too young to measure.
|
||||
appliedFixes: AppliedFix[]
|
||||
}
|
||||
|
||||
export type ActReportOptions = {
|
||||
|
|
@ -470,6 +475,63 @@ function isSaneRecord(r: ActionRecord): boolean {
|
|||
return typeof r.at === 'string' && typeof r.status === 'string' && !Number.isNaN(new Date(r.at).getTime())
|
||||
}
|
||||
|
||||
// Turn the measured rows plus the still-young entries into one verdict per
|
||||
// active applied action. No second reconciliation: everything measurable comes
|
||||
// straight off the row `act report` already computed.
|
||||
function buildAppliedFixes(active: ActionRecord[], rows: ActReportRow[], now: Date): AppliedFix[] {
|
||||
const byId = new Map(rows.map(r => [r.id, r]))
|
||||
return active.map(rec => {
|
||||
const row = byId.get(rec.id)
|
||||
const base = {
|
||||
id: rec.id,
|
||||
kind: rec.kind,
|
||||
findingId: rec.findingId ?? null,
|
||||
appliedAt: rec.at,
|
||||
ageDays: ageDays(rec.at, now),
|
||||
undoCommand: `codeburn act undo ${shortId(rec.id)}`,
|
||||
}
|
||||
// No row means too young to measure; a row that is not a measured token
|
||||
// row (not-measurable, not yet in effect, reverted by the user, or a
|
||||
// correlation-only kind) has no reduction to judge either.
|
||||
if (!row) return { ...base, verdict: 'pending' as const, estimatedTokens: rec.baseline?.estimatedTokens ?? 0, realizedTokens: 0, note: '' }
|
||||
if (row.status !== 'measured' || !isTokenKind(row.kind)) {
|
||||
return { ...base, verdict: 'pending' as const, estimatedTokens: row.estimatedForWindow, realizedTokens: 0, note: row.note }
|
||||
}
|
||||
const estimatedTokens = row.estimatedForWindow
|
||||
const realizedTokens = row.realizedTokens
|
||||
const verdict: AppliedVerdict = realizedTokens <= 0
|
||||
? 'no-effect'
|
||||
: estimatedTokens <= 0 || realizedTokens >= estimatedTokens * VERDICT_WORKED_RATIO ? 'worked' : 'partial'
|
||||
return { ...base, verdict, estimatedTokens, realizedTokens, note: row.note }
|
||||
})
|
||||
}
|
||||
|
||||
// --auto-revert: undo the fixes that measured no reduction at all. CLAUDE.md
|
||||
// rules are never undone unattended, matching the --yes guardrail - the file
|
||||
// belongs to whatever project the user happened to be in.
|
||||
export async function autoRevertNoEffect(
|
||||
fixes: AppliedFix[], opts: { actionsDir?: string } = {},
|
||||
): Promise<{ lines: string[]; revertedIds: Set<string> }> {
|
||||
const lines: string[] = []
|
||||
const revertedIds = new Set<string>()
|
||||
for (const fix of fixes) {
|
||||
if (fix.verdict !== 'no-effect') continue
|
||||
const label = fix.findingId ?? fix.kind
|
||||
if (fix.kind === 'claude-md-rule') {
|
||||
lines.push(`Not auto-reverted: ${label} edits a CLAUDE.md. Revert: ${fix.undoCommand}`)
|
||||
continue
|
||||
}
|
||||
try {
|
||||
const record = await undoAction({ id: fix.id }, { actionsDir: opts.actionsDir })
|
||||
revertedIds.add(fix.id)
|
||||
lines.push(`Reverted ${shortId(record.id)}: ${record.description}`)
|
||||
} catch (err) {
|
||||
lines.push(`Could not revert ${label}: ${err instanceof Error ? err.message : String(err)}`)
|
||||
}
|
||||
}
|
||||
return { lines, revertedIds }
|
||||
}
|
||||
|
||||
export async function computeActReport(opts: ActReportOptions = {}): Promise<ActReport> {
|
||||
const now = opts.now ?? new Date()
|
||||
const rawRecords = await readRecords(opts.actionsDir ?? defaultActionsDir())
|
||||
|
|
@ -497,6 +559,7 @@ export async function computeActReport(opts: ActReportOptions = {}): Promise<Act
|
|||
observedDays: 0,
|
||||
malformedRecords,
|
||||
appliedByFinding,
|
||||
appliedFixes: buildAppliedFixes(active, [], now),
|
||||
}
|
||||
|
||||
const eligible = active.filter(r => ageDays(r.at, now) > REPORT_MIN_AGE_DAYS)
|
||||
|
|
@ -550,6 +613,7 @@ export async function computeActReport(opts: ActReportOptions = {}): Promise<Act
|
|||
observedDays,
|
||||
malformedRecords,
|
||||
appliedByFinding,
|
||||
appliedFixes: buildAppliedFixes(active, rows, now),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -651,6 +715,7 @@ export function buildActReportJson(report: ActReport): unknown {
|
|||
activeActions: report.activeCount,
|
||||
observedDays: report.observedDays,
|
||||
},
|
||||
appliedFixes: report.appliedFixes,
|
||||
footer: HONEST_FOOTER,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import { formatTokens } from '../format.js'
|
||||
|
||||
export type ActionKind =
|
||||
| 'mcp-remove' | 'mcp-project-scope'
|
||||
| 'defer-enable' | 'defer-alwaysload' | 'defer-threshold'
|
||||
|
|
@ -67,3 +69,61 @@ export type ActionPlan = {
|
|||
changes: PlannedChange[]
|
||||
baseline?: ActionBaseline
|
||||
}
|
||||
|
||||
// Applied actions are re-measured on every `codeburn optimize` run: only fixes
|
||||
// at least this old have a post-apply window to measure against.
|
||||
export const REPORT_MIN_AGE_DAYS = 3
|
||||
// A fix counts as having worked once it realizes this share of its
|
||||
// window-scaled estimate; anything above zero but below it is partial.
|
||||
export const VERDICT_WORKED_RATIO = 0.7
|
||||
|
||||
// Per-applied-entry judgement shown by `codeburn optimize` after an --apply.
|
||||
// Computed in act/report.ts from the same rows `act report` prints - there is
|
||||
// one reconciliation, not two. Lives here so the optimize renderer can format
|
||||
// it without importing report.ts back into optimize.ts.
|
||||
export type AppliedVerdict = 'worked' | 'partial' | 'no-effect' | 'pending'
|
||||
|
||||
export type AppliedFix = {
|
||||
id: string
|
||||
kind: ActionKind
|
||||
findingId: string | null
|
||||
appliedAt: string
|
||||
ageDays: number
|
||||
verdict: AppliedVerdict
|
||||
// Window-scaled estimate, the same column `act report` compares against.
|
||||
estimatedTokens: number
|
||||
realizedTokens: number
|
||||
note: string
|
||||
undoCommand: string
|
||||
}
|
||||
|
||||
const VERDICT_GLYPH: Record<AppliedVerdict, string> = {
|
||||
worked: '\u2713',
|
||||
partial: '~',
|
||||
'no-effect': '\u2717',
|
||||
pending: '\u2026',
|
||||
}
|
||||
|
||||
export function appliedFixGlyph(fix: AppliedFix): string {
|
||||
return VERDICT_GLYPH[fix.verdict]
|
||||
}
|
||||
|
||||
// One plain line per applied fix: what it estimated, what it measured, and for
|
||||
// a fix that did nothing, how to put it back.
|
||||
export function formatAppliedFix(fix: AppliedFix): string {
|
||||
const age = Math.max(0, Math.floor(fix.ageDays))
|
||||
const head = `${fix.findingId ?? fix.kind} (${age}d ago)`
|
||||
if (fix.verdict === 'pending') {
|
||||
const why = fix.note || (age <= REPORT_MIN_AGE_DAYS
|
||||
? `measuring, check back after ${REPORT_MIN_AGE_DAYS} days`
|
||||
: 'measuring')
|
||||
return `${head}: ${why}`
|
||||
}
|
||||
const pair = `est. ${formatTokens(fix.estimatedTokens)} -> measured ${formatTokens(fix.realizedTokens)}`
|
||||
if (fix.verdict === 'worked') return `${head}: ${pair}`
|
||||
if (fix.verdict === 'partial') {
|
||||
const under = Math.round((1 - fix.realizedTokens / fix.estimatedTokens) * 100)
|
||||
return `${head}: ${pair} (-${under}% vs estimate)`
|
||||
}
|
||||
return `${head}: ${pair} - did not help. Revert: ${fix.undoCommand}`
|
||||
}
|
||||
|
|
|
|||
45
src/main.ts
45
src/main.ts
|
|
@ -11,6 +11,7 @@ import { renderStatusBar } from './format.js'
|
|||
import { toDateString } from './daily-cache.js'
|
||||
import { dateKey } from './day-aggregator.js'
|
||||
import { CATEGORY_LABELS, type DateRange, type ProjectSummary, type TaskCategory } from './types.js'
|
||||
import type { AppliedFix } from './act/types.js'
|
||||
import { aggregateModelEfficiency } from './model-efficiency.js'
|
||||
import { buildPeriodData, buildMenubarPayloadForRange, buildDurablePeriod, type DurablePeriod } from './usage-aggregator.js'
|
||||
import { renderDashboard } from './dashboard.js'
|
||||
|
|
@ -1806,6 +1807,7 @@ program
|
|||
.option('--yes', 'With --apply: apply every appliable fix without prompting')
|
||||
.option('--dry-run', 'With --apply: print the plan and exit without changing anything')
|
||||
.option('--only <ids>', 'With --apply: restrict to a comma-separated list of finding ids')
|
||||
.option('--auto-revert', 'Undo applied fixes that measured no reduction (never CLAUDE.md rules)')
|
||||
.action(async (opts) => {
|
||||
assertProvider(opts.provider, 'optimize')
|
||||
const format = opts.json ? 'json' : opts.format
|
||||
|
|
@ -1835,24 +1837,31 @@ program
|
|||
return
|
||||
}
|
||||
assertFormat(format, ['text', 'json'], 'optimize')
|
||||
if (format === 'text') {
|
||||
// Surface realized savings from applied actions. Best effort: optimize
|
||||
// must never fail because of journal contents, so any error just drops
|
||||
// the header. computeActReport returns fast without scanning when the
|
||||
// journal has no eligible applied actions, so users who never opted in
|
||||
// see identical output.
|
||||
let appliedHeader: string | undefined
|
||||
let previouslyApplied: Record<string, string> | undefined
|
||||
try {
|
||||
const { computeActReport, buildOptimizeAppliedHeader } = await import('./act/report.js')
|
||||
const applied = await computeActReport()
|
||||
appliedHeader = buildOptimizeAppliedHeader(applied) ?? undefined
|
||||
previouslyApplied = applied.appliedByFinding
|
||||
} catch { /* the header is optional; never block the findings */ }
|
||||
await runOptimize(projects, label, range, { format, appliedHeader, previouslyApplied, provider: opts.provider })
|
||||
} else {
|
||||
await runOptimize(projects, label, range, { format, provider: opts.provider })
|
||||
}
|
||||
// Surface realized savings from applied actions, and re-measure every one
|
||||
// of them. Best effort: optimize must never fail because of journal
|
||||
// contents, so any error just drops the extras. computeActReport returns
|
||||
// fast without scanning when the journal has no applied actions, so users
|
||||
// who never opted in see identical output.
|
||||
let appliedHeader: string | undefined
|
||||
let previouslyApplied: Record<string, string> | undefined
|
||||
let appliedFixes: AppliedFix[] | undefined
|
||||
try {
|
||||
const { computeActReport, buildOptimizeAppliedHeader, autoRevertNoEffect } = await import('./act/report.js')
|
||||
const applied = await computeActReport()
|
||||
appliedHeader = buildOptimizeAppliedHeader(applied) ?? undefined
|
||||
previouslyApplied = applied.appliedByFinding
|
||||
appliedFixes = applied.appliedFixes
|
||||
if (opts.autoRevert) {
|
||||
const { lines, revertedIds } = await autoRevertNoEffect(appliedFixes)
|
||||
appliedFixes = appliedFixes.filter(f => !revertedIds.has(f.id))
|
||||
// JSON output must stay parseable, so the revert log goes to stderr there.
|
||||
for (const line of lines) {
|
||||
if (format === 'json') process.stderr.write(` ${line}\n`)
|
||||
else console.log(` ${line}`)
|
||||
}
|
||||
}
|
||||
} catch { /* the applied section is optional; never block the findings */ }
|
||||
await runOptimize(projects, label, range, { format, appliedHeader, previouslyApplied, appliedFixes, provider: opts.provider })
|
||||
})
|
||||
|
||||
program
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import type { DateRange, ProjectSummary, SessionSummary } from './types.js'
|
|||
import { formatCost } from './currency.js'
|
||||
import { formatTokens } from './format.js'
|
||||
import { recommendModelDefault, type ModelDefaultRecommendation } from './act/model-defaults.js'
|
||||
import { appliedFixGlyph, formatAppliedFix, type AppliedFix } from './act/types.js'
|
||||
import { aggregateFileChurn, buildCoachingNotes, scanUserCorrections, medianTimeToFirstEditMs, worstOneShotCategory, type ReworkedFile } from './workflow-insights.js'
|
||||
|
||||
// ============================================================================
|
||||
|
|
@ -491,6 +492,8 @@ export type OptimizeJsonReport = {
|
|||
/// 1-3 templated one-liners keyed on the strongest workflow signals.
|
||||
coachingNotes: string[]
|
||||
modelRecommendations?: Array<ModelDefaultRecommendation>
|
||||
/// One entry per still-applied fix, re-measured on every run (see act/report.ts).
|
||||
appliedFixes: Array<Omit<AppliedFix, 'ageDays' | 'note'>>
|
||||
}
|
||||
|
||||
export type ToolCall = {
|
||||
|
|
@ -3365,6 +3368,25 @@ function renderWorkflowSection(reworkedFiles: ReworkedFile[], coachingNotes: str
|
|||
return lines
|
||||
}
|
||||
|
||||
const APPLIED_FIX_COLORS: Record<AppliedFix['verdict'], string> = {
|
||||
worked: GREEN,
|
||||
partial: GOLD,
|
||||
'no-effect': RED,
|
||||
pending: DIM,
|
||||
}
|
||||
|
||||
// Closes the loop after --apply: every still-applied fix gets its measured
|
||||
// verdict back here, on every run.
|
||||
function renderAppliedFixes(appliedFixes: AppliedFix[]): string[] {
|
||||
if (appliedFixes.length === 0) return []
|
||||
const lines = [chalk.bold.hex(ORANGE)(' Applied fixes'), '']
|
||||
for (const fix of appliedFixes) {
|
||||
lines.push(chalk.hex(APPLIED_FIX_COLORS[fix.verdict])(` ${appliedFixGlyph(fix)} ${formatAppliedFix(fix)}`))
|
||||
}
|
||||
lines.push('')
|
||||
return lines
|
||||
}
|
||||
|
||||
export function renderOptimize(
|
||||
findings: WasteFinding[],
|
||||
costRate: number,
|
||||
|
|
@ -3379,6 +3401,7 @@ export function renderOptimize(
|
|||
appliedHeader?: string,
|
||||
previouslyApplied?: Record<string, string>,
|
||||
modelRecommendations?: ModelDefaultRecommendation[],
|
||||
appliedFixes: AppliedFix[] = [],
|
||||
): string {
|
||||
const lines: string[] = []
|
||||
lines.push('')
|
||||
|
|
@ -3406,6 +3429,7 @@ export function renderOptimize(
|
|||
lines.push(chalk.dim(' token waste: junk directory reads, duplicate file reads, unused'))
|
||||
lines.push(chalk.dim(' agents/skills/MCP servers, bloated CLAUDE.md, and more.'))
|
||||
lines.push('')
|
||||
lines.push(...renderAppliedFixes(appliedFixes))
|
||||
lines.push(...renderWorkflowSection(reworkedFiles, coachingNotes))
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
|
@ -3441,6 +3465,7 @@ export function renderOptimize(
|
|||
lines.push(chalk.hex(DIM)(' ' + SEP.repeat(PANEL_WIDTH)))
|
||||
lines.push('')
|
||||
|
||||
lines.push(...renderAppliedFixes(appliedFixes))
|
||||
lines.push(...renderWorkflowSection(reworkedFiles, coachingNotes))
|
||||
|
||||
if (modelRecommendations && modelRecommendations.length > 0) {
|
||||
|
|
@ -3478,7 +3503,7 @@ export async function runOptimize(
|
|||
projects: ProjectSummary[],
|
||||
periodLabel: string,
|
||||
dateRange?: DateRange,
|
||||
opts: { format?: 'text' | 'json'; appliedHeader?: string; previouslyApplied?: Record<string, string>; provider?: string } = {},
|
||||
opts: { format?: 'text' | 'json'; appliedHeader?: string; previouslyApplied?: Record<string, string>; appliedFixes?: AppliedFix[]; provider?: string } = {},
|
||||
): Promise<void> {
|
||||
const format = opts.format ?? 'text'
|
||||
if (projects.length === 0 && format === 'text') {
|
||||
|
|
@ -3497,12 +3522,12 @@ export async function runOptimize(
|
|||
const callCount = projects.reduce((s, p) => s + p.totalApiCalls, 0)
|
||||
|
||||
if (format === 'json') {
|
||||
console.log(JSON.stringify(buildOptimizeJsonReport(projects, periodLabel, result, dateRange), null, 2))
|
||||
console.log(JSON.stringify(buildOptimizeJsonReport(projects, periodLabel, result, dateRange, opts.appliedFixes), null, 2))
|
||||
return
|
||||
}
|
||||
|
||||
const { topReworkedFiles, coachingNotes } = buildWorkflowReport(projects)
|
||||
const output = renderOptimize(findings, costRate, periodLabel, periodCost, sessions.length, callCount, healthScore, healthGrade, topReworkedFiles, coachingNotes, opts.appliedHeader, opts.previouslyApplied, result.modelRecommendations)
|
||||
const output = renderOptimize(findings, costRate, periodLabel, periodCost, sessions.length, callCount, healthScore, healthGrade, topReworkedFiles, coachingNotes, opts.appliedHeader, opts.previouslyApplied, result.modelRecommendations, opts.appliedFixes)
|
||||
console.log(output)
|
||||
}
|
||||
|
||||
|
|
@ -3511,6 +3536,7 @@ export function buildOptimizeJsonReport(
|
|||
periodLabel: string,
|
||||
result: OptimizeResult,
|
||||
dateRange?: DateRange,
|
||||
appliedFixes: AppliedFix[] = [],
|
||||
): OptimizeJsonReport {
|
||||
const sessions = projects.flatMap(p => p.sessions)
|
||||
const periodCostUSD = projects.reduce((s, p) => s + p.totalCostUSD, 0)
|
||||
|
|
@ -3557,5 +3583,15 @@ export function buildOptimizeJsonReport(
|
|||
})),
|
||||
...buildWorkflowReport(projects),
|
||||
modelRecommendations: result.modelRecommendations,
|
||||
appliedFixes: appliedFixes.map(f => ({
|
||||
id: f.id,
|
||||
kind: f.kind,
|
||||
findingId: f.findingId,
|
||||
appliedAt: f.appliedAt,
|
||||
verdict: f.verdict,
|
||||
estimatedTokens: f.estimatedTokens,
|
||||
realizedTokens: f.realizedTokens,
|
||||
undoCommand: f.undoCommand,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,12 +5,14 @@ import { join } from 'node:path'
|
|||
|
||||
import { journalPath } from '../src/act/journal.js'
|
||||
import {
|
||||
autoRevertNoEffect,
|
||||
buildActReportJson,
|
||||
buildOptimizeAppliedHeader,
|
||||
captureBaseline,
|
||||
computeActReport,
|
||||
renderActReport,
|
||||
} from '../src/act/report.js'
|
||||
import { formatAppliedFix, REPORT_MIN_AGE_DAYS } from '../src/act/types.js'
|
||||
import type { ActionRecord } from '../src/act/types.js'
|
||||
import type { WasteFinding } from '../src/optimize.js'
|
||||
import type { ClassifiedTurn, ProjectSummary } from '../src/types.js'
|
||||
|
|
@ -762,3 +764,129 @@ describe('defer baseline capture', () => {
|
|||
expect(b).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('applied-fix verdicts', () => {
|
||||
const fixOf = async (records: ActionRecord[], projects: ProjectSummary[]) => {
|
||||
const actionsDir = await writeJournal(records)
|
||||
const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load(projects) })
|
||||
return { report, fixes: report.appliedFixes, actionsDir }
|
||||
}
|
||||
|
||||
it('calls a fix that realized its whole window estimate "worked"', async () => {
|
||||
const { fixes } = await fixOf([mcpRecord()], [projectOf(sessionsAt(20, daysAgo(5)))])
|
||||
expect(fixes).toHaveLength(1)
|
||||
expect(fixes[0]!.verdict).toBe('worked')
|
||||
expect(fixes[0]!.estimatedTokens).toBe(40_000)
|
||||
expect(fixes[0]!.realizedTokens).toBe(40_000)
|
||||
expect(fixes[0]!.undoCommand).toBe('codeburn act undo a1')
|
||||
})
|
||||
|
||||
it('holds the worked/partial boundary at the 70% ratio', async () => {
|
||||
const rec = mcpRecord({ kind: 'mcp-project-scope' })
|
||||
// 14 of 20 sessions saved = exactly 70% of the window estimate.
|
||||
const at70 = await fixOf(
|
||||
[rec],
|
||||
[projectOf([...sessionsAt(14, daysAgo(5)), ...sessionsAt(6, daysAgo(4), { mcpInventory: ['mcp__brave-search__search'] })])],
|
||||
)
|
||||
expect(at70.fixes[0]!.verdict).toBe('worked')
|
||||
|
||||
const below = await fixOf(
|
||||
[rec],
|
||||
[projectOf([...sessionsAt(13, daysAgo(5)), ...sessionsAt(7, daysAgo(4), { mcpInventory: ['mcp__brave-search__search'] })])],
|
||||
)
|
||||
expect(below.fixes[0]!.verdict).toBe('partial')
|
||||
expect(formatAppliedFix(below.fixes[0]!)).toContain('-35% vs estimate')
|
||||
})
|
||||
|
||||
it('calls a fix that realized nothing "no-effect" and offers the undo', async () => {
|
||||
const rec = mcpRecord({ kind: 'mcp-project-scope' })
|
||||
const stillLoading = sessionsAt(20, daysAgo(5), { mcpInventory: ['mcp__brave-search__search'] })
|
||||
const { fixes } = await fixOf([rec], [projectOf(stillLoading)])
|
||||
expect(fixes[0]!.verdict).toBe('no-effect')
|
||||
expect(fixes[0]!.realizedTokens).toBe(0)
|
||||
expect(formatAppliedFix(fixes[0]!)).toContain('did not help. Revert: codeburn act undo a1')
|
||||
})
|
||||
|
||||
it('treats an estimate of zero as worked only when something was realized', async () => {
|
||||
const zeroEstimate = { windowDays: 14, capturedAt: daysAgo(10), estimatedTokens: 0, sessions: 0, metrics: {} }
|
||||
const { fixes } = await fixOf(
|
||||
[mcpRecord({ baseline: { ...zeroEstimate, metrics: { 'brave-search': 2000 } } })],
|
||||
[projectOf(sessionsAt(20, daysAgo(5)))],
|
||||
)
|
||||
expect(fixes[0]!.estimatedTokens).toBe(40_000)
|
||||
expect(fixes[0]!.verdict).toBe('worked')
|
||||
})
|
||||
|
||||
it('leaves entries younger than the measurement window pending', async () => {
|
||||
const { fixes } = await fixOf([mcpRecord({ at: daysAgo(1) })], [projectOf(sessionsAt(20, daysAgo(1)))])
|
||||
expect(fixes[0]!.verdict).toBe('pending')
|
||||
expect(formatAppliedFix(fixes[0]!)).toBe(`unused-mcp (1d ago): measuring, check back after ${REPORT_MIN_AGE_DAYS} days`)
|
||||
})
|
||||
|
||||
it('keeps a user-reverted entry out of the verdicts and carries its note', async () => {
|
||||
const back = sessionsAt(20, daysAgo(5), { mcpInventory: ['mcp__brave-search__search'] })
|
||||
const { fixes } = await fixOf([mcpRecord()], [projectOf(back)])
|
||||
expect(fixes[0]!.verdict).toBe('pending')
|
||||
expect(fixes[0]!.note).toMatch(/reverted by user/)
|
||||
})
|
||||
|
||||
it('drops undone journal entries entirely', async () => {
|
||||
const rec = mcpRecord()
|
||||
const { fixes } = await fixOf(
|
||||
[rec, { ...rec, status: 'undone', undoneAt: daysAgo(2) }],
|
||||
[projectOf(sessionsAt(20, daysAgo(5)))],
|
||||
)
|
||||
expect(fixes).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('never judges a correlation-only kind, so --auto-revert can never touch it', async () => {
|
||||
const { fixes } = await fixOf([modelDefaultRecord()], [modelProject('app', '/tmp/app', 'candidate-model', 20, 19)])
|
||||
expect(fixes[0]!.verdict).toBe('pending')
|
||||
})
|
||||
})
|
||||
|
||||
describe('autoRevertNoEffect', () => {
|
||||
const noEffect = (over: Partial<ActionRecord> = {}) => ({
|
||||
...mcpRecord({ kind: 'mcp-project-scope', ...over }),
|
||||
})
|
||||
|
||||
it('undoes the no-effect entries and leaves the rest alone', async () => {
|
||||
const worked = noEffect({ id: 'keep1', findingId: 'kept' })
|
||||
const actionsDir = await writeJournal([noEffect(), worked])
|
||||
const stillLoading = sessionsAt(20, daysAgo(5), { mcpInventory: ['mcp__brave-search__search'] })
|
||||
const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(stillLoading)]) })
|
||||
// Both are no-effect here; pin that only the ones we hand over get undone.
|
||||
const target = report.appliedFixes.filter(f => f.id === 'a1')
|
||||
const { lines, revertedIds } = await autoRevertNoEffect(target, { actionsDir })
|
||||
|
||||
expect([...revertedIds]).toEqual(['a1'])
|
||||
expect(lines).toEqual(['Reverted a1: Remove an MCP server from config'])
|
||||
const after = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(stillLoading)]) })
|
||||
expect(after.appliedFixes.map(f => f.id)).toEqual(['keep1'])
|
||||
})
|
||||
|
||||
it('never auto-reverts a CLAUDE.md rule, it prints the undo command instead', async () => {
|
||||
const rec = mcpRecord({
|
||||
id: 'cm1',
|
||||
kind: 'claude-md-rule',
|
||||
findingId: 'read-edit-ratio',
|
||||
baseline: { windowDays: 14, capturedAt: daysAgo(10), estimatedTokens: 10_000, sessions: 20, metrics: { reads: 10, edits: 10 } },
|
||||
})
|
||||
const actionsDir = await writeJournal([rec])
|
||||
const sessions = sessionsAt(20, daysAgo(5), { toolBreakdown: { Edit: { calls: 10, tokens: 0 }, Read: { calls: 10, tokens: 0 } } as never })
|
||||
const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessions)]) })
|
||||
|
||||
expect(report.appliedFixes[0]!.verdict).toBe('no-effect')
|
||||
const { lines, revertedIds } = await autoRevertNoEffect(report.appliedFixes, { actionsDir })
|
||||
expect(revertedIds.size).toBe(0)
|
||||
expect(lines).toEqual(['Not auto-reverted: read-edit-ratio edits a CLAUDE.md. Revert: codeburn act undo cm1'])
|
||||
})
|
||||
|
||||
it('ignores partial and pending entries', async () => {
|
||||
const actionsDir = await writeJournal([mcpRecord({ at: daysAgo(1) })])
|
||||
const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(1)))]) })
|
||||
const { lines, revertedIds } = await autoRevertNoEffect(report.appliedFixes, { actionsDir })
|
||||
expect(lines).toEqual([])
|
||||
expect(revertedIds.size).toBe(0)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -433,11 +433,19 @@ describe('runOptimizeApply end-to-end', () => {
|
|||
expect(out).toContain(`Applied ${shortId(rec.id)}`)
|
||||
expect(out).toContain(`Undo anytime: codeburn act undo ${shortId(rec.id)}`)
|
||||
}
|
||||
expect(out).toContain('CodeBurn will re-measure these on your next optimize run after 3 days.')
|
||||
expect(JSON.parse(await readFile(join(fx.home, '.claude.json'), 'utf-8')).mcpServers).toEqual({})
|
||||
expect(existsSync(join(fx.home, '.claude', 'skills', '.archived', 'foo'))).toBe(true)
|
||||
expect(existsSync(join(fx.home, '.zshrc'))).toBe(true)
|
||||
})
|
||||
|
||||
it('does not promise a re-measure when nothing was applied', async () => {
|
||||
const { fx, findings } = await threeFindingFixture()
|
||||
const io = makeIo('q\n')
|
||||
await runOptimizeApply([], undefined, applyOpts(fx, io, { findings }))
|
||||
expect(io.stdout()).not.toContain('re-measure')
|
||||
})
|
||||
|
||||
it('interactive pick "2" applies only the second plan', async () => {
|
||||
const { fx, findings } = await threeFindingFixture()
|
||||
const io = makeIo('2\n')
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ import {
|
|||
type OptimizeResult,
|
||||
} from '../src/optimize.js'
|
||||
import type { ProjectSummary } from '../src/types.js'
|
||||
import type { AppliedFix } from '../src/act/types.js'
|
||||
|
||||
function call(name: string, input: Record<string, unknown>, sessionId = 's1', project = 'p1'): ToolCall {
|
||||
return { name, input, sessionId, project }
|
||||
|
|
@ -1320,6 +1321,7 @@ describe('buildOptimizeJsonReport', () => {
|
|||
expect(classes.reduce((s, c) => s + c.tokensSaved, 0)).toBe(report.summary.potentialSavingsTokens)
|
||||
expect(classes.reduce((s, c) => s + c.savingsUSD, 0)).toBeCloseTo(report.summary.potentialSavingsCostUSD, 10)
|
||||
expect(classes.reduce((s, c) => s + c.count, 0)).toBe(report.summary.findingCount)
|
||||
expect(report.appliedFixes).toEqual([])
|
||||
expect(report.findings[0]).toMatchObject({
|
||||
title: 'Trim stale context',
|
||||
severity: 'medium',
|
||||
|
|
@ -1374,3 +1376,56 @@ describe('renderOptimize grouping', () => {
|
|||
expect(out).not.toContain('Estimates only.')
|
||||
})
|
||||
})
|
||||
|
||||
describe('renderOptimize applied-fixes section', () => {
|
||||
const plain = (s: string): string => s.replace(/\[[0-9;]*m/g, '')
|
||||
|
||||
function fixture(over: Partial<AppliedFix>): AppliedFix {
|
||||
return {
|
||||
id: 'abcdef12',
|
||||
kind: 'mcp-remove',
|
||||
findingId: 'unused-mcp',
|
||||
appliedAt: '2026-05-01T00:00:00.000Z',
|
||||
ageDays: 4,
|
||||
verdict: 'worked',
|
||||
estimatedTokens: 300_000,
|
||||
realizedTokens: 280_000,
|
||||
note: '',
|
||||
undoCommand: 'codeburn act undo abcdef12',
|
||||
...over,
|
||||
}
|
||||
}
|
||||
|
||||
const findings: WasteFinding[] = [{
|
||||
id: 'bash-output-cap',
|
||||
title: 'Cap bash output',
|
||||
explanation: 'why',
|
||||
impact: 'medium',
|
||||
tokensSaved: 1000,
|
||||
fix: { type: 'paste', destination: 'prompt', label: 'ask', text: 'ask' },
|
||||
}]
|
||||
|
||||
const render = (appliedFixes: AppliedFix[], f = findings): string =>
|
||||
plain(renderOptimize(f, 0.00001, '7 Days', 10, 5, 100, 80, 'B', [], [], undefined, undefined, undefined, appliedFixes))
|
||||
|
||||
it('renders one line per verdict with its own glyph', () => {
|
||||
const out = render([
|
||||
fixture({}),
|
||||
fixture({ id: 'b', findingId: 'mcp-defer-threshold', verdict: 'partial', ageDays: 3, estimatedTokens: 600_000, realizedTokens: 420_000 }),
|
||||
fixture({ id: 'c', findingId: 'bash-output-cap', verdict: 'no-effect', ageDays: 5, estimatedTokens: 41_000, realizedTokens: 0, undoCommand: 'codeburn act undo cccccccc' }),
|
||||
fixture({ id: 'd', findingId: 'mcp-remove-linear', verdict: 'pending', ageDays: 1 }),
|
||||
])
|
||||
|
||||
expect(out).toContain('Applied fixes')
|
||||
expect(out).toContain('\u2713 unused-mcp (4d ago): est. 300.0K -> measured 280.0K')
|
||||
expect(out).toContain('~ mcp-defer-threshold (3d ago): est. 600.0K -> measured 420.0K (-30% vs estimate)')
|
||||
expect(out).toContain('\u2717 bash-output-cap (5d ago): est. 41.0K -> measured 0 - did not help. Revert: codeburn act undo cccccccc')
|
||||
expect(out).toContain('\u2026 mcp-remove-linear (1d ago): measuring, check back after 3 days')
|
||||
})
|
||||
|
||||
it('shows the section on a clean setup too, and omits it when nothing is applied', () => {
|
||||
expect(render([fixture({})], [])).toContain('Applied fixes')
|
||||
expect(render([])).not.toContain('Applied fixes')
|
||||
expect(render([], [])).not.toContain('Applied fixes')
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue