mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-07-10 01:29:41 +00:00
feat(act): realized savings measurement (codeburn act report) (#606)
Capture a trailing-14-day before-baseline when a fix is applied and re-measure it against the post-apply window so optimize can show realized numbers next to estimates. - ActionBaseline (windowDays, capturedAt, estimatedTokens, sessions, metrics) persisted by runAction; captured in the optimize --apply flow and at guard install time. - codeburn act report [--json]: applied, not-undone actions older than 3 days, re-running the detectors over apply-date-to-now (capped 30 days). Per-kind realized deltas: MCP/archive tokens-per-session times saved sessions with reverted-by-user detection; read-edit deficit reduction; guard yield split labeled correlation. Bash cap is marked not measurable (result sizes are not retained). Low confidence under 20 post-window sessions or past a 2x volume shift. Realized numbers rounded down, estimate kept visible. - optimize gains one header line only when a measured action exists, and appends "(previously applied <date>, re-flagged)" to re-triggered findings. No change for users with no applied actions. Reuses scanAndDetect helpers over a date-bounded range; exports the token constants and read/edit tool sets rather than duplicating the math.
This commit is contained in:
AgentSeal
parent
be2470d0e5
commit
ad471f3d8c
8 changed files with 932 additions and 16 deletions
|
|
@ -2,6 +2,7 @@ import type { Command } from 'commander'
|
|||
import { renderTable } from '../text-table.js'
|
||||
import { defaultActionsDir, readRecords, shortId } from './journal.js'
|
||||
import { DriftError, undoAction } from './undo.js'
|
||||
import { buildActReportJson, computeActReport, renderActReport } from './report.js'
|
||||
|
||||
function formatWhen(at: string): string {
|
||||
return at.replace('T', ' ').slice(0, 16)
|
||||
|
|
@ -63,4 +64,22 @@ export function registerActCommands(program: Command): void {
|
|||
process.exitCode = 1
|
||||
}
|
||||
})
|
||||
|
||||
act
|
||||
.command('report')
|
||||
.description('Realized vs estimated savings for applied actions older than 3 days')
|
||||
.option('--json', 'Output the realized report as JSON')
|
||||
.action(async (opts: { json?: boolean }) => {
|
||||
try {
|
||||
const report = await computeActReport()
|
||||
if (opts.json) {
|
||||
console.log(JSON.stringify(buildActReportJson(report), null, 2))
|
||||
return
|
||||
}
|
||||
console.log(renderActReport(report))
|
||||
} catch (err) {
|
||||
console.error(err instanceof Error ? err.message : String(err))
|
||||
process.exitCode = 1
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -162,6 +162,15 @@ export async function runOptimizeApply(
|
|||
return
|
||||
}
|
||||
|
||||
// Stamp a trailing-14-day before-baseline onto each plan so runAction
|
||||
// persists it and `act report` can measure realized savings later. Best
|
||||
// effort: a scan failure leaves the baseline absent (reported "not
|
||||
// measurable"), never blocking the apply.
|
||||
try {
|
||||
const { captureBaselinesForPlans } = await import('./report.js')
|
||||
await captureBaselinesForPlans(selected)
|
||||
} catch { /* baseline is optional; apply proceeds without it */ }
|
||||
|
||||
print()
|
||||
for (const fp of selected) {
|
||||
try {
|
||||
|
|
|
|||
537
src/act/report.ts
Normal file
537
src/act/report.ts
Normal file
|
|
@ -0,0 +1,537 @@
|
|||
import { existsSync } from 'fs'
|
||||
import type { DateRange, ProjectSummary, SessionSummary } from '../types.js'
|
||||
import type { ActionBaseline, ActionKind, ActionRecord } from './types.js'
|
||||
import type { FindingPlan } from './plans.js'
|
||||
import {
|
||||
AVG_TOKENS_PER_READ,
|
||||
HEALTHY_READ_EDIT_RATIO,
|
||||
TOKENS_PER_MCP_TOOL,
|
||||
TOOLS_PER_MCP_SERVER,
|
||||
TOKENS_PER_SKILL_DEF,
|
||||
TOKENS_PER_AGENT_DEF,
|
||||
TOKENS_PER_COMMAND_DEF,
|
||||
READ_TOOL_NAMES,
|
||||
EDIT_TOOL_NAMES,
|
||||
aggregateMcpCoverage,
|
||||
computeInputCostRate,
|
||||
type McpServerCoverage,
|
||||
type WasteFinding,
|
||||
} from '../optimize.js'
|
||||
import { parseAllSessions } from '../parser.js'
|
||||
import { computeYield, type YieldSummary } from '../yield.js'
|
||||
import { defaultActionsDir, readRecords } from './journal.js'
|
||||
import { renderTable } from '../text-table.js'
|
||||
import { formatTokens } from '../format.js'
|
||||
import { formatCost } from '../currency.js'
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000
|
||||
const WINDOW_CAP_DAYS = 30
|
||||
const BASELINE_WINDOW_DAYS = 14
|
||||
const REPORT_MIN_AGE_DAYS = 3
|
||||
const MIN_POST_WINDOW_SESSIONS = 20
|
||||
const VOLUME_SHIFT_FACTOR = 2
|
||||
|
||||
// Encode the epic's honest-accounting rules where they are seen: each kind
|
||||
// measures only its own metric, guard is correlation, and realized figures are
|
||||
// rounded down with the estimate kept visible.
|
||||
const HONEST_FOOTER =
|
||||
'Each fix measures only its own metric; effects are never attributed across signals. '
|
||||
+ 'Guard rows are correlation, not attribution. Realized numbers are rounded down; the estimate stays visible.'
|
||||
|
||||
const MCP_KINDS = new Set<ActionKind>(['mcp-remove', 'mcp-project-scope'])
|
||||
const ARCHIVE_DEF_TOKENS: Partial<Record<ActionKind, number>> = {
|
||||
'archive-skill': TOKENS_PER_SKILL_DEF,
|
||||
'archive-agent': TOKENS_PER_AGENT_DEF,
|
||||
'archive-command': TOKENS_PER_COMMAND_DEF,
|
||||
}
|
||||
|
||||
export type RealizedStatus = 'measured' | 'reverted' | 'not-measurable'
|
||||
|
||||
export type ActReportRow = {
|
||||
id: string
|
||||
appliedAt: string
|
||||
date: string
|
||||
kind: ActionKind
|
||||
description: string
|
||||
estimatedTokens: number
|
||||
realizedTokens: number
|
||||
status: RealizedStatus
|
||||
confidence: 'low' | 'normal'
|
||||
note: string
|
||||
// guard-install only: yield split then vs now, labeled correlation.
|
||||
correlation?: {
|
||||
abandonedPctThen: number
|
||||
abandonedPctNow: number
|
||||
avgSessionCostThenUSD: number
|
||||
avgSessionCostNowUSD: number
|
||||
}
|
||||
}
|
||||
|
||||
export type ActReport = {
|
||||
generatedAt: string
|
||||
windowCapDays: number
|
||||
costRate: number
|
||||
rows: ActReportRow[]
|
||||
totalRealizedTokens: number
|
||||
totalRealizedCostUSD: number
|
||||
measuredCount: number
|
||||
activeCount: number
|
||||
observedDays: number
|
||||
// findingId -> earliest apply date of an active applied action; drives the
|
||||
// optimize "(previously applied ..., re-flagged)" title suffix.
|
||||
appliedByFinding: Record<string, string>
|
||||
}
|
||||
|
||||
export type ActReportOptions = {
|
||||
actionsDir?: string
|
||||
now?: Date
|
||||
cwd?: string
|
||||
loadProjects?: (range: DateRange) => Promise<ProjectSummary[]>
|
||||
computeYield?: (range: DateRange) => Promise<YieldSummary>
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared measurement helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function ageDays(iso: string, now: Date): number {
|
||||
return (now.getTime() - new Date(iso).getTime()) / DAY_MS
|
||||
}
|
||||
|
||||
function allSessions(projects: ProjectSummary[]): SessionSummary[] {
|
||||
return projects.flatMap(p => p.sessions)
|
||||
}
|
||||
|
||||
function sessionsInWindow(projects: ProjectSummary[], start: Date, end: Date): SessionSummary[] {
|
||||
const out: SessionSummary[] = []
|
||||
for (const p of projects) {
|
||||
for (const s of p.sessions) {
|
||||
if (!s.firstTimestamp) continue
|
||||
const t = new Date(s.firstTimestamp).getTime()
|
||||
if (t >= start.getTime() && t <= end.getTime()) out.push(s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
function countToolCalls(sessions: SessionSummary[], names: ReadonlySet<string>): number {
|
||||
let n = 0
|
||||
for (const s of sessions) {
|
||||
for (const [tool, data] of Object.entries(s.toolBreakdown)) {
|
||||
if (names.has(tool)) n += data.calls
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
function countBashCalls(sessions: SessionSummary[]): number {
|
||||
let n = 0
|
||||
for (const s of sessions) {
|
||||
for (const data of Object.values(s.bashBreakdown)) n += data.calls
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
function sessionLoadsAny(s: SessionSummary, servers: string[]): boolean {
|
||||
for (const fqn of s.mcpInventory ?? []) {
|
||||
const seg = fqn.split('__')[1]
|
||||
if (seg && servers.includes(seg)) return true
|
||||
}
|
||||
for (const server of Object.keys(s.mcpBreakdown)) {
|
||||
if (servers.includes(server)) return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function countSessionsLoading(projects: ProjectSummary[], servers: string[]): number {
|
||||
return allSessions(projects).filter(s => sessionLoadsAny(s, servers)).length
|
||||
}
|
||||
|
||||
// A kind whose realized effect is a token saving (everything except guard,
|
||||
// which is a dollars/yield correlation, and out-of-scope kinds).
|
||||
function isTokenKind(kind: ActionKind): boolean {
|
||||
return kind !== 'guard-install' && kind !== 'guard-uninstall' && kind !== 'model-default'
|
||||
}
|
||||
|
||||
function confidenceFor(afterSessions: number, baseline: ActionBaseline, afterStart: Date, now: Date): 'low' | 'normal' {
|
||||
if (afterSessions < MIN_POST_WINDOW_SESSIONS) return 'low'
|
||||
if (baseline.sessions > 0 && baseline.windowDays > 0) {
|
||||
const afterDays = Math.max((now.getTime() - afterStart.getTime()) / DAY_MS, 1)
|
||||
const shift = (afterSessions / afterDays) / (baseline.sessions / baseline.windowDays)
|
||||
if (shift > VOLUME_SHIFT_FACTOR || shift < 1 / VOLUME_SHIFT_FACTOR) return 'low'
|
||||
}
|
||||
return 'normal'
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Per-kind realized deltas
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function mcpRow(
|
||||
base: ActReportRow, rec: ActionRecord, sessions: SessionSummary[],
|
||||
baseline: ActionBaseline, afterStart: Date, now: Date,
|
||||
): ActReportRow {
|
||||
const servers = Object.keys(baseline.metrics)
|
||||
const perSessionTokens = Object.values(baseline.metrics).reduce((a, b) => a + b, 0)
|
||||
if (servers.length === 0 || perSessionTokens === 0) return { ...base, note: 'not measurable: empty baseline' }
|
||||
const stillLoading = sessions.filter(s => sessionLoadsAny(s, servers)).length
|
||||
const confidence = confidenceFor(sessions.length, baseline, afterStart, now)
|
||||
if (rec.kind === 'mcp-remove' && stillLoading > 0) {
|
||||
return {
|
||||
...base,
|
||||
status: 'reverted',
|
||||
confidence,
|
||||
note: `reverted by user: ${servers.join(', ')} loaded again in ${stillLoading} post-apply session${stillLoading === 1 ? '' : 's'}`,
|
||||
}
|
||||
}
|
||||
const savedSessions = Math.max(0, sessions.length - stillLoading)
|
||||
return { ...base, status: 'measured', realizedTokens: Math.floor(perSessionTokens * savedSessions), confidence }
|
||||
}
|
||||
|
||||
function archiveRow(
|
||||
base: ActReportRow, rec: ActionRecord, sessions: SessionSummary[],
|
||||
baseline: ActionBaseline, afterStart: Date, now: Date,
|
||||
): ActReportRow {
|
||||
const perSessionTokens = Object.values(baseline.metrics).reduce((a, b) => a + b, 0)
|
||||
if (perSessionTokens === 0) return { ...base, note: 'not measurable: empty baseline' }
|
||||
const confidence = confidenceFor(sessions.length, baseline, afterStart, now)
|
||||
const restored = rec.changes.some(c => c.op === 'move' && existsSync(c.path))
|
||||
if (restored) {
|
||||
return { ...base, status: 'reverted', confidence, note: 'reverted by user: an archived item was moved back into place' }
|
||||
}
|
||||
return { ...base, status: 'measured', realizedTokens: Math.floor(perSessionTokens * sessions.length), confidence }
|
||||
}
|
||||
|
||||
function readEditRow(
|
||||
base: ActReportRow, sessions: SessionSummary[],
|
||||
baseline: ActionBaseline, afterStart: Date, now: Date,
|
||||
): ActReportRow {
|
||||
const editsNow = countToolCalls(sessions, EDIT_TOOL_NAMES)
|
||||
const readsNow = countToolCalls(sessions, READ_TOOL_NAMES)
|
||||
const editsThen = baseline.metrics['edits'] ?? 0
|
||||
const readsThen = baseline.metrics['reads'] ?? 0
|
||||
if (editsThen <= 0 || editsNow <= 0) return { ...base, note: 'not measurable: not enough edit activity to compare' }
|
||||
const ratioThen = readsThen / editsThen
|
||||
const ratioNow = readsNow / editsNow
|
||||
// Detector estimate math: reads short of HEALTHY_READ_EDIT_RATIO per edit are
|
||||
// the retry-prone deficit. Credit only the reduction in that deficit, scaled
|
||||
// by current edits; a worsened ratio claims nothing.
|
||||
const deficitThen = Math.max(HEALTHY_READ_EDIT_RATIO - ratioThen, 0)
|
||||
const deficitNow = Math.max(HEALTHY_READ_EDIT_RATIO - ratioNow, 0)
|
||||
const realized = Math.floor(Math.max(0, deficitThen - deficitNow) * editsNow * AVG_TOKENS_PER_READ)
|
||||
return {
|
||||
...base,
|
||||
status: 'measured',
|
||||
realizedTokens: realized,
|
||||
confidence: confidenceFor(sessions.length, baseline, afterStart, now),
|
||||
note: `read:edit ${ratioThen.toFixed(1)}:1 -> ${ratioNow.toFixed(1)}:1`,
|
||||
}
|
||||
}
|
||||
|
||||
async function guardRow(
|
||||
base: ActReportRow, afterStart: Date, now: Date,
|
||||
baseline: ActionBaseline, opts: ActReportOptions,
|
||||
): Promise<ActReportRow> {
|
||||
const abandonedThen = baseline.metrics['abandonedPct']
|
||||
const avgThen = baseline.metrics['avgSessionCostUSD']
|
||||
if (abandonedThen === undefined || avgThen === undefined) {
|
||||
return { ...base, note: 'not measurable: no yield baseline captured at install time' }
|
||||
}
|
||||
const yieldFn = opts.computeYield ?? ((range: DateRange) => computeYield(range, opts.cwd ?? process.cwd()))
|
||||
let summary: YieldSummary
|
||||
try {
|
||||
summary = await yieldFn({ start: afterStart, end: now })
|
||||
} catch {
|
||||
return { ...base, note: 'not measurable: yield could not be computed for the post-apply window' }
|
||||
}
|
||||
const abandonedNow = summary.total.cost > 0 ? Math.round((summary.abandoned.cost / summary.total.cost) * 100) : 0
|
||||
const avgNow = summary.total.sessions > 0 ? summary.total.cost / summary.total.sessions : 0
|
||||
return {
|
||||
...base,
|
||||
status: 'measured',
|
||||
confidence: summary.total.sessions < MIN_POST_WINDOW_SESSIONS ? 'low' : 'normal',
|
||||
note: 'correlation, not attribution',
|
||||
correlation: {
|
||||
abandonedPctThen: abandonedThen,
|
||||
abandonedPctNow: abandonedNow,
|
||||
avgSessionCostThenUSD: avgThen,
|
||||
avgSessionCostNowUSD: avgNow,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async function computeRow(rec: ActionRecord, sessions: SessionSummary[], afterStart: Date, now: Date, opts: ActReportOptions): Promise<ActReportRow> {
|
||||
const base: ActReportRow = {
|
||||
id: rec.id,
|
||||
appliedAt: rec.at,
|
||||
date: rec.at.slice(0, 10),
|
||||
kind: rec.kind,
|
||||
description: rec.description,
|
||||
estimatedTokens: rec.baseline?.estimatedTokens ?? 0,
|
||||
realizedTokens: 0,
|
||||
status: 'not-measurable',
|
||||
confidence: 'normal',
|
||||
note: '',
|
||||
}
|
||||
const baseline = rec.baseline
|
||||
if (!baseline) return { ...base, note: 'not measurable: no baseline captured at apply time' }
|
||||
|
||||
if (MCP_KINDS.has(rec.kind)) return mcpRow(base, rec, sessions, baseline, afterStart, now)
|
||||
if (rec.kind in ARCHIVE_DEF_TOKENS) return archiveRow(base, rec, sessions, baseline, afterStart, now)
|
||||
if (rec.kind === 'claude-md-rule') return readEditRow(base, sessions, baseline, afterStart, now)
|
||||
if (rec.kind === 'shell-config') return { ...base, note: 'not measurable: bash result token sizes are not retained in the summary' }
|
||||
if (rec.kind === 'guard-install') return guardRow(base, afterStart, now, baseline, opts)
|
||||
return { ...base, note: 'not measurable: kind is not tracked by act report' }
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Report
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function computeActReport(opts: ActReportOptions = {}): Promise<ActReport> {
|
||||
const now = opts.now ?? new Date()
|
||||
const records = await readRecords(opts.actionsDir ?? defaultActionsDir())
|
||||
const active = records.filter(r => r.status === 'applied')
|
||||
|
||||
const appliedByFinding: Record<string, string> = {}
|
||||
for (const r of active) {
|
||||
if (!r.findingId) continue
|
||||
const date = r.at.slice(0, 10)
|
||||
const prev = appliedByFinding[r.findingId]
|
||||
if (!prev || date < prev) appliedByFinding[r.findingId] = date
|
||||
}
|
||||
|
||||
const empty: ActReport = {
|
||||
generatedAt: now.toISOString(),
|
||||
windowCapDays: WINDOW_CAP_DAYS,
|
||||
costRate: 0,
|
||||
rows: [],
|
||||
totalRealizedTokens: 0,
|
||||
totalRealizedCostUSD: 0,
|
||||
measuredCount: 0,
|
||||
activeCount: active.length,
|
||||
observedDays: 0,
|
||||
appliedByFinding,
|
||||
}
|
||||
|
||||
const eligible = active.filter(r => ageDays(r.at, now) > REPORT_MIN_AGE_DAYS)
|
||||
if (eligible.length === 0) return empty
|
||||
|
||||
const windowStart = new Date(now.getTime() - WINDOW_CAP_DAYS * DAY_MS)
|
||||
const loadProjects = opts.loadProjects ?? ((range: DateRange) => parseAllSessions(range, 'claude'))
|
||||
const projects = await loadProjects({ start: windowStart, end: now })
|
||||
const costRate = computeInputCostRate(projects)
|
||||
|
||||
const rows: ActReportRow[] = []
|
||||
for (const rec of eligible) {
|
||||
const afterStart = new Date(Math.max(new Date(rec.at).getTime(), windowStart.getTime()))
|
||||
rows.push(await computeRow(rec, sessionsInWindow(projects, afterStart, now), afterStart, now, opts))
|
||||
}
|
||||
|
||||
const measuredRows = rows.filter(r => r.status === 'measured' && isTokenKind(r.kind))
|
||||
const totalRealizedTokens = measuredRows.reduce((s, r) => s + r.realizedTokens, 0)
|
||||
const observedDays = Math.min(
|
||||
WINDOW_CAP_DAYS,
|
||||
measuredRows.reduce((mx, r) => Math.max(mx, Math.ceil(ageDays(r.appliedAt, now))), 0),
|
||||
)
|
||||
|
||||
return {
|
||||
generatedAt: now.toISOString(),
|
||||
windowCapDays: WINDOW_CAP_DAYS,
|
||||
costRate,
|
||||
rows,
|
||||
totalRealizedTokens,
|
||||
totalRealizedCostUSD: totalRealizedTokens * costRate,
|
||||
measuredCount: measuredRows.length,
|
||||
activeCount: active.length,
|
||||
observedDays,
|
||||
appliedByFinding,
|
||||
}
|
||||
}
|
||||
|
||||
export function buildOptimizeAppliedHeader(report: ActReport): string | null {
|
||||
if (report.measuredCount === 0) return null
|
||||
const cost = report.costRate > 0 ? ` (~${formatCost(report.totalRealizedCostUSD)})` : ''
|
||||
const days = report.observedDays
|
||||
return `Applied fixes: ${report.activeCount} active, realized ~${formatTokens(report.totalRealizedTokens)} tokens${cost} over ${days} day${days === 1 ? '' : 's'}. Details: codeburn act report`
|
||||
}
|
||||
|
||||
function realizedCell(r: ActReportRow): string {
|
||||
if (r.status === 'reverted') return 'reverted'
|
||||
if (r.status === 'not-measurable') return 'not measurable'
|
||||
if (r.correlation) return `abandoned ${r.correlation.abandonedPctThen}% -> ${r.correlation.abandonedPctNow}% (corr.)`
|
||||
return formatTokens(r.realizedTokens)
|
||||
}
|
||||
|
||||
export function renderActReport(report: ActReport): string {
|
||||
if (report.rows.length === 0) {
|
||||
const lines = ['', ' No applied actions to measure yet.']
|
||||
if (report.activeCount > 0) {
|
||||
lines.push(` ${report.activeCount} action${report.activeCount === 1 ? '' : 's'} applied; measurement starts after ${REPORT_MIN_AGE_DAYS} days.`)
|
||||
} else {
|
||||
lines.push(' Apply fixes with codeburn optimize --apply, then check back after a few days.')
|
||||
}
|
||||
lines.push('')
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
const rows = report.rows.map(r => [
|
||||
r.date,
|
||||
r.description,
|
||||
r.estimatedTokens > 0 ? formatTokens(r.estimatedTokens) : '-',
|
||||
realizedCell(r),
|
||||
r.status === 'measured' && isTokenKind(r.kind) ? r.confidence : '-',
|
||||
])
|
||||
const totalCost = report.costRate > 0 ? ` (~${formatCost(report.totalRealizedCostUSD)})` : ''
|
||||
rows.push(['', 'Total realized', '', `${formatTokens(report.totalRealizedTokens)}${totalCost}`, ''])
|
||||
|
||||
const table = renderTable(
|
||||
[{ header: 'Applied' }, { header: 'Action' }, { header: 'Estimated', right: true }, { header: 'Realized', right: true }, { header: 'Confidence' }],
|
||||
rows,
|
||||
{ boldRows: new Set([rows.length - 1]) },
|
||||
)
|
||||
|
||||
const details: string[] = []
|
||||
for (const r of report.rows) {
|
||||
if (r.status === 'measured' && isTokenKind(r.kind)) continue
|
||||
if (r.note) details.push(` ${r.date} ${r.kind}: ${r.note}`)
|
||||
if (r.correlation) {
|
||||
details.push(` avg session cost ${formatCost(r.correlation.avgSessionCostThenUSD)} -> ${formatCost(r.correlation.avgSessionCostNowUSD)}`)
|
||||
}
|
||||
}
|
||||
|
||||
return ['', table, ...(details.length > 0 ? ['', ...details] : []), '', ' ' + HONEST_FOOTER, ''].join('\n')
|
||||
}
|
||||
|
||||
export function buildActReportJson(report: ActReport): unknown {
|
||||
return {
|
||||
generatedAt: report.generatedAt,
|
||||
windowCapDays: report.windowCapDays,
|
||||
actions: report.rows.map(r => {
|
||||
const tokenMeasured = r.status === 'measured' && isTokenKind(r.kind)
|
||||
return {
|
||||
id: r.id,
|
||||
date: r.date,
|
||||
kind: r.kind,
|
||||
description: r.description,
|
||||
estimatedTokens: r.estimatedTokens,
|
||||
realizedTokens: tokenMeasured ? r.realizedTokens : null,
|
||||
status: r.status,
|
||||
confidence: tokenMeasured ? r.confidence : null,
|
||||
note: r.note,
|
||||
...(r.correlation ? { correlation: r.correlation } : {}),
|
||||
}
|
||||
}),
|
||||
totals: {
|
||||
realizedTokens: report.totalRealizedTokens,
|
||||
realizedCostUSD: report.totalRealizedCostUSD,
|
||||
measuredActions: report.measuredCount,
|
||||
activeActions: report.activeCount,
|
||||
observedDays: report.observedDays,
|
||||
},
|
||||
footer: HONEST_FOOTER,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Baseline capture (apply time)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type CaptureCtx = {
|
||||
projects: ProjectSummary[]
|
||||
coverage: McpServerCoverage[]
|
||||
windowDays: number
|
||||
now: Date
|
||||
}
|
||||
|
||||
function mcpServersFromApply(finding: WasteFinding): string[] {
|
||||
if (finding.apply?.kind === 'mcp-remove') return finding.apply.servers
|
||||
if (finding.apply?.kind === 'mcp-project-scope') return finding.apply.servers.map(s => s.server)
|
||||
return []
|
||||
}
|
||||
|
||||
function needsConfigBaseline(kind: ActionKind): boolean {
|
||||
return MCP_KINDS.has(kind) || kind in ARCHIVE_DEF_TOKENS || kind === 'claude-md-rule' || kind === 'shell-config'
|
||||
}
|
||||
|
||||
export function captureBaseline(finding: WasteFinding, kind: ActionKind, ctx: CaptureCtx): ActionBaseline | undefined {
|
||||
const common = {
|
||||
windowDays: ctx.windowDays,
|
||||
capturedAt: ctx.now.toISOString(),
|
||||
estimatedTokens: Math.max(0, Math.round(finding.tokensSaved)),
|
||||
}
|
||||
|
||||
if (MCP_KINDS.has(kind)) {
|
||||
const servers = mcpServersFromApply(finding)
|
||||
if (servers.length === 0) return undefined
|
||||
const covByServer = new Map(ctx.coverage.map(c => [c.server, c]))
|
||||
const metrics: Record<string, number> = {}
|
||||
for (const server of servers) {
|
||||
const cov = covByServer.get(server)
|
||||
const tools = cov && cov.toolsAvailable > 0 ? cov.toolsAvailable : TOOLS_PER_MCP_SERVER
|
||||
metrics[server] = tools * TOKENS_PER_MCP_TOOL
|
||||
}
|
||||
return { ...common, sessions: countSessionsLoading(ctx.projects, servers), metrics }
|
||||
}
|
||||
|
||||
const defTokens = ARCHIVE_DEF_TOKENS[kind]
|
||||
if (defTokens !== undefined) {
|
||||
const names = finding.apply?.kind === 'archive' ? finding.apply.names : []
|
||||
if (names.length === 0) return undefined
|
||||
const metrics: Record<string, number> = {}
|
||||
for (const name of names) metrics[name] = defTokens
|
||||
return { ...common, sessions: allSessions(ctx.projects).length, metrics }
|
||||
}
|
||||
|
||||
const sessions = allSessions(ctx.projects)
|
||||
if (kind === 'claude-md-rule') {
|
||||
return { ...common, sessions: sessions.length, metrics: { reads: countToolCalls(sessions, READ_TOOL_NAMES), edits: countToolCalls(sessions, EDIT_TOOL_NAMES) } }
|
||||
}
|
||||
if (kind === 'shell-config') {
|
||||
return { ...common, sessions: sessions.length, metrics: { calls: countBashCalls(sessions) } }
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
// Scan the trailing 14 days once and stamp a baseline onto every appliable
|
||||
// plan that carries one, so runAction persists it for `act report` to diff.
|
||||
export async function captureBaselinesForPlans(
|
||||
plans: FindingPlan[],
|
||||
opts: { now?: Date; loadProjects?: (range: DateRange) => Promise<ProjectSummary[]> } = {},
|
||||
): Promise<void> {
|
||||
const applicable = plans.filter(fp => fp.plan && needsConfigBaseline(fp.plan.kind))
|
||||
if (applicable.length === 0) return
|
||||
const now = opts.now ?? new Date()
|
||||
const start = new Date(now.getTime() - BASELINE_WINDOW_DAYS * DAY_MS)
|
||||
const loadProjects = opts.loadProjects ?? ((range: DateRange) => parseAllSessions(range, 'claude'))
|
||||
const projects = await loadProjects({ start, end: now })
|
||||
const ctx: CaptureCtx = { projects, coverage: aggregateMcpCoverage(projects), windowDays: BASELINE_WINDOW_DAYS, now }
|
||||
for (const fp of applicable) {
|
||||
const baseline = captureBaseline(fp.finding, fp.plan!.kind, ctx)
|
||||
if (baseline) fp.plan!.baseline = baseline
|
||||
}
|
||||
}
|
||||
|
||||
export async function captureGuardBaseline(
|
||||
opts: { now?: Date; cwd?: string; computeYield?: (range: DateRange) => Promise<YieldSummary> } = {},
|
||||
): Promise<ActionBaseline | undefined> {
|
||||
const now = opts.now ?? new Date()
|
||||
const range = { start: new Date(now.getTime() - BASELINE_WINDOW_DAYS * DAY_MS), end: now }
|
||||
const yieldFn = opts.computeYield ?? ((r: DateRange) => computeYield(r, opts.cwd ?? process.cwd()))
|
||||
let summary: YieldSummary
|
||||
try {
|
||||
summary = await yieldFn(range)
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
return {
|
||||
windowDays: BASELINE_WINDOW_DAYS,
|
||||
capturedAt: now.toISOString(),
|
||||
estimatedTokens: 0,
|
||||
sessions: summary.total.sessions,
|
||||
metrics: {
|
||||
abandonedPct: summary.total.cost > 0 ? Math.round((summary.abandoned.cost / summary.total.cost) * 100) : 0,
|
||||
avgSessionCostUSD: summary.total.sessions > 0 ? summary.total.cost / summary.total.sessions : 0,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
@ -14,6 +14,27 @@ export type FileChange = {
|
|||
afterHash: string // sha256 of the post-apply bytes, checked for drift on undo
|
||||
}
|
||||
|
||||
// Before/after measurement captured when an action is applied, diffed against
|
||||
// the post-apply window by `act report`. `metrics` holds the kind-specific
|
||||
// numbers:
|
||||
// mcp-remove / mcp-project-scope: server name -> schema tokens per session
|
||||
// archive-skill|agent|command: item name -> definition tokens per session
|
||||
// claude-md-rule (read/edit rule): { reads, edits }
|
||||
// shell-config (bash cap): { calls }
|
||||
// guard-install: { abandonedPct, avgSessionCostUSD }
|
||||
// estimatedTokens is the finding's estimate at apply time (0 for guard, which
|
||||
// is a correlation signal, not a token estimate). sessions is the affected-scope
|
||||
// session count over the window, kept out of `metrics` so it can never collide
|
||||
// with a server literally named "sessions"; it feeds only the volume-shift
|
||||
// confidence check.
|
||||
export type ActionBaseline = {
|
||||
windowDays: number
|
||||
capturedAt: string
|
||||
estimatedTokens: number
|
||||
sessions: number
|
||||
metrics: Record<string, number>
|
||||
}
|
||||
|
||||
export type ActionRecord = {
|
||||
id: string // crypto.randomUUID()
|
||||
at: string // ISO timestamp
|
||||
|
|
@ -23,7 +44,7 @@ export type ActionRecord = {
|
|||
changes: FileChange[]
|
||||
status: 'applied' | 'undone'
|
||||
undoneAt?: string
|
||||
baseline?: Record<string, number>
|
||||
baseline?: ActionBaseline
|
||||
}
|
||||
|
||||
// expectedHash: sha256 of the raw on-disk bytes the plan's content was
|
||||
|
|
@ -41,5 +62,5 @@ export type ActionPlan = {
|
|||
description: string
|
||||
findingId?: string | null
|
||||
changes: PlannedChange[]
|
||||
baseline?: Record<string, number>
|
||||
baseline?: ActionBaseline
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,6 +41,14 @@ async function doInstall(scope: Scope, statusline: boolean): Promise<void> {
|
|||
const built = buildInstall(path, { statusline })
|
||||
for (const note of built.notes) console.log(chalk.yellow(` ! ${note}`))
|
||||
if (built.plan) {
|
||||
// Capture a trailing-14-day yield baseline so `act report` can correlate
|
||||
// the guard install against later yield. Best effort; a failure just
|
||||
// leaves the guard row "not measurable".
|
||||
try {
|
||||
const { captureGuardBaseline } = await import('../act/report.js')
|
||||
const baseline = await captureGuardBaseline({ cwd: process.cwd() })
|
||||
if (baseline) built.plan.baseline = baseline
|
||||
} catch { /* baseline is optional */ }
|
||||
const record = await runAction(built.plan)
|
||||
console.log(` Installed ${chalk.bold(shortId(record.id))} ${built.plan.description}`)
|
||||
console.log(chalk.dim(` Undo anytime: codeburn act undo ${shortId(record.id)}`))
|
||||
|
|
|
|||
15
src/main.ts
15
src/main.ts
|
|
@ -1350,7 +1350,20 @@ program
|
|||
return
|
||||
}
|
||||
assertFormat(format, ['text', 'json'], 'optimize')
|
||||
await runOptimize(projects, label, range, { format })
|
||||
if (format === 'text') {
|
||||
// Surface realized savings from applied actions. computeActReport returns
|
||||
// fast without scanning when the journal has no eligible applied actions,
|
||||
// so users who never opted in see identical output.
|
||||
const { computeActReport, buildOptimizeAppliedHeader } = await import('./act/report.js')
|
||||
const applied = await computeActReport()
|
||||
await runOptimize(projects, label, range, {
|
||||
format,
|
||||
appliedHeader: buildOptimizeAppliedHeader(applied) ?? undefined,
|
||||
previouslyApplied: applied.appliedByFinding,
|
||||
})
|
||||
} else {
|
||||
await runOptimize(projects, label, range, { format })
|
||||
}
|
||||
})
|
||||
|
||||
program
|
||||
|
|
|
|||
|
|
@ -26,12 +26,12 @@ const RED = '#F55B5B'
|
|||
// Token estimation constants
|
||||
// ============================================================================
|
||||
|
||||
const AVG_TOKENS_PER_READ = 600
|
||||
const TOKENS_PER_MCP_TOOL = 400
|
||||
const TOOLS_PER_MCP_SERVER = 5
|
||||
const TOKENS_PER_AGENT_DEF = 80
|
||||
const TOKENS_PER_SKILL_DEF = 80
|
||||
const TOKENS_PER_COMMAND_DEF = 60
|
||||
export const AVG_TOKENS_PER_READ = 600
|
||||
export const TOKENS_PER_MCP_TOOL = 400
|
||||
export const TOOLS_PER_MCP_SERVER = 5
|
||||
export const TOKENS_PER_AGENT_DEF = 80
|
||||
export const TOKENS_PER_SKILL_DEF = 80
|
||||
export const TOKENS_PER_COMMAND_DEF = 60
|
||||
const CLAUDEMD_TOKENS_PER_LINE = 13
|
||||
const BASH_TOKENS_PER_CHAR = 0.25
|
||||
|
||||
|
|
@ -48,7 +48,7 @@ const MIN_DUPLICATE_READS_TO_FLAG = 5
|
|||
const DUPLICATE_READS_HIGH_THRESHOLD = 30
|
||||
const DUPLICATE_READS_MEDIUM_THRESHOLD = 10
|
||||
const MIN_EDITS_FOR_RATIO = 10
|
||||
const HEALTHY_READ_EDIT_RATIO = 4
|
||||
export const HEALTHY_READ_EDIT_RATIO = 4
|
||||
const LOW_RATIO_HIGH_THRESHOLD = 2
|
||||
const LOW_RATIO_MEDIUM_THRESHOLD = 3
|
||||
const MIN_API_CALLS_FOR_CACHE = 10
|
||||
|
|
@ -1605,8 +1605,8 @@ export function detectBloatedClaudeMd(projectCwds: Set<string>): WasteFinding |
|
|||
}
|
||||
}
|
||||
|
||||
const READ_TOOL_NAMES = new Set(['Read', 'Grep', 'Glob', 'FileReadTool', 'GrepTool', 'GlobTool'])
|
||||
const EDIT_TOOL_NAMES = new Set(['Edit', 'Write', 'FileEditTool', 'FileWriteTool', 'NotebookEdit'])
|
||||
export const READ_TOOL_NAMES = new Set(['Read', 'Grep', 'Glob', 'FileReadTool', 'GrepTool', 'GlobTool'])
|
||||
export const EDIT_TOOL_NAMES = new Set(['Edit', 'Write', 'FileEditTool', 'FileWriteTool', 'NotebookEdit'])
|
||||
|
||||
export function detectLowReadEditRatio(calls: ToolCall[]): WasteFinding | null {
|
||||
let reads = 0
|
||||
|
|
@ -2343,7 +2343,7 @@ function sessionTrend(
|
|||
const INPUT_COST_RATIO = 0.7
|
||||
const DEFAULT_COST_PER_TOKEN = 0
|
||||
|
||||
function computeInputCostRate(projects: ProjectSummary[]): number {
|
||||
export function computeInputCostRate(projects: ProjectSummary[]): number {
|
||||
const sessions = projects.flatMap(p => p.sessions)
|
||||
const totalCost = sessions.reduce((s, sess) => s + sess.totalCostUSD, 0)
|
||||
const totalTokens = sessions.reduce((s, sess) =>
|
||||
|
|
@ -2526,6 +2526,8 @@ function renderOptimize(
|
|||
callCount: number,
|
||||
healthScore: number,
|
||||
healthGrade: HealthGrade,
|
||||
appliedHeader?: string,
|
||||
previouslyApplied?: Record<string, string>,
|
||||
): string {
|
||||
const lines: string[] = []
|
||||
lines.push('')
|
||||
|
|
@ -2539,6 +2541,7 @@ function renderOptimize(
|
|||
chalk.hex(GOLD)(formatCost(periodCost)),
|
||||
`Health: ${chalk.bold.hex(GRADE_COLORS[healthGrade])(healthGrade)}${chalk.dim(` (${healthScore}/100${issueSuffix})`)}`,
|
||||
].join(chalk.hex(DIM)(' ')))
|
||||
if (appliedHeader) lines.push(' ' + chalk.hex(GREEN)(appliedHeader))
|
||||
lines.push('')
|
||||
|
||||
if (findings.length === 0) {
|
||||
|
|
@ -2561,7 +2564,10 @@ function renderOptimize(
|
|||
lines.push('')
|
||||
|
||||
for (let i = 0; i < findings.length; i++) {
|
||||
lines.push(...renderFinding(i + 1, findings[i], costRate))
|
||||
const f = findings[i]!
|
||||
const appliedOn = previouslyApplied?.[f.id]
|
||||
const shown = appliedOn ? { ...f, title: `${f.title} (previously applied ${appliedOn}, re-flagged)` } : f
|
||||
lines.push(...renderFinding(i + 1, shown, costRate))
|
||||
}
|
||||
|
||||
lines.push(chalk.hex(DIM)(' ' + SEP.repeat(PANEL_WIDTH)))
|
||||
|
|
@ -2574,7 +2580,7 @@ export async function runOptimize(
|
|||
projects: ProjectSummary[],
|
||||
periodLabel: string,
|
||||
dateRange?: DateRange,
|
||||
opts: { format?: 'text' | 'json' } = {},
|
||||
opts: { format?: 'text' | 'json'; appliedHeader?: string; previouslyApplied?: Record<string, string> } = {},
|
||||
): Promise<void> {
|
||||
const format = opts.format ?? 'text'
|
||||
if (projects.length === 0 && format === 'text') {
|
||||
|
|
@ -2597,7 +2603,7 @@ export async function runOptimize(
|
|||
return
|
||||
}
|
||||
|
||||
const output = renderOptimize(findings, costRate, periodLabel, periodCost, sessions.length, callCount, healthScore, healthGrade)
|
||||
const output = renderOptimize(findings, costRate, periodLabel, periodCost, sessions.length, callCount, healthScore, healthGrade, opts.appliedHeader, opts.previouslyApplied)
|
||||
console.log(output)
|
||||
}
|
||||
|
||||
|
|
|
|||
303
tests/act-report.test.ts
Normal file
303
tests/act-report.test.ts
Normal file
|
|
@ -0,0 +1,303 @@
|
|||
import { afterAll, describe, expect, it } from 'vitest'
|
||||
import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
|
||||
import { journalPath } from '../src/act/journal.js'
|
||||
import {
|
||||
buildActReportJson,
|
||||
buildOptimizeAppliedHeader,
|
||||
computeActReport,
|
||||
renderActReport,
|
||||
} from '../src/act/report.js'
|
||||
import type { ActionRecord } from '../src/act/types.js'
|
||||
import type { ProjectSummary } from '../src/types.js'
|
||||
|
||||
type Session = ProjectSummary['sessions'][number]
|
||||
|
||||
const roots: string[] = []
|
||||
afterAll(async () => { for (const r of roots) await rm(r, { recursive: true, force: true }) })
|
||||
|
||||
const NOW = new Date('2026-07-01T00:00:00Z')
|
||||
function daysAgo(n: number): string {
|
||||
return new Date(NOW.getTime() - n * 86_400_000).toISOString()
|
||||
}
|
||||
|
||||
async function writeJournal(records: ActionRecord[]): Promise<string> {
|
||||
const root = await mkdtemp(join(tmpdir(), 'codeburn-act-report-'))
|
||||
roots.push(root)
|
||||
const actionsDir = join(root, 'actions')
|
||||
await mkdir(actionsDir, { recursive: true })
|
||||
await writeFile(journalPath(actionsDir), records.map(r => JSON.stringify(r)).join('\n') + (records.length ? '\n' : ''))
|
||||
return actionsDir
|
||||
}
|
||||
|
||||
function makeSession(id: string, firstTimestamp: string, over: Partial<Session> = {}): Session {
|
||||
return {
|
||||
sessionId: id,
|
||||
project: 'app',
|
||||
firstTimestamp,
|
||||
lastTimestamp: firstTimestamp,
|
||||
totalCostUSD: 1,
|
||||
totalSavingsUSD: 0,
|
||||
totalInputTokens: 1000,
|
||||
totalOutputTokens: 0,
|
||||
totalReasoningTokens: 0,
|
||||
totalCacheReadTokens: 0,
|
||||
totalCacheWriteTokens: 0,
|
||||
apiCalls: 1,
|
||||
turns: [],
|
||||
modelBreakdown: {},
|
||||
toolBreakdown: {},
|
||||
mcpBreakdown: {},
|
||||
bashBreakdown: {},
|
||||
categoryBreakdown: {} as Session['categoryBreakdown'],
|
||||
skillBreakdown: {},
|
||||
subagentBreakdown: {},
|
||||
...over,
|
||||
}
|
||||
}
|
||||
|
||||
function projectOf(sessions: Session[]): ProjectSummary {
|
||||
return {
|
||||
project: 'app',
|
||||
projectPath: '/tmp/app',
|
||||
sessions,
|
||||
totalCostUSD: sessions.reduce((s, x) => s + x.totalCostUSD, 0),
|
||||
totalSavingsUSD: 0,
|
||||
totalApiCalls: sessions.length,
|
||||
totalProxiedCostUSD: 0,
|
||||
}
|
||||
}
|
||||
|
||||
function sessionsAt(count: number, ts: string, over: Partial<Session> = {}): Session[] {
|
||||
return Array.from({ length: count }, (_, i) => makeSession(`s${i}`, ts, over))
|
||||
}
|
||||
|
||||
function mcpRecord(over: Partial<ActionRecord> = {}): ActionRecord {
|
||||
const at = daysAgo(10)
|
||||
return {
|
||||
id: 'a1',
|
||||
at,
|
||||
kind: 'mcp-remove',
|
||||
findingId: 'unused-mcp',
|
||||
description: 'Remove an MCP server from config',
|
||||
changes: [],
|
||||
status: 'applied',
|
||||
baseline: { windowDays: 14, capturedAt: at, estimatedTokens: 56_000, sessions: 28, metrics: { 'brave-search': 2000 } },
|
||||
...over,
|
||||
}
|
||||
}
|
||||
|
||||
const load = (projects: ProjectSummary[]) => async () => projects
|
||||
|
||||
describe('mcp realized delta', () => {
|
||||
it('multiplies baseline tokens-per-session by post-window sessions (exact)', async () => {
|
||||
const actionsDir = await writeJournal([mcpRecord()])
|
||||
const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(5)))]) })
|
||||
|
||||
expect(report.rows).toHaveLength(1)
|
||||
const row = report.rows[0]!
|
||||
expect(row.status).toBe('measured')
|
||||
expect(row.realizedTokens).toBe(40_000) // 2000 tokens/session * 20 sessions
|
||||
expect(row.estimatedTokens).toBe(56_000)
|
||||
expect(row.confidence).toBe('normal')
|
||||
expect(report.totalRealizedTokens).toBe(40_000)
|
||||
})
|
||||
|
||||
it('reports "reverted" with zero savings when the server reappears in the window', async () => {
|
||||
const actionsDir = await writeJournal([mcpRecord()])
|
||||
const back = sessionsAt(20, daysAgo(5), { mcpInventory: ['mcp__brave-search__search'] })
|
||||
const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(back)]) })
|
||||
|
||||
const row = report.rows[0]!
|
||||
expect(row.status).toBe('reverted')
|
||||
expect(row.realizedTokens).toBe(0)
|
||||
expect(row.note).toMatch(/reverted by user/)
|
||||
expect(report.totalRealizedTokens).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('confidence markers', () => {
|
||||
it('marks low when fewer than 20 post-window sessions', async () => {
|
||||
const actionsDir = await writeJournal([mcpRecord()])
|
||||
const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(10, daysAgo(5)))]) })
|
||||
|
||||
const row = report.rows[0]!
|
||||
expect(row.status).toBe('measured')
|
||||
expect(row.realizedTokens).toBe(20_000) // 2000 * 10
|
||||
expect(row.confidence).toBe('low')
|
||||
})
|
||||
|
||||
it('marks low when volume shifts more than 2x versus baseline', async () => {
|
||||
// 25 post-window sessions (>= 20, so not the count rule) over 10 days is
|
||||
// 2.5/day against a 1/day baseline (14 sessions / 14 days) -> 2.5x shift.
|
||||
const rec = mcpRecord({ baseline: { windowDays: 14, capturedAt: daysAgo(10), estimatedTokens: 56_000, sessions: 14, metrics: { 'brave-search': 2000 } } })
|
||||
const actionsDir = await writeJournal([rec])
|
||||
const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(25, daysAgo(5)))]) })
|
||||
|
||||
const row = report.rows[0]!
|
||||
expect(row.status).toBe('measured')
|
||||
expect(row.confidence).toBe('low')
|
||||
})
|
||||
|
||||
it('stays normal when volume is comparable to baseline', async () => {
|
||||
const actionsDir = await writeJournal([mcpRecord()]) // baseline 28/14 = 2/day
|
||||
const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(5)))]) }) // 20/10 = 2/day
|
||||
expect(report.rows[0]!.confidence).toBe('normal')
|
||||
})
|
||||
})
|
||||
|
||||
describe('eligibility', () => {
|
||||
it('excludes undone actions and actions younger than 3 days', async () => {
|
||||
const records = [
|
||||
mcpRecord({ id: 'old', at: daysAgo(10) }),
|
||||
mcpRecord({ id: 'young', at: daysAgo(1) }),
|
||||
mcpRecord({ id: 'undone', at: daysAgo(20), status: 'undone' }),
|
||||
]
|
||||
const actionsDir = await writeJournal(records)
|
||||
const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(5)))]) })
|
||||
|
||||
expect(report.rows).toHaveLength(1)
|
||||
expect(report.rows[0]!.id).toBe('old')
|
||||
expect(report.activeCount).toBe(2) // old + young are applied; undone is not
|
||||
})
|
||||
})
|
||||
|
||||
describe('read-edit realized delta', () => {
|
||||
it('credits the reduction in the read deficit using the detector estimate math', async () => {
|
||||
// Baseline ratio 1:1 (deficit 3 reads/edit). After window: 120 reads / 40
|
||||
// edits = 3:1 (deficit 1). Credit (3 - 1) * 40 edits * 600 = 48000.
|
||||
const at = daysAgo(10)
|
||||
const rec: ActionRecord = {
|
||||
id: 'r1', at, kind: 'claude-md-rule', findingId: 'read-edit-ratio',
|
||||
description: 'Add the read-edit-ratio rule block', changes: [], status: 'applied',
|
||||
baseline: { windowDays: 14, capturedAt: at, estimatedTokens: 12_000, sessions: 28, metrics: { reads: 10, edits: 10 } },
|
||||
}
|
||||
const actionsDir = await writeJournal([rec])
|
||||
const session = makeSession('s0', daysAgo(5), { toolBreakdown: { Read: { calls: 120 }, Edit: { calls: 40 } } })
|
||||
const filler = sessionsAt(19, daysAgo(4))
|
||||
const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf([session, ...filler])]) })
|
||||
|
||||
const row = report.rows[0]!
|
||||
expect(row.status).toBe('measured')
|
||||
expect(row.realizedTokens).toBe(48_000)
|
||||
expect(row.note).toMatch(/1\.0:1 -> 3\.0:1/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('archive realized delta', () => {
|
||||
it('measures per-item definition tokens times sessions and detects un-archive', async () => {
|
||||
const at = daysAgo(10)
|
||||
const kept = join(tmpdir(), 'codeburn-act-report-absent-skill-xyz') // absent -> not reverted
|
||||
const base = { windowDays: 14, capturedAt: at, estimatedTokens: 160, sessions: 28, metrics: { 'skill-a': 80, 'skill-b': 80 } }
|
||||
const rec: ActionRecord = {
|
||||
id: 'ar1', at, kind: 'archive-skill', findingId: 'unused-skills',
|
||||
description: 'Archive 2 unused skills', status: 'applied',
|
||||
changes: [{ path: kept, backup: null, op: 'move', movedTo: kept + '.archived', afterHash: '' }],
|
||||
baseline: base,
|
||||
}
|
||||
const actionsDir = await writeJournal([rec])
|
||||
const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(5)))]) })
|
||||
expect(report.rows[0]!.status).toBe('measured')
|
||||
expect(report.rows[0]!.realizedTokens).toBe(3200) // 160 tokens/session * 20
|
||||
|
||||
// Now the original path exists again -> reverted, zero savings.
|
||||
const restoredPath = join(actionsDir, 'restored-skill')
|
||||
await writeFile(restoredPath, 'x')
|
||||
const rec2: ActionRecord = { ...rec, changes: [{ path: restoredPath, backup: null, op: 'move', movedTo: restoredPath + '.archived', afterHash: '' }] }
|
||||
const dir2 = await writeJournal([rec2])
|
||||
const report2 = await computeActReport({ actionsDir: dir2, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(5)))]) })
|
||||
expect(report2.rows[0]!.status).toBe('reverted')
|
||||
expect(report2.rows[0]!.realizedTokens).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('unmeasured kinds', () => {
|
||||
it('marks bash cap not measurable but keeps the estimate visible', async () => {
|
||||
const at = daysAgo(10)
|
||||
const rec: ActionRecord = {
|
||||
id: 'b1', at, kind: 'shell-config', findingId: 'bash-output-cap',
|
||||
description: 'Set the bash output cap', changes: [], status: 'applied',
|
||||
baseline: { windowDays: 14, capturedAt: at, estimatedTokens: 3750, sessions: 28, metrics: { calls: 200 } },
|
||||
}
|
||||
const actionsDir = await writeJournal([rec])
|
||||
const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(5)))]) })
|
||||
expect(report.rows[0]!.status).toBe('not-measurable')
|
||||
expect(report.rows[0]!.estimatedTokens).toBe(3750)
|
||||
expect(report.measuredCount).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('optimize header', () => {
|
||||
it('appears only when a measured token action exists', async () => {
|
||||
const actionsDir = await writeJournal([mcpRecord()])
|
||||
const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(5)))]) })
|
||||
const header = buildOptimizeAppliedHeader(report)
|
||||
expect(header).toMatch(/^Applied fixes: 1 active, realized ~40\.0K tokens.*over 10 days\. Details: codeburn act report$/)
|
||||
})
|
||||
|
||||
it('returns null and never scans when the journal has no eligible actions', async () => {
|
||||
const emptyDir = await writeJournal([])
|
||||
const report = await computeActReport({
|
||||
actionsDir: emptyDir, now: NOW,
|
||||
loadProjects: async () => { throw new Error('should not scan for an empty journal') },
|
||||
})
|
||||
expect(report.rows).toHaveLength(0)
|
||||
expect(report.activeCount).toBe(0)
|
||||
expect(buildOptimizeAppliedHeader(report)).toBeNull()
|
||||
})
|
||||
|
||||
it('records the earliest apply date per finding for re-flagging', async () => {
|
||||
const records = [
|
||||
mcpRecord({ id: 'x1', at: daysAgo(9), findingId: 'unused-mcp' }),
|
||||
mcpRecord({ id: 'x2', at: daysAgo(4), findingId: 'unused-mcp' }),
|
||||
]
|
||||
const actionsDir = await writeJournal(records)
|
||||
const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(3)))]) })
|
||||
expect(report.appliedByFinding['unused-mcp']).toBe(daysAgo(9).slice(0, 10))
|
||||
})
|
||||
})
|
||||
|
||||
describe('json + render shape', () => {
|
||||
it('mirrors the rows and totals in --json', async () => {
|
||||
const actionsDir = await writeJournal([mcpRecord()])
|
||||
const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(5)))]) })
|
||||
const json = buildActReportJson(report) as {
|
||||
actions: Array<Record<string, unknown>>
|
||||
totals: Record<string, unknown>
|
||||
footer: string
|
||||
windowCapDays: number
|
||||
}
|
||||
|
||||
expect(Array.isArray(json.actions)).toBe(true)
|
||||
expect(json.actions[0]).toMatchObject({
|
||||
kind: 'mcp-remove',
|
||||
estimatedTokens: 56_000,
|
||||
realizedTokens: 40_000,
|
||||
status: 'measured',
|
||||
confidence: 'normal',
|
||||
})
|
||||
expect(json.totals).toMatchObject({ realizedTokens: 40_000, measuredActions: 1, activeActions: 1 })
|
||||
expect(json.windowCapDays).toBe(30)
|
||||
expect(typeof json.footer).toBe('string')
|
||||
expect(json.footer).toMatch(/correlation/)
|
||||
})
|
||||
|
||||
it('renders an empty state without a table when nothing is measurable', async () => {
|
||||
const emptyDir = await writeJournal([])
|
||||
const report = await computeActReport({ actionsDir: emptyDir, now: NOW, loadProjects: async () => [] })
|
||||
const out = renderActReport(report)
|
||||
expect(out).toMatch(/No applied actions to measure yet/)
|
||||
expect(out).not.toMatch(/Total realized/)
|
||||
})
|
||||
|
||||
it('renders a table with a total row when measurements exist', async () => {
|
||||
const actionsDir = await writeJournal([mcpRecord()])
|
||||
const report = await computeActReport({ actionsDir, now: NOW, loadProjects: load([projectOf(sessionsAt(20, daysAgo(5)))]) })
|
||||
const out = renderActReport(report)
|
||||
expect(out).toMatch(/Total realized/)
|
||||
expect(out).toMatch(/40\.0K/)
|
||||
expect(out).toMatch(/measures only its own metric/)
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue