mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-08-21 22:44:31 +00:00
optimize: classify findings as fix/nudge/keep and mark measured vs estimated
Every finding now resolves to a class (apply-able fix, habit nudge, or informational keep) and a basis (measured from provider-counted usage, or estimated from a schema/heuristic model), both from one table next to the FindingId union. The class follows the plan layer: an id is 'fix' only when buildPlan routes it, and an instance drops to 'nudge' when it lacks the payload or cause its builder needs. CLI, TUI and the desktop app group findings under Fix now / Habits / FYI with continuous numbering; the CLI header reports 'N measured · M estimated' in place of the blanket 'Estimates only.' footer. The JSON report gains class + basis per finding and summary.measuredSavingsUSD; existing fields are unchanged. The menubar's top three follow the same order, since every surface reads the sorted findings list. Sessions whose cost the provider never reported leave the cost-outliers peer math; when nothing else is priced the comparison falls back to them and the finding reports itself as estimated instead of disappearing.
This commit is contained in:
iamtoruk
parent
d5b3720079
commit
7c54cf85c2
11 changed files with 352 additions and 28 deletions
|
|
@ -138,7 +138,7 @@ function installDefaultMocks() {
|
|||
summary: {
|
||||
healthScore: 100, healthGrade: 'A', findingCount: 0, periodCostUSD: 0,
|
||||
sessions: 0, calls: 0, potentialSavingsTokens: 0, potentialSavingsCostUSD: 0,
|
||||
potentialSavingsPercent: 0, costRateUSD: 0,
|
||||
potentialSavingsPercent: 0, costRateUSD: 0, measuredSavingsUSD: 0,
|
||||
},
|
||||
findings: [],
|
||||
})
|
||||
|
|
|
|||
|
|
@ -407,6 +407,8 @@ export type WasteAction =
|
|||
| { type: 'command'; label: string; text: string }
|
||||
| { type: 'file-content'; label: string; path: string; content: string }
|
||||
|
||||
export type FindingClass = 'fix' | 'nudge' | 'keep'
|
||||
|
||||
export type OptimizeJsonReport = {
|
||||
period: { label: string; start: string | null; end: string | null }
|
||||
summary: {
|
||||
|
|
@ -420,6 +422,7 @@ export type OptimizeJsonReport = {
|
|||
potentialSavingsCostUSD: number
|
||||
potentialSavingsPercent: number | null
|
||||
costRateUSD: number
|
||||
measuredSavingsUSD: number
|
||||
}
|
||||
findings: Array<{
|
||||
id: string
|
||||
|
|
@ -429,6 +432,8 @@ export type OptimizeJsonReport = {
|
|||
trend: 'active' | 'improving' | null
|
||||
tokensSaved: number
|
||||
estimatedSavingsUSD: number
|
||||
class: FindingClass
|
||||
basis: 'measured' | 'estimated'
|
||||
fix: WasteAction
|
||||
}>
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,23 +48,26 @@ function makeOptimizeReport(): OptimizeJsonReport {
|
|||
healthScore: 72, healthGrade: 'C', findingCount: 3, periodCostUSD: 612.48,
|
||||
sessions: 88, calls: 1220, potentialSavingsTokens: 184_000,
|
||||
potentialSavingsCostUSD: 94.4, potentialSavingsPercent: 15.4, costRateUSD: 0.0005,
|
||||
measuredSavingsUSD: 27.8,
|
||||
},
|
||||
findings: [
|
||||
{
|
||||
id: 'cost-outliers', title: 'Opus is doing your small talk',
|
||||
id: 'unused-mcp', title: 'Opus is doing your small talk',
|
||||
explanation: 'Small conversational requests are running on an expensive model.',
|
||||
severity: 'high', trend: 'active', tokensSaved: 18_200, estimatedSavingsUSD: 9.1,
|
||||
class: 'fix', basis: 'estimated',
|
||||
fix: { type: 'paste', label: 'Paste into CLAUDE.md', text: 'Use Sonnet for routine questions.', destination: 'claude-md' },
|
||||
},
|
||||
{
|
||||
id: 'context-heavy-sessions', title: 'Cache hit is low in agentseal-dash',
|
||||
id: 'cost-outliers', title: 'Cache hit is low in agentseal-dash',
|
||||
explanation: 'Repeated context is not being served from cache.', severity: 'medium',
|
||||
trend: null, tokensSaved: 17_400, estimatedSavingsUSD: 8.7,
|
||||
trend: null, tokensSaved: 17_400, estimatedSavingsUSD: 8.7, class: 'nudge', basis: 'measured',
|
||||
fix: { type: 'command', label: 'Run this command', text: 'codeburn cache inspect' },
|
||||
},
|
||||
{
|
||||
id: 'warmup-heavy', title: 'Batch tiny requests', explanation: 'Many short sessions repeat setup work.',
|
||||
id: 'context-heavy-sessions', title: 'Batch tiny requests', explanation: 'Many short sessions repeat setup work.',
|
||||
severity: 'low', trend: 'improving', tokensSaved: 4_800, estimatedSavingsUSD: 2.4,
|
||||
class: 'keep', basis: 'measured',
|
||||
fix: { type: 'file-content', label: 'Create configuration', path: '~/.codeburn/config.json', content: '{"batch":true}' },
|
||||
},
|
||||
],
|
||||
|
|
@ -121,6 +124,14 @@ describe('Optimize', () => {
|
|||
Object.defineProperty(navigator, 'clipboard', { configurable: true, value: { writeText } })
|
||||
})
|
||||
|
||||
it('groups Waste findings under the fix / habits / FYI headers in order', async () => {
|
||||
render(<Optimize period="30days" provider="all" />)
|
||||
|
||||
await screen.findByText('Opus is doing your small talk')
|
||||
const groups = document.querySelectorAll('.opt-group')
|
||||
expect([...groups].map(g => g.textContent)).toEqual(['Fix now (apply-able)', 'Habits', 'FYI'])
|
||||
})
|
||||
|
||||
it('renders tabs and actionable Waste findings with impact, savings, explanation, and copy-paste fix', async () => {
|
||||
render(<Optimize period="30days" provider="all" />)
|
||||
|
||||
|
|
@ -130,7 +141,7 @@ describe('Optimize', () => {
|
|||
expect(screen.getByText('Medium')).toHaveClass('opt-impact-medium')
|
||||
expect(screen.getByText('Low')).toHaveClass('opt-impact-low')
|
||||
expect(screen.getByText('$9.10')).toHaveClass('opt-finding-savings')
|
||||
expect(screen.getByText('18.2K tokens')).toBeInTheDocument()
|
||||
expect(screen.getByText('18.2K tokens · estimated')).toBeInTheDocument()
|
||||
expect(screen.getByRole('tab', { name: 'Waste $94.40' })).toBeInTheDocument()
|
||||
expect(screen.getByRole('tab', { name: 'Reverts $107.00' })).toBeInTheDocument()
|
||||
expect(screen.getByRole('tab', { name: 'Abandoned $65.40' })).toBeInTheDocument()
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import { StaleBanner } from '../components/StaleBanner'
|
|||
import { type Polled, usePolled } from '../hooks/usePolled'
|
||||
import { formatCompact, formatUsd } from '../lib/format'
|
||||
import { codeburn } from '../lib/ipc'
|
||||
import type { DateRange, MenubarPayload, OptimizeJsonReport, Period, SessionYieldJson, WasteAction, YieldJsonReport } from '../lib/types'
|
||||
import type { DateRange, FindingClass, MenubarPayload, OptimizeJsonReport, Period, SessionYieldJson, WasteAction, YieldJsonReport } from '../lib/types'
|
||||
|
||||
type OptimizeTab = 'waste' | 'reverts' | 'abandoned' | 'fixes'
|
||||
|
||||
|
|
@ -114,6 +114,12 @@ const IMPACT_ICON: Record<'high' | 'medium' | 'low', string> = {
|
|||
low: '↓',
|
||||
}
|
||||
|
||||
const CLASS_HEADERS: Record<FindingClass, string> = {
|
||||
fix: 'Fix now (apply-able)',
|
||||
nudge: 'Habits',
|
||||
keep: 'FYI',
|
||||
}
|
||||
|
||||
function actionText(fix: WasteAction): string {
|
||||
return fix.type === 'file-content' ? fix.content : fix.text
|
||||
}
|
||||
|
|
@ -132,10 +138,14 @@ function ActionableFindingRows({ findings }: { findings: OptimizeFinding[] }) {
|
|||
|
||||
return (
|
||||
<div className="opt-findings">
|
||||
{findings.map(finding => {
|
||||
{findings.map((finding, i) => {
|
||||
const expanded = expandedId === finding.id
|
||||
// Findings arrive class-sorted from the CLI, so a header goes in
|
||||
// wherever the class changes.
|
||||
const showHeader = finding.class !== findings[i - 1]?.class
|
||||
return (
|
||||
<Fragment key={finding.id}>
|
||||
{showHeader && <div className="opt-group">{CLASS_HEADERS[finding.class]}</div>}
|
||||
<button
|
||||
className="opt-finding opt-finding-toggle"
|
||||
type="button"
|
||||
|
|
@ -153,7 +163,7 @@ function ActionableFindingRows({ findings }: { findings: OptimizeFinding[] }) {
|
|||
)}
|
||||
</span>
|
||||
<span className="opt-finding-savings">{formatUsd(finding.estimatedSavingsUSD)}</span>
|
||||
<span className="opt-finding-tokens">{formatCompact(finding.tokensSaved)} tokens</span>
|
||||
<span className="opt-finding-tokens">{formatCompact(finding.tokensSaved)} tokens · {finding.basis}</span>
|
||||
<span className="opt-finding-chevron" aria-hidden="true">›</span>
|
||||
</button>
|
||||
{expanded && (
|
||||
|
|
|
|||
|
|
@ -640,6 +640,9 @@ td:first-child { font-size: var(--fs-body); font-weight: var(--fw-body); }
|
|||
.opt-waste { min-width: 0; }
|
||||
.opt-summary { padding: 0 0 10px; color: var(--mut); font-size: 11.5px; font-variant-numeric: tabular-nums; }
|
||||
.opt-findings { display: grid; min-width: 0; }
|
||||
.opt-group { padding: 11px 0 5px; color: var(--mut2); font-size: 10px; font-weight: 600; letter-spacing: 0.06em; text-transform: uppercase; }
|
||||
.opt-group:first-child { padding-top: 0; }
|
||||
.opt-group + .opt-finding { border-top: 0; }
|
||||
.opt-finding { display: grid; align-items: center; column-gap: 12px; min-height: 43px; border-top: 1px solid var(--line2); }
|
||||
.opt-finding:first-child { border-top: 0; }
|
||||
.opt-finding-legacy { grid-template-columns: 28px minmax(0, 1fr) 104px 86px; }
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import {
|
|||
ALWAYSLOAD_STARTUP_CAP_SECONDS,
|
||||
ENABLE_TOOL_SEARCH_VAR,
|
||||
parseVersion,
|
||||
SHELL_PROFILE_SCOPE,
|
||||
versionPredates,
|
||||
} from '../optimize.js'
|
||||
import type { WasteFinding } from '../optimize.js'
|
||||
|
|
@ -386,7 +387,6 @@ const NEXT_SESSION_NOTE = 'takes effect on the next session (this config is read
|
|||
|
||||
// findDeferralEnvSetting (src/optimize.ts) reports shell-profile hits with
|
||||
// exactly this scope string; the plan layer keys its refusal on it.
|
||||
const SHELL_PROFILE_SCOPE = 'shell profile'
|
||||
|
||||
const SHELL_TOOL_SEARCH_LINE = new RegExp(`^\\s*(?:export\\s+)?${ENABLE_TOOL_SEARCH_VAR}\\s*=.*$`, 'm')
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { homedir } from 'os'
|
||||
|
||||
import React, { useState, useCallback, useEffect, useLayoutEffect, useMemo, useRef } from 'react'
|
||||
import React, { Fragment, useState, useCallback, useEffect, useLayoutEffect, useMemo, useRef } from 'react'
|
||||
import { render, Box, Text, measureElement, useInput, useApp, useWindowSize, type DOMElement } from 'ink'
|
||||
import { CATEGORY_LABELS, type DateRange, type ProjectSummary, type TaskCategory } from './types.js'
|
||||
import { formatCost, formatTokens, markEstimated, carriedCostNote } from './format.js'
|
||||
|
|
@ -10,7 +10,7 @@ import { findUnpricedModels, isExpectedFreeModel, loadPricing } from './models.j
|
|||
import { aggregateModelTotals } from './model-breakdown.js'
|
||||
import { buildDurablePeriod } from './usage-aggregator.js'
|
||||
import { getAllProviders } from './providers/index.js'
|
||||
import { scanAndDetect, type WasteFinding, type WasteAction, type OptimizeResult } from './optimize.js'
|
||||
import { CLASS_HEADERS, findingBasis, findingClass, scanAndDetect, type FindingClass, type WasteFinding, type WasteAction, type OptimizeResult } from './optimize.js'
|
||||
import { aggregateFileChurn, buildCoachingNotes, computePricingCoverage, medianTimeToFirstEditMs, scanUserCorrections, worstOneShotCategory, type ReworkedFile } from './workflow-insights.js'
|
||||
import { estimateContextBudget, type ContextBudget } from './context-budget.js'
|
||||
import { dateKey } from './day-aggregator.js'
|
||||
|
|
@ -1079,7 +1079,7 @@ function FindingPanel({ index, finding, costRate, width }: { index: number; find
|
|||
{trendBadge && <Text color="#5BF5A0">{trendBadge}</Text>}
|
||||
</Text>
|
||||
<Text dimColor wrap="wrap">{finding.explanation}</Text>
|
||||
<Text color={GOLD}>Savings: ~{formatTokens(finding.tokensSaved)} tokens (~{formatCost(costSaved)})</Text>
|
||||
<Text color={GOLD}>Savings: ~{formatTokens(finding.tokensSaved)} tokens (~{formatCost(costSaved)})<Text dimColor> {findingBasis(finding)}</Text></Text>
|
||||
<Text> </Text>
|
||||
<FindingAction action={finding.fix} />
|
||||
</Box>
|
||||
|
|
@ -1119,8 +1119,18 @@ function OptimizeView({ findings, costRate, projects, label, width, healthScore,
|
|||
<Text dimColor>Showing {start + 1}–{end} of {total} · j/k to scroll</Text>
|
||||
)}
|
||||
</Box>
|
||||
{visible.map((f, i) => <FindingPanel key={start + i} index={start + i + 1} finding={f} costRate={costRate} width={width} />)}
|
||||
<Box paddingX={1} width={width}><Text dimColor>Token estimates are approximate.</Text></Box>
|
||||
{visible.map((f, i) => {
|
||||
// Findings arrive class-sorted, so a header goes in wherever the class
|
||||
// changes (including the top of the window after paging).
|
||||
const cls = findingClass(f)
|
||||
const previous: FindingClass | null = i > 0 ? findingClass(visible[i - 1]!) : null
|
||||
return (
|
||||
<Fragment key={start + i}>
|
||||
{cls !== previous && <Box paddingX={1} width={width}><Text bold color={ORANGE}>{CLASS_HEADERS[cls]}</Text></Box>}
|
||||
<FindingPanel index={start + i + 1} finding={f} costRate={costRate} width={width} />
|
||||
</Fragment>
|
||||
)
|
||||
})}
|
||||
</Box>
|
||||
)
|
||||
}
|
||||
|
|
|
|||
164
src/optimize.ts
164
src/optimize.ts
|
|
@ -264,6 +264,113 @@ export type FindingId =
|
|||
| 'unused-skills'
|
||||
| 'unused-commands'
|
||||
|
||||
/// How a finding is meant to be acted on:
|
||||
/// - `fix` CodeBurn can write the change itself (`codeburn optimize --apply`)
|
||||
/// - `nudge` behavioural, the user changes a habit
|
||||
/// - `keep` informational; the cost may well be justified
|
||||
export type FindingClass = 'fix' | 'nudge' | 'keep'
|
||||
|
||||
/// Where a finding's `tokensSaved` number comes from:
|
||||
/// - `measured` summed from provider-counted usage on the parsed calls
|
||||
/// - `estimated` a schema/heuristic model (per-tool sizes, recovery fractions)
|
||||
/// A detector that mixes the two counts as `estimated`.
|
||||
export type FindingBasis = 'measured' | 'estimated'
|
||||
|
||||
/// Static class per finding id. `fix` entries are exactly the ids `buildPlan`
|
||||
/// (src/act/plans.ts) routes to a plan builder; tests assert the two lists
|
||||
/// stay equal. Instances that lack the payload their builder needs fall back
|
||||
/// to `nudge` via `findingClass`.
|
||||
export const FINDING_CLASS: Record<FindingId, FindingClass> = {
|
||||
'read-edit-ratio': 'fix', // CLAUDE.md rule block
|
||||
'build-folder-reads': 'fix', // CLAUDE.md rule block
|
||||
'redundant-rereads': 'nudge',
|
||||
'warmup-heavy': 'nudge',
|
||||
'unused-mcp': 'fix',
|
||||
'mcp-low-coverage': 'fix',
|
||||
'mcp-project-scope': 'fix',
|
||||
'mcp-deferral-off': 'fix',
|
||||
'mcp-alwaysload-hygiene': 'fix',
|
||||
'mcp-defer-threshold': 'fix',
|
||||
'retry-heavy-capabilities': 'nudge',
|
||||
'low-worth-sessions': 'nudge',
|
||||
'context-heavy-sessions': 'keep', // context-heavy work is often load-bearing
|
||||
'cost-outliers': 'nudge',
|
||||
'claude-md-too-long': 'nudge', // trimming is a judgement call, not a rule block
|
||||
'bash-output-cap': 'fix',
|
||||
'unused-agents': 'fix',
|
||||
'unused-skills': 'fix',
|
||||
'unused-commands': 'fix',
|
||||
}
|
||||
|
||||
/// Ids whose plan is built from the `apply` payload: without it the plan
|
||||
/// builder returns null, so the finding is only a nudge.
|
||||
const CLASS_NEEDS_APPLY: ReadonlySet<FindingId> = new Set<FindingId>([
|
||||
'unused-mcp',
|
||||
'mcp-low-coverage',
|
||||
'mcp-project-scope',
|
||||
'mcp-deferral-off',
|
||||
'mcp-alwaysload-hygiene',
|
||||
'mcp-defer-threshold',
|
||||
'unused-agents',
|
||||
'unused-skills',
|
||||
'unused-commands',
|
||||
])
|
||||
|
||||
/// Static basis per finding id. Only the two session-level detectors sum
|
||||
/// provider-counted tokens end to end; everything else multiplies a modelled
|
||||
/// per-unit size or a recovery fraction.
|
||||
export const FINDING_BASIS: Record<FindingId, FindingBasis> = {
|
||||
'read-edit-ratio': 'estimated', // reads x AVG_TOKENS_PER_READ
|
||||
'build-folder-reads': 'estimated', // reads x AVG_TOKENS_PER_READ
|
||||
'redundant-rereads': 'estimated', // reads x AVG_TOKENS_PER_READ
|
||||
'warmup-heavy': 'estimated', // observed median minus a modelled baseline
|
||||
'unused-mcp': 'estimated', // tools x TOKENS_PER_MCP_TOOL x sessions
|
||||
'mcp-low-coverage': 'estimated', // schema-size model, only capped by observed cache tokens
|
||||
'mcp-project-scope': 'estimated', // same schema-size model
|
||||
'mcp-deferral-off': 'estimated', // schema-size model x affected sessions
|
||||
'mcp-alwaysload-hygiene': 'estimated', // tools x TOKENS_PER_MCP_TOOL x loaded sessions
|
||||
'mcp-defer-threshold': 'estimated', // definition-size model x sessions
|
||||
'retry-heavy-capabilities': 'estimated', // real turn tokens x recovery fraction
|
||||
'low-worth-sessions': 'estimated', // real session tokens x recovery fraction
|
||||
'context-heavy-sessions': 'measured', // counted input/cache tokens above the target ratio
|
||||
'cost-outliers': 'measured', // counted session tokens above the peer average
|
||||
'claude-md-too-long': 'estimated', // lines x CLAUDEMD_TOKENS_PER_LINE
|
||||
'bash-output-cap': 'estimated', // chars x BASH_TOKENS_PER_CHAR
|
||||
'unused-agents': 'estimated', // count x TOKENS_PER_AGENT_DEF
|
||||
'unused-skills': 'estimated', // count x TOKENS_PER_SKILL_DEF
|
||||
'unused-commands': 'estimated', // count x TOKENS_PER_COMMAND_DEF
|
||||
}
|
||||
|
||||
/// Scope label for a setting that lives in ~/.zshrc / ~/.bashrc. Plans never
|
||||
/// rewrite shell profiles, they only report them.
|
||||
export const SHELL_PROFILE_SCOPE = 'shell profile'
|
||||
|
||||
export function findingClass(f: WasteFinding): FindingClass {
|
||||
const base = FINDING_CLASS[f.id]
|
||||
if (base !== 'fix') return base
|
||||
if (CLASS_NEEDS_APPLY.has(f.id) && !f.apply) return 'nudge'
|
||||
const apply = f.apply
|
||||
if ((apply?.kind === 'defer-enable' || apply?.kind === 'defer-threshold') && apply.settingScope === SHELL_PROFILE_SCOPE) {
|
||||
return 'nudge'
|
||||
}
|
||||
// Of the deferral causes only these two have a plan; the rest are manual
|
||||
// advice (Vertex policy, an outdated Claude Code, an unverified proxy).
|
||||
if (apply?.kind === 'defer-enable' && apply.cause !== 'env-false' && apply.cause !== 'proxy-verified') return 'nudge'
|
||||
return 'fix'
|
||||
}
|
||||
|
||||
export function findingBasis(f: WasteFinding): FindingBasis {
|
||||
return f.basis ?? FINDING_BASIS[f.id]
|
||||
}
|
||||
|
||||
const CLASS_ORDER: Record<FindingClass, number> = { fix: 0, nudge: 1, keep: 2 }
|
||||
|
||||
export const CLASS_HEADERS: Record<FindingClass, string> = {
|
||||
fix: 'Fix now (apply-able) — codeburn optimize --apply',
|
||||
nudge: 'Habits',
|
||||
keep: 'FYI',
|
||||
}
|
||||
|
||||
// Cause taxonomy for defer-enable plans (mcp-deferral-off findings).
|
||||
// 'proxy-verified' is never produced by the detector today: it is reserved
|
||||
// for the #614 part-3 proxy verifier, which upgrades 'proxy-unknown' once a
|
||||
|
|
@ -303,6 +410,9 @@ export type WasteFinding = {
|
|||
fix: WasteAction
|
||||
trend?: Trend
|
||||
apply?: FindingApply
|
||||
/// Set only when a detector's basis varies per run (see detectSessionOutliers);
|
||||
/// otherwise `FINDING_BASIS[id]` applies. Read through `findingBasis`.
|
||||
basis?: FindingBasis
|
||||
}
|
||||
|
||||
export type OptimizeResult = {
|
||||
|
|
@ -330,6 +440,9 @@ export type OptimizeJsonReport = {
|
|||
potentialSavingsCostUSD: number
|
||||
potentialSavingsPercent: number | null
|
||||
costRateUSD: number
|
||||
/// Portion of `potentialSavingsCostUSD` coming from `measured`-basis
|
||||
/// findings. The total keeps its old meaning: measured plus estimated.
|
||||
measuredSavingsUSD: number
|
||||
}
|
||||
findings: Array<{
|
||||
id: FindingId
|
||||
|
|
@ -339,6 +452,8 @@ export type OptimizeJsonReport = {
|
|||
trend: Trend | null
|
||||
tokensSaved: number
|
||||
estimatedSavingsUSD: number
|
||||
class: FindingClass
|
||||
basis: FindingBasis
|
||||
fix: WasteAction
|
||||
}>
|
||||
/// Files most reworked by edit-family calls, relative to project root (top 15).
|
||||
|
|
@ -1716,7 +1831,7 @@ export function findDeferralEnvSetting(
|
|||
const content = readSessionFileSync(path)
|
||||
if (content === null) continue
|
||||
const match = content.match(linePattern)
|
||||
if (match) return { value: match[1]!, scope: 'shell profile', path }
|
||||
if (match) return { value: match[1]!, scope: SHELL_PROFILE_SCOPE, path }
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
|
@ -2814,9 +2929,17 @@ export function detectSessionOutliers(projects: ProjectSummary[], excludedSessio
|
|||
}
|
||||
|
||||
const outliers: Outlier[] = []
|
||||
// Modelled costs (Kiro, Cursor, some Cline sessions) are not comparable
|
||||
// against provider-reported ones, so they leave the peer math. Providers
|
||||
// that only ever estimate would lose the finding entirely, so those fall
|
||||
// back to the full set and the finding reports itself as estimated.
|
||||
let usedEstimatedCosts = false
|
||||
|
||||
for (const project of projects) {
|
||||
const sessions = project.sessions.filter(s => s.totalCostUSD > 0)
|
||||
const costed = project.sessions.filter(s => s.totalCostUSD > 0)
|
||||
const exact = costed.filter(s => (s.totalEstimatedCostUSD ?? 0) === 0)
|
||||
const sessions = exact.length >= MIN_SESSIONS_FOR_OUTLIER ? exact : costed
|
||||
const fellBack = sessions.length > exact.length
|
||||
if (sessions.length < MIN_SESSIONS_FOR_OUTLIER) continue
|
||||
|
||||
const totalCost = sessions.reduce((sum, s) => sum + s.totalCostUSD, 0)
|
||||
|
|
@ -2835,6 +2958,7 @@ export function detectSessionOutliers(projects: ProjectSummary[], excludedSessio
|
|||
// "tighter constraint" advice here.
|
||||
if (excludedSessionIds?.has(session.sessionId)) continue
|
||||
|
||||
if (fellBack) usedEstimatedCosts = true
|
||||
outliers.push({
|
||||
project: project.project,
|
||||
sessionId: session.sessionId,
|
||||
|
|
@ -2864,6 +2988,7 @@ export function detectSessionOutliers(projects: ProjectSummary[], excludedSessio
|
|||
explanation: `Sessions costing more than ${SESSION_OUTLIER_MULTIPLIER}x their peer-session average in the same project: ${list}${extra}. These usually come from broad prompts, runaway loops, or context-heavy work that should be split into smaller sessions.`,
|
||||
impact: outliers.length >= 3 || totalExcessCost >= 10 ? 'high' : 'medium',
|
||||
tokensSaved,
|
||||
...(usedEstimatedCosts ? { basis: 'estimated' as const } : {}),
|
||||
fix: {
|
||||
type: 'paste',
|
||||
destination: 'session-opener',
|
||||
|
|
@ -3080,7 +3205,10 @@ export async function scanAndDetect(
|
|||
: []
|
||||
for (const f of ghostResults) if (f) findings.push(f)
|
||||
|
||||
// Urgency first, then class: every surface lists the apply-able fixes
|
||||
// before the habit nudges, and orders by urgency inside each group.
|
||||
findings.sort((a, b) => urgencyScore(b) - urgencyScore(a))
|
||||
findings.sort((a, b) => CLASS_ORDER[findingClass(a)] - CLASS_ORDER[findingClass(b)])
|
||||
const { score, grade } = computeHealth(findings)
|
||||
|
||||
const modelRecommendations: ModelDefaultRecommendation[] = []
|
||||
|
|
@ -3164,7 +3292,7 @@ function renderFinding(n: number, f: WasteFinding, costRate: number): string[] {
|
|||
lines.push('')
|
||||
lines.push(wrap(f.explanation, PANEL_WIDTH - 4, ' '))
|
||||
lines.push('')
|
||||
lines.push(chalk.hex(GOLD)(` Potential savings: ${savings}`))
|
||||
lines.push(chalk.hex(GOLD)(` Potential savings: ${savings}`) + chalk.dim(` ${findingBasis(f)}`))
|
||||
lines.push('')
|
||||
|
||||
// Destination header — issue #277. Tells the user where each suggestion
|
||||
|
|
@ -3207,7 +3335,7 @@ function renderWorkflowSection(reworkedFiles: ReworkedFile[], coachingNotes: str
|
|||
return lines
|
||||
}
|
||||
|
||||
function renderOptimize(
|
||||
export function renderOptimize(
|
||||
findings: WasteFinding[],
|
||||
costRate: number,
|
||||
periodLabel: string,
|
||||
|
|
@ -3228,12 +3356,16 @@ function renderOptimize(
|
|||
lines.push(chalk.hex(DIM)(' ' + SEP.repeat(PANEL_WIDTH)))
|
||||
|
||||
const issueSuffix = findings.length > 0 ? `, ${findings.length} issue${findings.length > 1 ? 's' : ''}` : ''
|
||||
const measured = findings.filter(f => findingBasis(f) === 'measured').length
|
||||
lines.push(' ' + [
|
||||
`${sessionCount} sessions`,
|
||||
`${callCount.toLocaleString()} calls`,
|
||||
chalk.hex(GOLD)(formatCost(periodCost)),
|
||||
`Health: ${chalk.bold.hex(GRADE_COLORS[healthGrade])(healthGrade)}${chalk.dim(` (${healthScore}/100${issueSuffix})`)}`,
|
||||
].join(chalk.hex(DIM)(' ')))
|
||||
if (findings.length > 0) {
|
||||
lines.push(chalk.dim(` ${measured} measured · ${findings.length - measured} estimated`))
|
||||
}
|
||||
if (appliedHeader) lines.push(' ' + chalk.hex(GREEN)(appliedHeader))
|
||||
lines.push('')
|
||||
|
||||
|
|
@ -3257,15 +3389,22 @@ function renderOptimize(
|
|||
lines.push(chalk.hex(GREEN)(` Potential savings: ~${formatTokens(totalTokens)} tokens${costText}`))
|
||||
lines.push('')
|
||||
|
||||
for (let i = 0; i < findings.length; i++) {
|
||||
const f = findings[i]!
|
||||
const appliedOn = previouslyApplied?.[f.id]
|
||||
const shown = appliedOn ? { ...f, title: `${f.title} (previously applied ${appliedOn}, re-flagged)` } : f
|
||||
lines.push(...renderFinding(i + 1, shown, costRate))
|
||||
// One block per class, in fix -> nudge -> keep order; numbering runs
|
||||
// continuously across the blocks so `--only` picks stay unambiguous.
|
||||
let n = 0
|
||||
for (const cls of ['fix', 'nudge', 'keep'] as const) {
|
||||
const group = findings.filter(f => findingClass(f) === cls)
|
||||
if (group.length === 0) continue
|
||||
lines.push(chalk.bold.hex(ORANGE)(` ${CLASS_HEADERS[cls]}`))
|
||||
lines.push('')
|
||||
for (const f of group) {
|
||||
const appliedOn = previouslyApplied?.[f.id]
|
||||
const shown = appliedOn ? { ...f, title: `${f.title} (previously applied ${appliedOn}, re-flagged)` } : f
|
||||
lines.push(...renderFinding(++n, shown, costRate))
|
||||
}
|
||||
}
|
||||
|
||||
lines.push(chalk.hex(DIM)(' ' + SEP.repeat(PANEL_WIDTH)))
|
||||
lines.push(chalk.dim(' Estimates only.'))
|
||||
lines.push('')
|
||||
|
||||
lines.push(...renderWorkflowSection(reworkedFiles, coachingNotes))
|
||||
|
|
@ -3365,6 +3504,9 @@ export function buildOptimizeJsonReport(
|
|||
potentialSavingsCostUSD,
|
||||
potentialSavingsPercent,
|
||||
costRateUSD: result.costRate,
|
||||
measuredSavingsUSD: result.findings
|
||||
.filter(f => findingBasis(f) === 'measured')
|
||||
.reduce((s, f) => s + f.tokensSaved * result.costRate, 0),
|
||||
},
|
||||
findings: result.findings.map(f => ({
|
||||
id: f.id,
|
||||
|
|
@ -3374,6 +3516,8 @@ export function buildOptimizeJsonReport(
|
|||
trend: f.trend ?? null,
|
||||
tokensSaved: f.tokensSaved,
|
||||
estimatedSavingsUSD: f.tokensSaved * result.costRate,
|
||||
class: findingClass(f),
|
||||
basis: findingBasis(f),
|
||||
fix: f.fix,
|
||||
})),
|
||||
...buildWorkflowReport(projects),
|
||||
|
|
|
|||
|
|
@ -712,12 +712,12 @@ describe('InteractiveDashboard refresh', () => {
|
|||
expect(activityHeader.indexOf('turns') + 'turns'.length).toBe(activityRow.indexOf('12') + '12'.length)
|
||||
expect(activityHeader.indexOf('1-shot') + '1-shot'.length).toBe(activityRow.indexOf('50%') + '50%'.length)
|
||||
stdin.write('o')
|
||||
for (let i = 0; i < 20 && !frames.some(frame => frame.includes('Token estimates are approximate.')); i++) {
|
||||
for (let i = 0; i < 20 && !frames.some(frame => frame.includes('Savings: ~')); i++) {
|
||||
await vi.advanceTimersByTimeAsync(50)
|
||||
}
|
||||
const beforeRefresh = frames.filter(frame => frame.trim()).at(-1) ?? ''
|
||||
expect(beforeRefresh).toContain('CodeBurn Optimize')
|
||||
expect(beforeRefresh).toContain('Token estimates are approximate.')
|
||||
expect(beforeRefresh).toContain('CodeBurn Optimize')
|
||||
|
||||
frames.length = 0
|
||||
await vi.advanceTimersByTimeAsync(60_000)
|
||||
|
|
@ -726,7 +726,7 @@ describe('InteractiveDashboard refresh', () => {
|
|||
const frame = frames.filter(value => value.trim()).at(-1) ?? beforeRefresh
|
||||
expect(frame).toBe(beforeRefresh)
|
||||
expect(frame).toContain('CodeBurn Optimize')
|
||||
expect(frame).toContain('Token estimates are approximate.')
|
||||
expect(frame).toContain('CodeBurn Optimize')
|
||||
expect(frame).toContain('b back')
|
||||
expect(frame).not.toContain('Loading Today')
|
||||
expect(frame).not.toContain('Scanning Today')
|
||||
|
|
|
|||
|
|
@ -12,6 +12,9 @@ import { runAction } from '../src/act/apply.js'
|
|||
import { undoAction } from '../src/act/undo.js'
|
||||
import { readRecords, shortId } from '../src/act/journal.js'
|
||||
import {
|
||||
FINDING_BASIS,
|
||||
FINDING_CLASS,
|
||||
findingClass,
|
||||
detectBloatedClaudeMd,
|
||||
detectDuplicateReads,
|
||||
detectJunkReads,
|
||||
|
|
@ -653,3 +656,78 @@ describe('stale-plan detection', () => {
|
|||
expect(await readFile(p, 'utf-8')).toBe('overwritten')
|
||||
})
|
||||
})
|
||||
|
||||
describe('finding class', () => {
|
||||
it('covers every finding id with a class and a basis', () => {
|
||||
expect(Object.keys(FINDING_BASIS).sort()).toEqual(Object.keys(FINDING_CLASS).sort())
|
||||
})
|
||||
|
||||
it("classes a finding 'fix' exactly when a plan can be built for it", async () => {
|
||||
const fx = await makeFixture()
|
||||
const claudeJson = join(fx.home, '.claude.json')
|
||||
await writeFile(claudeJson, JSON.stringify({ mcpServers: { srv: { command: 's' } } }, null, 2) + '\n')
|
||||
const settings = join(fx.project, '.claude', 'settings.json')
|
||||
await mkdir(join(fx.project, '.claude'), { recursive: true })
|
||||
await writeFile(settings, JSON.stringify({ env: { ENABLE_TOOL_SEARCH: 'auto' } }, null, 2) + '\n')
|
||||
const mcpJson = join(fx.project, '.mcp.json')
|
||||
await writeFile(mcpJson, JSON.stringify({ mcpServers: { pinned: { command: 'x', alwaysLoad: true } } }, null, 2) + '\n')
|
||||
await mkdir(join(fx.home, '.claude', 'skills', 'ghost'), { recursive: true })
|
||||
await mkdir(join(fx.home, '.claude', 'agents'), { recursive: true })
|
||||
await mkdir(join(fx.home, '.claude', 'commands'), { recursive: true })
|
||||
await writeFile(join(fx.home, '.claude', 'agents', 'ghost.md'), 'x')
|
||||
await writeFile(join(fx.home, '.claude', 'commands', 'ghost.md'), 'x')
|
||||
|
||||
const CLAUDE_MD_FIX: WasteAction = { type: 'paste', destination: 'claude-md', label: '', text: 'rule' }
|
||||
const SHELL_FIX: WasteAction = { type: 'paste', destination: 'shell-config', label: '', text: 'export X=1' }
|
||||
const PROMPT_FIX: WasteAction = { type: 'paste', destination: 'prompt', label: '', text: 'ask' }
|
||||
const OPENER_FIX: WasteAction = { type: 'paste', destination: 'session-opener', label: '', text: 'o' }
|
||||
|
||||
// One representative finding per id, carrying the payload its plan
|
||||
// builder needs. Ids without a builder get a plain prompt fix.
|
||||
const representatives: Record<FindingId, WasteFinding> = {
|
||||
'read-edit-ratio': makeFinding('read-edit-ratio', CLAUDE_MD_FIX),
|
||||
'build-folder-reads': makeFinding('build-folder-reads', CLAUDE_MD_FIX),
|
||||
'redundant-rereads': makeFinding('redundant-rereads', PROMPT_FIX),
|
||||
'warmup-heavy': makeFinding('warmup-heavy', SHELL_FIX),
|
||||
'unused-mcp': makeFinding('unused-mcp', CMD_FIX, { kind: 'mcp-remove', servers: ['srv'] }),
|
||||
'mcp-low-coverage': makeFinding('mcp-low-coverage', CMD_FIX, { kind: 'mcp-remove', servers: ['srv'] }),
|
||||
'mcp-project-scope': makeFinding('mcp-project-scope', PROMPT_FIX, {
|
||||
kind: 'mcp-project-scope',
|
||||
servers: [{ server: 'srv', keepProjects: [fx.project], removeProjects: [] }],
|
||||
}),
|
||||
'mcp-deferral-off': makeFinding('mcp-deferral-off', CMD_FIX, {
|
||||
kind: 'defer-enable', cause: 'env-false', settingPath: settings, settingScope: 'project settings', value: 'false',
|
||||
}),
|
||||
'mcp-alwaysload-hygiene': makeFinding('mcp-alwaysload-hygiene', PROMPT_FIX, {
|
||||
kind: 'defer-alwaysload',
|
||||
servers: [{ server: 'pinned', paths: [mcpJson] }],
|
||||
}),
|
||||
'mcp-defer-threshold': makeFinding('mcp-defer-threshold', PROMPT_FIX, {
|
||||
kind: 'defer-threshold', settingPath: settings, settingScope: 'project settings',
|
||||
value: 'auto', recommendedPercent: 2, removeOverride: false,
|
||||
}),
|
||||
'retry-heavy-capabilities': makeFinding('retry-heavy-capabilities', PROMPT_FIX),
|
||||
'low-worth-sessions': makeFinding('low-worth-sessions', OPENER_FIX),
|
||||
'context-heavy-sessions': makeFinding('context-heavy-sessions', OPENER_FIX),
|
||||
'cost-outliers': makeFinding('cost-outliers', OPENER_FIX),
|
||||
'claude-md-too-long': makeFinding('claude-md-too-long', PROMPT_FIX),
|
||||
'bash-output-cap': makeFinding('bash-output-cap', SHELL_FIX),
|
||||
'unused-agents': makeFinding('unused-agents', CMD_FIX, { kind: 'archive', names: ['ghost'] }),
|
||||
'unused-skills': makeFinding('unused-skills', CMD_FIX, { kind: 'archive', names: ['ghost'] }),
|
||||
'unused-commands': makeFinding('unused-commands', CMD_FIX, { kind: 'archive', names: ['ghost'] }),
|
||||
}
|
||||
|
||||
const planCtx: PlanContext = { homeDir: fx.home, cwd: fx.project, shell: '/bin/zsh', claudeVersion: () => '2.1.130' }
|
||||
for (const finding of Object.values(representatives)) {
|
||||
const hasPlan = planFor(finding, planCtx) !== null
|
||||
expect([finding.id, hasPlan]).toEqual([finding.id, findingClass(finding) === 'fix'])
|
||||
}
|
||||
})
|
||||
|
||||
it("drops to 'nudge' when the instance lacks the payload its plan needs", () => {
|
||||
const finding = makeFinding('mcp-deferral-off', { type: 'paste', destination: 'shell-config', label: '', text: 'x' })
|
||||
expect(FINDING_CLASS['mcp-deferral-off']).toBe('fix')
|
||||
expect(findingClass(finding)).toBe('nudge')
|
||||
expect(planFor(finding)).toBeNull()
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -26,6 +26,9 @@ import {
|
|||
computeHealth,
|
||||
computeTrend,
|
||||
buildOptimizeJsonReport,
|
||||
renderOptimize,
|
||||
findingBasis,
|
||||
type FindingId,
|
||||
type ToolCall,
|
||||
type ApiCallMeta,
|
||||
type WasteFinding,
|
||||
|
|
@ -1004,6 +1007,29 @@ describe('detectSessionOutliers', () => {
|
|||
expect(finding!.tokensSaved).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('keeps estimated-cost sessions out of the peer math', () => {
|
||||
const project = projectWithSessions([1, 1, 1, 10])
|
||||
// The expensive session is priced from modelled tokens, so it is not
|
||||
// comparable against the provider-reported peers and never gets flagged.
|
||||
project.sessions[3]!.totalEstimatedCostUSD = project.sessions[3]!.totalCostUSD
|
||||
expect(detectSessionOutliers([project])).toBeNull()
|
||||
})
|
||||
|
||||
it('falls back to estimated costs when nothing else is priced, and says so', () => {
|
||||
const project = projectWithSessions([1, 1, 1, 10])
|
||||
for (const s of project.sessions) s.totalEstimatedCostUSD = s.totalCostUSD
|
||||
const finding = detectSessionOutliers([project])
|
||||
expect(finding).not.toBeNull()
|
||||
expect(finding!.basis).toBe('estimated')
|
||||
expect(findingBasis(finding!)).toBe('estimated')
|
||||
})
|
||||
|
||||
it('reports measured basis when every peer cost is provider-reported', () => {
|
||||
const finding = detectSessionOutliers([projectWithSessions([1, 1, 1, 10])])
|
||||
expect(finding!.basis).toBeUndefined()
|
||||
expect(findingBasis(finding!)).toBe('measured')
|
||||
})
|
||||
|
||||
it('ignores tiny absolute-cost outliers', () => {
|
||||
expect(detectSessionOutliers([projectWithSessions([0.01, 0.01, 0.01, 0.2])])).toBeNull()
|
||||
})
|
||||
|
|
@ -1240,6 +1266,7 @@ describe('buildOptimizeJsonReport', () => {
|
|||
healthGrade: 'C',
|
||||
findings: [
|
||||
{
|
||||
id: 'claude-md-too-long',
|
||||
title: 'Trim stale context',
|
||||
explanation: 'Old instructions are loaded every turn.',
|
||||
impact: 'medium',
|
||||
|
|
@ -1283,12 +1310,15 @@ describe('buildOptimizeJsonReport', () => {
|
|||
potentialSavingsPercent: 20,
|
||||
costRateUSD: 0.00002,
|
||||
})
|
||||
expect(report.summary.measuredSavingsUSD).toBe(0)
|
||||
expect(report.findings[0]).toMatchObject({
|
||||
title: 'Trim stale context',
|
||||
severity: 'medium',
|
||||
trend: 'active',
|
||||
tokensSaved: 50_000,
|
||||
estimatedSavingsUSD: 1,
|
||||
class: 'nudge',
|
||||
basis: 'estimated',
|
||||
fix: {
|
||||
type: 'paste',
|
||||
destination: 'claude-md',
|
||||
|
|
@ -1296,3 +1326,36 @@ describe('buildOptimizeJsonReport', () => {
|
|||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('renderOptimize grouping', () => {
|
||||
const plain = (s: string): string => s.replace(/\[[0-9;]*m/g, '')
|
||||
|
||||
function finding(id: FindingId, title: string): WasteFinding {
|
||||
return {
|
||||
id,
|
||||
title,
|
||||
explanation: 'why',
|
||||
impact: 'medium',
|
||||
tokensSaved: 1000,
|
||||
fix: { type: 'paste', destination: 'prompt', label: 'ask', text: 'ask' },
|
||||
}
|
||||
}
|
||||
|
||||
it('groups findings under fix / habits / FYI with continuous numbering and a basis split', () => {
|
||||
const findings = [
|
||||
finding('bash-output-cap', 'Cap bash output'),
|
||||
finding('claude-md-too-long', 'Trim CLAUDE.md'),
|
||||
finding('context-heavy-sessions', 'Context-heavy sessions'),
|
||||
]
|
||||
const out = plain(renderOptimize(findings, 0.00001, '7 Days', 10, 5, 100, 80, 'B', [], []))
|
||||
|
||||
const headers = ['Fix now (apply-able)', 'Habits', 'FYI'].map(h => out.indexOf(h))
|
||||
expect(headers.every(i => i >= 0)).toBe(true)
|
||||
expect(headers).toEqual([...headers].sort((a, b) => a - b))
|
||||
expect(out).toContain('1. Cap bash output')
|
||||
expect(out).toContain('2. Trim CLAUDE.md')
|
||||
expect(out).toContain('3. Context-heavy sessions')
|
||||
expect(out).toContain('1 measured · 2 estimated')
|
||||
expect(out).not.toContain('Estimates only.')
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue