mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-07-10 01:29:41 +00:00
feat: implement act apply-model and report baseline tripwire (#607)
This commit is contained in:
parent
1678cbd883
commit
ee0ed31a8e
8 changed files with 568 additions and 6 deletions
|
|
@ -65,6 +65,44 @@ export function registerActCommands(program: Command): void {
|
|||
}
|
||||
})
|
||||
|
||||
act
|
||||
.command('apply-model <project>')
|
||||
.description('Apply the model default recommendation for a project')
|
||||
.action(async (project: string) => {
|
||||
try {
|
||||
const { parseAllSessions, filterProjectsByName } = await import('../parser.js')
|
||||
const { recommendModelDefault, buildApplyModelDefaultPlan } = await import('./model-defaults.js')
|
||||
const { runAction } = await import('./apply.js')
|
||||
const chalk = (await import('chalk')).default
|
||||
|
||||
const projects = filterProjectsByName(await parseAllSessions(), [project])
|
||||
const p = projects[0]
|
||||
if (!p) {
|
||||
console.error(`Project "${project}" not found in session history.`)
|
||||
process.exitCode = 1
|
||||
return
|
||||
}
|
||||
|
||||
const recommendation = recommendModelDefault(p)
|
||||
if (!recommendation) {
|
||||
console.error(`No default model recommendation available for ${project} at this time.`)
|
||||
process.exitCode = 1
|
||||
return
|
||||
}
|
||||
|
||||
const plan = await buildApplyModelDefaultPlan(recommendation)
|
||||
const record = await runAction(plan)
|
||||
|
||||
console.log(`Applied default model ${chalk.green(recommendation.candidateModel)} for ${project}`)
|
||||
console.log(chalk.dim(` Evidence: ${recommendation.candidateEditTurns} turns, ${(recommendation.candidateOneShotRate * 100).toFixed(1)}% one-shot, $${recommendation.candidateCostPerEdit.toFixed(3)}/edit`))
|
||||
console.log(chalk.dim(` Undo anytime: codeburn act undo ${shortId(record.id)}`))
|
||||
console.log(chalk.dim(` Per-session override: --model <name>`))
|
||||
} catch (err) {
|
||||
console.error(err instanceof Error ? err.message : String(err))
|
||||
process.exitCode = 1
|
||||
}
|
||||
})
|
||||
|
||||
act
|
||||
.command('report')
|
||||
.description('Realized vs estimated savings for applied actions older than 3 days')
|
||||
|
|
|
|||
170
src/act/model-defaults.ts
Normal file
170
src/act/model-defaults.ts
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
import { readFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
|
||||
import { aggregateModelStats, type ModelStats } from '../compare-stats.js'
|
||||
import type { ProjectSummary } from '../types.js'
|
||||
import { sha256File } from './backup.js'
|
||||
import type { ActionPlan } from './types.js'
|
||||
|
||||
const MIN_EDIT_TURNS = 30
|
||||
const MAX_COST_RATIO = 0.6
|
||||
const ONE_SHOT_TOLERANCE = 0.03
|
||||
const DEBUGGING_HEAVY_THRESHOLD = 0.4
|
||||
const RECENCY_DAYS = 14
|
||||
const MS_PER_DAY = 24 * 60 * 60 * 1000
|
||||
|
||||
export type ModelDefaultRecommendation = {
|
||||
project: string
|
||||
projectPath: string
|
||||
currentModel: string
|
||||
candidateModel: string
|
||||
provider: string
|
||||
currentEditTurns: number
|
||||
candidateEditTurns: number
|
||||
currentOneShotRate: number
|
||||
candidateOneShotRate: number
|
||||
currentCostPerEdit: number
|
||||
candidateCostPerEdit: number
|
||||
savingsPct: number
|
||||
debuggingHeavy: boolean
|
||||
}
|
||||
|
||||
function oneShotRate(s: ModelStats): number {
|
||||
return s.editTurns > 0 ? s.oneShotTurns / s.editTurns : 0
|
||||
}
|
||||
|
||||
function costPerEdit(s: ModelStats): number {
|
||||
return s.editTurns > 0 ? s.editCost / s.editTurns : Number.POSITIVE_INFINITY
|
||||
}
|
||||
|
||||
function isRecent(lastSeen: string, now: Date): boolean {
|
||||
if (!lastSeen) return false
|
||||
const seen = new Date(lastSeen)
|
||||
if (Number.isNaN(seen.getTime())) return false
|
||||
return now.getTime() - seen.getTime() <= RECENCY_DAYS * MS_PER_DAY
|
||||
}
|
||||
|
||||
function providerByModel(project: ProjectSummary): Map<string, string> {
|
||||
const providers = new Map<string, string>()
|
||||
for (const session of project.sessions) {
|
||||
for (const turn of session.turns) {
|
||||
const primary = turn.assistantCalls[0]
|
||||
if (!primary || primary.model === '<synthetic>') continue
|
||||
if (!providers.has(primary.model)) providers.set(primary.model, primary.provider)
|
||||
for (const call of turn.assistantCalls) {
|
||||
if (call.model === '<synthetic>') continue
|
||||
if (!providers.has(call.model)) providers.set(call.model, call.provider)
|
||||
}
|
||||
}
|
||||
}
|
||||
return providers
|
||||
}
|
||||
|
||||
function isDebuggingHeavy(project: ProjectSummary): boolean {
|
||||
let debuggingEditTurns = 0
|
||||
let totalEditTurns = 0
|
||||
for (const session of project.sessions) {
|
||||
for (const breakdown of Object.values(session.categoryBreakdown)) {
|
||||
totalEditTurns += breakdown.editTurns
|
||||
}
|
||||
debuggingEditTurns += session.categoryBreakdown.debugging?.editTurns ?? 0
|
||||
}
|
||||
return totalEditTurns > 0 && debuggingEditTurns / totalEditTurns > DEBUGGING_HEAVY_THRESHOLD
|
||||
}
|
||||
|
||||
export function recommendModelDefault(project: ProjectSummary, opts: { now?: Date } = {}): ModelDefaultRecommendation | null {
|
||||
const now = opts.now ?? new Date()
|
||||
const stats = aggregateModelStats([project])
|
||||
.filter(s => s.model !== '<synthetic>' && s.editTurns >= MIN_EDIT_TURNS)
|
||||
.sort((a, b) => b.editTurns - a.editTurns || b.editCost - a.editCost)
|
||||
|
||||
const current = stats[0]
|
||||
if (!current) return null
|
||||
|
||||
const providers = providerByModel(project)
|
||||
const provider = providers.get(current.model)
|
||||
if (!provider || !isRecent(current.lastSeen, now)) return null
|
||||
|
||||
const currentRate = oneShotRate(current)
|
||||
const currentCost = costPerEdit(current)
|
||||
if (!Number.isFinite(currentCost) || currentCost <= 0) return null
|
||||
|
||||
const debuggingHeavy = isDebuggingHeavy(project)
|
||||
const tolerance = debuggingHeavy ? 0 : ONE_SHOT_TOLERANCE
|
||||
|
||||
const candidates = stats
|
||||
.slice(1)
|
||||
.filter(candidate => providers.get(candidate.model) === provider)
|
||||
.filter(candidate => isRecent(candidate.lastSeen, now))
|
||||
.map(candidate => ({
|
||||
candidate,
|
||||
candidateRate: oneShotRate(candidate),
|
||||
candidateCost: costPerEdit(candidate),
|
||||
}))
|
||||
.filter(({ candidateRate }) => candidateRate >= currentRate - tolerance)
|
||||
.filter(({ candidateCost }) => candidateCost <= currentCost * MAX_COST_RATIO)
|
||||
.sort((a, b) => {
|
||||
const savingsA = 1 - a.candidateCost / currentCost
|
||||
const savingsB = 1 - b.candidateCost / currentCost
|
||||
return savingsB - savingsA || b.candidateRate - a.candidateRate
|
||||
})
|
||||
|
||||
const best = candidates[0]
|
||||
if (!best) return null
|
||||
|
||||
return {
|
||||
project: project.project,
|
||||
projectPath: project.projectPath,
|
||||
currentModel: current.model,
|
||||
candidateModel: best.candidate.model,
|
||||
provider,
|
||||
currentEditTurns: current.editTurns,
|
||||
candidateEditTurns: best.candidate.editTurns,
|
||||
currentOneShotRate: currentRate,
|
||||
candidateOneShotRate: best.candidateRate,
|
||||
currentCostPerEdit: currentCost,
|
||||
candidateCostPerEdit: best.candidateCost,
|
||||
savingsPct: (1 - best.candidateCost / currentCost) * 100,
|
||||
debuggingHeavy,
|
||||
}
|
||||
}
|
||||
|
||||
export async function buildApplyModelDefaultPlan(recommendation: ModelDefaultRecommendation): Promise<ActionPlan> {
|
||||
const settingsPath = join(recommendation.projectPath, '.claude', 'settings.json')
|
||||
let settings: Record<string, unknown> = {}
|
||||
let expectedHash: string | null = null
|
||||
|
||||
try {
|
||||
const raw = await readFile(settingsPath, 'utf-8')
|
||||
expectedHash = await sha256File(settingsPath)
|
||||
settings = JSON.parse(raw) as Record<string, unknown>
|
||||
if (!settings || Array.isArray(settings) || typeof settings !== 'object') settings = {}
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code
|
||||
if (code !== 'ENOENT') throw err
|
||||
}
|
||||
|
||||
settings.model = recommendation.candidateModel
|
||||
|
||||
return {
|
||||
kind: 'model-default',
|
||||
findingId: `model-default:${recommendation.project}`,
|
||||
description: `Set Claude Code default model to ${recommendation.candidateModel} for ${recommendation.project}`,
|
||||
changes: [{
|
||||
op: 'edit',
|
||||
path: settingsPath,
|
||||
content: JSON.stringify(settings, null, 2) + '\n',
|
||||
expectedHash,
|
||||
}],
|
||||
baseline: {
|
||||
windowDays: 30,
|
||||
capturedAt: new Date().toISOString(),
|
||||
estimatedTokens: 0,
|
||||
sessions: recommendation.currentEditTurns + recommendation.candidateEditTurns,
|
||||
metrics: {
|
||||
[recommendation.candidateModel]: recommendation.candidateOneShotRate,
|
||||
[recommendation.currentModel]: recommendation.currentOneShotRate,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
@ -284,6 +284,53 @@ async function guardRow(
|
|||
}
|
||||
}
|
||||
|
||||
async function modelDefaultRow(
|
||||
base: ActReportRow, rec: ActionRecord, sessions: SessionSummary[],
|
||||
baseline: ActionBaseline, afterStart: Date, now: Date,
|
||||
): Promise<ActReportRow> {
|
||||
const models = Object.keys(baseline.metrics)
|
||||
if (models.length < 2) return { ...base, note: 'not measurable: invalid baseline' }
|
||||
const candidateModel = models[0]!
|
||||
const preApplyRate = baseline.metrics[candidateModel]!
|
||||
|
||||
const mockProject: ProjectSummary = {
|
||||
project: 'mock',
|
||||
projectPath: 'mock',
|
||||
totalCostUSD: 0,
|
||||
totalSavingsUSD: 0,
|
||||
totalApiCalls: 0,
|
||||
totalProxiedCostUSD: 0,
|
||||
sessions,
|
||||
}
|
||||
|
||||
const { aggregateModelStats } = await import('../compare-stats.js')
|
||||
const stats = aggregateModelStats([mockProject]).find(s => s.model === candidateModel)
|
||||
|
||||
if (!stats || stats.editTurns < 20) {
|
||||
return { ...base, note: `not measurable: < 20 edit turns for ${candidateModel} since apply` }
|
||||
}
|
||||
|
||||
const postApplyRate = stats.oneShotTurns / stats.editTurns
|
||||
|
||||
if (postApplyRate < preApplyRate - 0.05) {
|
||||
return {
|
||||
...base,
|
||||
status: 'measured',
|
||||
realizedTokens: 0,
|
||||
confidence: 'low',
|
||||
note: `quality regression, consider undo: one-shot rate ${(preApplyRate * 100).toFixed(1)}% -> ${(postApplyRate * 100).toFixed(1)}%`
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
...base,
|
||||
status: 'measured',
|
||||
realizedTokens: 0,
|
||||
confidence: 'normal',
|
||||
note: `correlation, not attribution: one-shot rate ${(preApplyRate * 100).toFixed(1)}% -> ${(postApplyRate * 100).toFixed(1)}%`
|
||||
}
|
||||
}
|
||||
|
||||
async function computeRow(rec: ActionRecord, sessions: SessionSummary[], afterStart: Date, now: Date, opts: ActReportOptions): Promise<ActReportRow> {
|
||||
const estimatedAtApply = rec.baseline?.estimatedTokens ?? 0
|
||||
const base: ActReportRow = {
|
||||
|
|
@ -307,6 +354,7 @@ async function computeRow(rec: ActionRecord, sessions: SessionSummary[], afterSt
|
|||
if (rec.kind === 'claude-md-rule') return readEditRow(base, sessions, baseline, afterStart, now)
|
||||
if (rec.kind === 'shell-config') return { ...base, note: 'not measurable: bash result token sizes are not retained in the summary' }
|
||||
if (rec.kind === 'guard-install') return guardRow(base, afterStart, now, baseline, opts)
|
||||
if (rec.kind === 'model-default') return modelDefaultRow(base, rec, sessions, baseline, afterStart, now)
|
||||
return { ...base, note: 'not measurable: kind is not tracked by act report' }
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import { parseAllSessions } from './parser.js'
|
|||
import { getAllProviders } from './providers/index.js'
|
||||
import type { ProjectSummary, DateRange } from './types.js'
|
||||
import { patchStdoutForWindows } from './ink-win.js'
|
||||
import { recommendModelDefault, type ModelDefaultRecommendation } from './act/model-defaults.js'
|
||||
|
||||
const ORANGE = '#FF8C42'
|
||||
const GREEN = '#5BF5A0'
|
||||
|
|
@ -51,11 +52,12 @@ function barWidth(rate: number): number {
|
|||
|
||||
type ModelSelectorProps = {
|
||||
models: ModelStats[]
|
||||
recommendations: ModelDefaultRecommendation[]
|
||||
onSelect: (a: ModelStats, b: ModelStats) => void
|
||||
onBack: () => void
|
||||
}
|
||||
|
||||
function ModelSelector({ models, onSelect, onBack }: ModelSelectorProps) {
|
||||
function ModelSelector({ models, recommendations, onSelect, onBack }: ModelSelectorProps) {
|
||||
const { exit } = useApp()
|
||||
const [cursor, setCursor] = useState(0)
|
||||
const [selected, setSelected] = useState<Set<number>>(new Set())
|
||||
|
|
@ -126,6 +128,26 @@ function ModelSelector({ models, onSelect, onBack }: ModelSelectorProps) {
|
|||
<Text color={ORANGE} bold>[esc]</Text><Text dimColor> back </Text>
|
||||
<Text color={ORANGE} bold>[q]</Text><Text dimColor> quit</Text>
|
||||
</Text>
|
||||
|
||||
{recommendations.length > 0 && (
|
||||
<Box flexDirection="column" marginTop={1} borderStyle="round" borderColor={ORANGE} paddingX={1}>
|
||||
<Text bold color={ORANGE}>Model defaults recommendation</Text>
|
||||
<Text> </Text>
|
||||
{recommendations.map(rec => (
|
||||
<Box flexDirection="column" key={rec.project} marginBottom={1}>
|
||||
<Text>
|
||||
<Text>{rec.project}: </Text>
|
||||
<Text bold>{rec.currentModel}</Text>
|
||||
<Text>{' -> '}</Text>
|
||||
<Text bold color={GREEN}>{rec.candidateModel}</Text>
|
||||
</Text>
|
||||
<Text color={DIM}> Current: {(rec.currentOneShotRate*100).toFixed(1)}% one-shot over {rec.currentEditTurns} edits, {formatCost(rec.currentCostPerEdit)}/edit</Text>
|
||||
<Text color={DIM}> Candidate: {(rec.candidateOneShotRate*100).toFixed(1)}% one-shot over {rec.candidateEditTurns} edits, {formatCost(rec.candidateCostPerEdit)}/edit</Text>
|
||||
<Text> To apply: <Text color={CYAN}>codeburn act apply-model {rec.project}</Text></Text>
|
||||
</Box>
|
||||
))}
|
||||
</Box>
|
||||
)}
|
||||
</Box>
|
||||
)
|
||||
}
|
||||
|
|
@ -317,6 +339,14 @@ export function CompareView({ projects, onBack }: CompareViewProps) {
|
|||
const { exit } = useApp()
|
||||
const [phase, setPhase] = useState<'select' | 'loading' | 'results'>('select')
|
||||
const [models, setModels] = useState<ModelStats[]>(() => aggregateModelStats(projects))
|
||||
const [recommendations, setRecommendations] = useState<ModelDefaultRecommendation[]>(() => {
|
||||
const recs: ModelDefaultRecommendation[] = []
|
||||
for (const p of projects) {
|
||||
const rec = recommendModelDefault(p)
|
||||
if (rec) recs.push(rec)
|
||||
}
|
||||
return recs
|
||||
})
|
||||
const [pickedNames, setPickedNames] = useState<[string, string] | null>(null)
|
||||
const [selectedA, setSelectedA] = useState<ModelStats | null>(null)
|
||||
const [selectedB, setSelectedB] = useState<ModelStats | null>(null)
|
||||
|
|
@ -331,6 +361,13 @@ export function CompareView({ projects, onBack }: CompareViewProps) {
|
|||
const newModels = aggregateModelStats(projects)
|
||||
setModels(newModels)
|
||||
|
||||
const recs: ModelDefaultRecommendation[] = []
|
||||
for (const p of projects) {
|
||||
const rec = recommendModelDefault(p)
|
||||
if (rec) recs.push(rec)
|
||||
}
|
||||
setRecommendations(recs)
|
||||
|
||||
if (!pickedNames) return
|
||||
const hasA = newModels.some(m => m.model === pickedNames[0])
|
||||
const hasB = newModels.some(m => m.model === pickedNames[1])
|
||||
|
|
@ -460,6 +497,7 @@ export function CompareView({ projects, onBack }: CompareViewProps) {
|
|||
return (
|
||||
<ModelSelector
|
||||
models={models}
|
||||
recommendations={recommendations}
|
||||
onSelect={handleSelect}
|
||||
onBack={onBack}
|
||||
/>
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -10,6 +10,7 @@ import { parseJsonlLine, shouldSkipLine } from './parser.js'
|
|||
import type { DateRange, ProjectSummary } from './types.js'
|
||||
import { formatCost } from './currency.js'
|
||||
import { formatTokens } from './format.js'
|
||||
import { recommendModelDefault, type ModelDefaultRecommendation } from './act/model-defaults.js'
|
||||
|
||||
// ============================================================================
|
||||
// Display constants
|
||||
|
|
@ -233,6 +234,7 @@ export type OptimizeResult = {
|
|||
costRate: number
|
||||
healthScore: number
|
||||
healthGrade: HealthGrade
|
||||
modelRecommendations?: ModelDefaultRecommendation[]
|
||||
}
|
||||
|
||||
export type OptimizeJsonReport = {
|
||||
|
|
@ -263,6 +265,7 @@ export type OptimizeJsonReport = {
|
|||
estimatedSavingsUSD: number
|
||||
fix: WasteAction
|
||||
}>
|
||||
modelRecommendations?: Array<ModelDefaultRecommendation>
|
||||
}
|
||||
|
||||
export type ToolCall = {
|
||||
|
|
@ -2370,7 +2373,7 @@ export async function scanAndDetect(
|
|||
dateRange?: DateRange,
|
||||
): Promise<OptimizeResult> {
|
||||
if (projects.length === 0) {
|
||||
return { findings: [], costRate: 0, healthScore: 100, healthGrade: 'A' }
|
||||
return { findings: [], costRate: 0, healthScore: 100, healthGrade: 'A', modelRecommendations: [] }
|
||||
}
|
||||
|
||||
const key = cacheKey(projects, dateRange)
|
||||
|
|
@ -2421,7 +2424,14 @@ export async function scanAndDetect(
|
|||
|
||||
findings.sort((a, b) => urgencyScore(b) - urgencyScore(a))
|
||||
const { score, grade } = computeHealth(findings)
|
||||
const result: OptimizeResult = { findings, costRate, healthScore: score, healthGrade: grade }
|
||||
|
||||
const modelRecommendations: ModelDefaultRecommendation[] = []
|
||||
for (const project of projects) {
|
||||
const rec = recommendModelDefault(project, { now: dateRange?.end })
|
||||
if (rec) modelRecommendations.push(rec)
|
||||
}
|
||||
|
||||
const result: OptimizeResult = { findings, costRate, healthScore: score, healthGrade: grade, modelRecommendations }
|
||||
resultCache.set(key, { data: result, ts: Date.now() })
|
||||
return result
|
||||
}
|
||||
|
|
@ -2528,6 +2538,7 @@ function renderOptimize(
|
|||
healthGrade: HealthGrade,
|
||||
appliedHeader?: string,
|
||||
previouslyApplied?: Record<string, string>,
|
||||
modelRecommendations?: ModelDefaultRecommendation[],
|
||||
): string {
|
||||
const lines: string[] = []
|
||||
lines.push('')
|
||||
|
|
@ -2573,6 +2584,19 @@ function renderOptimize(
|
|||
lines.push(chalk.hex(DIM)(' ' + SEP.repeat(PANEL_WIDTH)))
|
||||
lines.push(chalk.dim(' Estimates only.'))
|
||||
lines.push('')
|
||||
|
||||
if (modelRecommendations && modelRecommendations.length > 0) {
|
||||
lines.push(chalk.bold.hex(ORANGE)(' Model defaults recommendation'))
|
||||
lines.push(chalk.hex(DIM)(' ' + SEP.repeat(PANEL_WIDTH)))
|
||||
for (const rec of modelRecommendations) {
|
||||
lines.push(` ${rec.project}: ${chalk.bold(rec.currentModel)} -> ${chalk.bold.hex(GREEN)(rec.candidateModel)}`)
|
||||
lines.push(chalk.dim(` Current: ${(rec.currentOneShotRate*100).toFixed(1)}% one-shot over ${rec.currentEditTurns} edits, ${formatCost(rec.currentCostPerEdit)}/edit`))
|
||||
lines.push(chalk.dim(` Candidate: ${(rec.candidateOneShotRate*100).toFixed(1)}% one-shot over ${rec.candidateEditTurns} edits, ${formatCost(rec.candidateCostPerEdit)}/edit`))
|
||||
lines.push(` To apply: ${chalk.hex(CYAN)(`codeburn act apply-model ${rec.project}`)}`)
|
||||
lines.push('')
|
||||
}
|
||||
}
|
||||
|
||||
return lines.join('\n')
|
||||
}
|
||||
|
||||
|
|
@ -2603,7 +2627,7 @@ export async function runOptimize(
|
|||
return
|
||||
}
|
||||
|
||||
const output = renderOptimize(findings, costRate, periodLabel, periodCost, sessions.length, callCount, healthScore, healthGrade, opts.appliedHeader, opts.previouslyApplied)
|
||||
const output = renderOptimize(findings, costRate, periodLabel, periodCost, sessions.length, callCount, healthScore, healthGrade, opts.appliedHeader, opts.previouslyApplied, result.modelRecommendations)
|
||||
console.log(output)
|
||||
}
|
||||
|
||||
|
|
@ -2650,5 +2674,6 @@ export function buildOptimizeJsonReport(
|
|||
estimatedSavingsUSD: f.tokensSaved * result.costRate,
|
||||
fix: f.fix,
|
||||
})),
|
||||
modelRecommendations: result.modelRecommendations,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
243
tests/act-model-defaults.test.ts
Normal file
243
tests/act-model-defaults.test.ts
Normal file
|
|
@ -0,0 +1,243 @@
|
|||
import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { runAction } from '../src/act/apply.js'
|
||||
import { undoAction } from '../src/act/undo.js'
|
||||
import { buildApplyModelDefaultPlan, recommendModelDefault } from '../src/act/model-defaults.js'
|
||||
import type { ClassifiedTurn, ProjectSummary, SessionSummary, TaskCategory } from '../src/types.js'
|
||||
|
||||
const NOW = new Date('2026-07-04T12:00:00.000Z')
|
||||
const RECENT = '2026-07-03T12:00:00.000Z'
|
||||
const OLD = '2026-06-18T12:00:00.000Z'
|
||||
|
||||
function usage() {
|
||||
return {
|
||||
inputTokens: 100,
|
||||
outputTokens: 50,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
}
|
||||
}
|
||||
|
||||
function turn(opts: {
|
||||
model: string
|
||||
provider?: string
|
||||
timestamp?: string
|
||||
costUSD?: number
|
||||
category?: TaskCategory
|
||||
retries?: number
|
||||
hasEdits?: boolean
|
||||
}): ClassifiedTurn {
|
||||
return {
|
||||
userMessage: 'edit the code',
|
||||
timestamp: opts.timestamp ?? RECENT,
|
||||
sessionId: `session-${opts.model}-${opts.timestamp ?? RECENT}-${opts.retries ?? 0}`,
|
||||
category: opts.category ?? 'feature',
|
||||
retries: opts.retries ?? 0,
|
||||
hasEdits: opts.hasEdits ?? true,
|
||||
assistantCalls: [{
|
||||
provider: opts.provider ?? 'claude',
|
||||
model: opts.model,
|
||||
usage: usage(),
|
||||
costUSD: opts.costUSD ?? 1,
|
||||
tools: [],
|
||||
mcpTools: [],
|
||||
skills: [],
|
||||
subagentTypes: [],
|
||||
hasAgentSpawn: false,
|
||||
hasPlanMode: false,
|
||||
speed: 'standard',
|
||||
timestamp: opts.timestamp ?? RECENT,
|
||||
bashCommands: [],
|
||||
deduplicationKey: `key-${opts.model}-${Math.random()}`,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
function repeatTurns(count: number, opts: Parameters<typeof turn>[0]): ClassifiedTurn[] {
|
||||
return Array.from({ length: count }, (_, i) => turn({ ...opts, timestamp: opts.timestamp ?? `2026-07-03T12:${String(i).padStart(2, '0')}:00.000Z` }))
|
||||
}
|
||||
|
||||
function modelTurns(opts: {
|
||||
model: string
|
||||
provider?: string
|
||||
editTurns: number
|
||||
oneShotTurns: number
|
||||
editCost: number
|
||||
timestamp?: string
|
||||
category?: TaskCategory
|
||||
}): ClassifiedTurn[] {
|
||||
const costPerEdit = opts.editCost / opts.editTurns
|
||||
const oneShot = repeatTurns(opts.oneShotTurns, {
|
||||
model: opts.model,
|
||||
provider: opts.provider,
|
||||
timestamp: opts.timestamp,
|
||||
costUSD: costPerEdit,
|
||||
retries: 0,
|
||||
category: opts.category,
|
||||
})
|
||||
const retried = repeatTurns(opts.editTurns - opts.oneShotTurns, {
|
||||
model: opts.model,
|
||||
provider: opts.provider,
|
||||
timestamp: opts.timestamp,
|
||||
costUSD: costPerEdit,
|
||||
retries: 1,
|
||||
category: opts.category,
|
||||
})
|
||||
return [...oneShot, ...retried]
|
||||
}
|
||||
|
||||
function emptyCategoryBreakdown(): SessionSummary['categoryBreakdown'] {
|
||||
return {
|
||||
coding: { turns: 0, costUSD: 0, savingsUSD: 0, retries: 0, editTurns: 0, oneShotTurns: 0 },
|
||||
debugging: { turns: 0, costUSD: 0, savingsUSD: 0, retries: 0, editTurns: 0, oneShotTurns: 0 },
|
||||
feature: { turns: 0, costUSD: 0, savingsUSD: 0, retries: 0, editTurns: 0, oneShotTurns: 0 },
|
||||
refactoring: { turns: 0, costUSD: 0, savingsUSD: 0, retries: 0, editTurns: 0, oneShotTurns: 0 },
|
||||
testing: { turns: 0, costUSD: 0, savingsUSD: 0, retries: 0, editTurns: 0, oneShotTurns: 0 },
|
||||
exploration: { turns: 0, costUSD: 0, savingsUSD: 0, retries: 0, editTurns: 0, oneShotTurns: 0 },
|
||||
planning: { turns: 0, costUSD: 0, savingsUSD: 0, retries: 0, editTurns: 0, oneShotTurns: 0 },
|
||||
delegation: { turns: 0, costUSD: 0, savingsUSD: 0, retries: 0, editTurns: 0, oneShotTurns: 0 },
|
||||
git: { turns: 0, costUSD: 0, savingsUSD: 0, retries: 0, editTurns: 0, oneShotTurns: 0 },
|
||||
'build/deploy': { turns: 0, costUSD: 0, savingsUSD: 0, retries: 0, editTurns: 0, oneShotTurns: 0 },
|
||||
conversation: { turns: 0, costUSD: 0, savingsUSD: 0, retries: 0, editTurns: 0, oneShotTurns: 0 },
|
||||
brainstorming: { turns: 0, costUSD: 0, savingsUSD: 0, retries: 0, editTurns: 0, oneShotTurns: 0 },
|
||||
general: { turns: 0, costUSD: 0, savingsUSD: 0, retries: 0, editTurns: 0, oneShotTurns: 0 },
|
||||
}
|
||||
}
|
||||
|
||||
function projectWithTurns(turns: ClassifiedTurn[], opts: { project?: string; projectPath?: string; debuggingTurns?: number } = {}): ProjectSummary {
|
||||
const categoryBreakdown = emptyCategoryBreakdown()
|
||||
const totalTurns = turns.length
|
||||
const debuggingTurns = opts.debuggingTurns ?? turns.filter(t => t.category === 'debugging').length
|
||||
categoryBreakdown.debugging.turns = debuggingTurns
|
||||
categoryBreakdown.debugging.editTurns = debuggingTurns
|
||||
categoryBreakdown.feature.turns = Math.max(0, totalTurns - debuggingTurns)
|
||||
categoryBreakdown.feature.editTurns = Math.max(0, totalTurns - debuggingTurns)
|
||||
|
||||
return {
|
||||
project: opts.project ?? 'demo-project',
|
||||
projectPath: opts.projectPath ?? '/tmp/demo-project',
|
||||
totalCostUSD: turns.reduce((sum, t) => sum + t.assistantCalls.reduce((s, c) => s + c.costUSD, 0), 0),
|
||||
totalSavingsUSD: 0,
|
||||
totalApiCalls: turns.reduce((sum, t) => sum + t.assistantCalls.length, 0),
|
||||
totalProxiedCostUSD: 0,
|
||||
sessions: [{
|
||||
sessionId: 'session-1',
|
||||
project: opts.project ?? 'demo-project',
|
||||
firstTimestamp: turns[0]?.timestamp ?? RECENT,
|
||||
lastTimestamp: turns.at(-1)?.timestamp ?? RECENT,
|
||||
totalCostUSD: turns.reduce((sum, t) => sum + t.assistantCalls.reduce((s, c) => s + c.costUSD, 0), 0),
|
||||
totalSavingsUSD: 0,
|
||||
totalInputTokens: 0,
|
||||
totalOutputTokens: 0,
|
||||
totalReasoningTokens: 0,
|
||||
totalCacheReadTokens: 0,
|
||||
totalCacheWriteTokens: 0,
|
||||
apiCalls: turns.reduce((sum, t) => sum + t.assistantCalls.length, 0),
|
||||
turns,
|
||||
modelBreakdown: {},
|
||||
toolBreakdown: {},
|
||||
mcpBreakdown: {},
|
||||
bashBreakdown: {},
|
||||
categoryBreakdown,
|
||||
skillBreakdown: {},
|
||||
subagentBreakdown: {},
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
function recommendationProject(overrides: {
|
||||
candidateEditTurns?: number
|
||||
candidateOneShotTurns?: number
|
||||
candidateEditCost?: number
|
||||
candidateProvider?: string
|
||||
candidateTimestamp?: string
|
||||
debuggingTurns?: number
|
||||
} = {}): ProjectSummary {
|
||||
return projectWithTurns([
|
||||
...modelTurns({ model: 'claude-sonnet-4-20250514', provider: 'claude', editTurns: 35, oneShotTurns: 32, editCost: 70 }),
|
||||
...modelTurns({
|
||||
model: 'claude-haiku-3-5-20241022',
|
||||
provider: overrides.candidateProvider ?? 'claude',
|
||||
editTurns: overrides.candidateEditTurns ?? 32,
|
||||
oneShotTurns: overrides.candidateOneShotTurns ?? 29,
|
||||
editCost: overrides.candidateEditCost ?? 30,
|
||||
timestamp: overrides.candidateTimestamp,
|
||||
}),
|
||||
], { debuggingTurns: overrides.debuggingTurns })
|
||||
}
|
||||
|
||||
describe('model default recommendations', () => {
|
||||
it('recommends a same-provider candidate with enough volume, recent data, similar quality, and <=60% cost per edit', () => {
|
||||
const recommendation = recommendModelDefault(recommendationProject(), { now: NOW })
|
||||
|
||||
expect(recommendation).toMatchObject({
|
||||
project: 'demo-project',
|
||||
currentModel: 'claude-sonnet-4-20250514',
|
||||
candidateModel: 'claude-haiku-3-5-20241022',
|
||||
provider: 'claude',
|
||||
})
|
||||
expect(recommendation?.currentOneShotRate).toBeCloseTo(32 / 35, 5)
|
||||
expect(recommendation?.candidateOneShotRate).toBeCloseTo(29 / 32, 5)
|
||||
expect(recommendation?.savingsPct).toBeGreaterThan(50)
|
||||
})
|
||||
|
||||
it('rejects candidates below the 30 edit-turn minimum', () => {
|
||||
expect(recommendModelDefault(recommendationProject({ candidateEditTurns: 29, candidateOneShotTurns: 27 }), { now: NOW })).toBeNull()
|
||||
})
|
||||
|
||||
it('rejects candidates more than 3pp below the current model one-shot rate', () => {
|
||||
expect(recommendModelDefault(recommendationProject({ candidateOneShotTurns: 28 }), { now: NOW })).toBeNull()
|
||||
})
|
||||
|
||||
it('rejects candidates that cost more than 60% of the current model per edit', () => {
|
||||
expect(recommendModelDefault(recommendationProject({ candidateEditCost: 43 }), { now: NOW })).toBeNull()
|
||||
})
|
||||
|
||||
it('rejects candidates last seen more than 14 days ago', () => {
|
||||
expect(recommendModelDefault(recommendationProject({ candidateTimestamp: OLD }), { now: NOW })).toBeNull()
|
||||
})
|
||||
|
||||
it('rejects cross-provider candidates in v1', () => {
|
||||
expect(recommendModelDefault(recommendationProject({ candidateProvider: 'openai' }), { now: NOW })).toBeNull()
|
||||
})
|
||||
|
||||
it('uses zero tolerance for debugging-heavy projects', () => {
|
||||
const project = recommendationProject({ debuggingTurns: 40 })
|
||||
|
||||
expect(recommendModelDefault(project, { now: NOW })).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('model default apply plan', () => {
|
||||
it('writes only the model key while preserving existing Claude settings and journals model-default', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'codeburn-model-default-'))
|
||||
const actionsDir = join(dir, '.codeburn-actions')
|
||||
const projectPath = join(dir, 'project')
|
||||
const settingsPath = join(projectPath, '.claude', 'settings.json')
|
||||
const original = '{\n "enabledTools": ["Bash"],\n "model": "claude-sonnet-4-20250514"\n}\n'
|
||||
|
||||
try {
|
||||
await mkdir(dirname(settingsPath), { recursive: true })
|
||||
await writeFile(settingsPath, original, { encoding: 'utf-8' })
|
||||
const recommendation = recommendModelDefault(recommendationProject(), { now: NOW })!
|
||||
const plan = await buildApplyModelDefaultPlan({ ...recommendation, projectPath })
|
||||
const record = await runAction(plan, actionsDir)
|
||||
|
||||
const updated = JSON.parse(await readFile(settingsPath, 'utf-8'))
|
||||
expect(updated).toEqual({ enabledTools: ['Bash'], model: 'claude-haiku-3-5-20241022' })
|
||||
expect(record.kind).toBe('model-default')
|
||||
expect(record.findingId).toBe('model-default:demo-project')
|
||||
|
||||
await undoAction({ id: record.id }, { actionsDir })
|
||||
expect(await readFile(settingsPath, 'utf-8')).toBe(original)
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue