mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-08-03 21:35:13 +00:00
343 lines
14 KiB
TypeScript
343 lines
14 KiB
TypeScript
import { describe, expect, it } from 'vitest'
|
|
|
|
import {
|
|
scanUserCorrections,
|
|
medianTimeToFirstEditMs,
|
|
sessionTimeToFirstEditMs,
|
|
aggregateFileChurn,
|
|
computePricingCoverage,
|
|
worstOneShotCategory,
|
|
buildCoachingNotes,
|
|
USER_CORRECTION_PATTERNS,
|
|
} from '../src/workflow-insights.js'
|
|
import { isExpectedFreeModel } from '../src/models.js'
|
|
import type { ClassifiedTurn, ParsedApiCall, ProjectSummary, SessionSummary, TaskCategory, ToolCall } from '../src/types.js'
|
|
|
|
function call(opts: { tools?: string[]; timestamp?: string; toolSequence?: ToolCall[][]; model?: string; costUSD?: number } = {}): ParsedApiCall {
|
|
return {
|
|
provider: 'claude',
|
|
model: opts.model ?? 'claude-sonnet-4-5',
|
|
usage: {
|
|
inputTokens: 0, outputTokens: 0, cacheCreationInputTokens: 0,
|
|
cacheReadInputTokens: 0, cachedInputTokens: 0, reasoningTokens: 0, webSearchRequests: 0,
|
|
},
|
|
costUSD: opts.costUSD ?? 0,
|
|
tools: opts.tools ?? [],
|
|
mcpTools: [],
|
|
skills: [],
|
|
subagentTypes: [],
|
|
hasAgentSpawn: false,
|
|
hasPlanMode: false,
|
|
speed: 'standard',
|
|
timestamp: opts.timestamp ?? '2026-05-05T00:00:00.000Z',
|
|
bashCommands: [],
|
|
deduplicationKey: 'k',
|
|
...(opts.toolSequence ? { toolSequence: opts.toolSequence } : {}),
|
|
}
|
|
}
|
|
|
|
function turn(opts: { userMessage?: string; calls?: ParsedApiCall[]; timestamp?: string; sessionId?: string } = {}): ClassifiedTurn {
|
|
return {
|
|
userMessage: opts.userMessage ?? '',
|
|
assistantCalls: opts.calls ?? [],
|
|
timestamp: opts.timestamp ?? '2026-05-05T00:00:00.000Z',
|
|
sessionId: opts.sessionId ?? 's1',
|
|
category: 'coding',
|
|
retries: 0,
|
|
hasEdits: (opts.calls ?? []).some(c => c.tools.some(t => t === 'Edit' || t === 'Write')),
|
|
}
|
|
}
|
|
|
|
function session(sessionId: string, turns: ClassifiedTurn[], categoryBreakdown: SessionSummary['categoryBreakdown'] = {} as SessionSummary['categoryBreakdown']): SessionSummary {
|
|
return {
|
|
sessionId,
|
|
project: 'app',
|
|
firstTimestamp: turns[0]?.timestamp ?? '2026-05-05T00:00:00.000Z',
|
|
lastTimestamp: '2026-05-05T01:00:00.000Z',
|
|
totalCostUSD: 0, totalInputTokens: 0, totalOutputTokens: 0, totalReasoningTokens: 0,
|
|
totalCacheReadTokens: 0, totalCacheWriteTokens: 0, totalSavingsUSD: 0,
|
|
apiCalls: turns.reduce((s, t) => s + t.assistantCalls.length, 0),
|
|
turns,
|
|
modelBreakdown: {}, toolBreakdown: {}, mcpBreakdown: {}, bashBreakdown: {},
|
|
categoryBreakdown,
|
|
skillBreakdown: {}, subagentBreakdown: {},
|
|
}
|
|
}
|
|
|
|
function project(sessions: SessionSummary[], projectPath = '/home/u/app'): ProjectSummary {
|
|
return {
|
|
project: 'app', projectPath, sessions,
|
|
totalCostUSD: 0, totalSavingsUSD: 0, totalApiCalls: 0, totalProxiedCostUSD: 0,
|
|
}
|
|
}
|
|
|
|
function cat(editTurns: number, oneShotTurns: number): SessionSummary['categoryBreakdown'][TaskCategory] {
|
|
return { turns: editTurns, costUSD: 0, savingsUSD: 0, retries: 0, editTurns, oneShotTurns }
|
|
}
|
|
|
|
describe('scanUserCorrections', () => {
|
|
it('counts follow-up turns whose user message signals a correction', () => {
|
|
const p = project([session('s1', [
|
|
turn({ userMessage: "add a new feature please" }),
|
|
turn({ userMessage: "no, I meant the other file" }),
|
|
turn({ userMessage: "that's not what I asked for" }),
|
|
turn({ userMessage: "you missed the edge case" }),
|
|
turn({ userMessage: "revert that change" }),
|
|
])])
|
|
const r = scanUserCorrections([p])
|
|
expect(r.corrections).toBe(4)
|
|
expect(r.userTurns).toBe(5)
|
|
expect(r.correctionRate).toBeCloseTo(0.8)
|
|
})
|
|
|
|
it('never counts the session-opening prompt, however correction-shaped', () => {
|
|
// An opener cannot be correcting THIS assistant; "revert the last release"
|
|
// as a first message is a task, not a correction.
|
|
const p = project([session('s1', [
|
|
turn({ userMessage: 'revert the last release' }),
|
|
turn({ userMessage: 'thanks, looks good' }),
|
|
])])
|
|
const r = scanUserCorrections([p])
|
|
expect(r.corrections).toBe(0)
|
|
expect(r.userTurns).toBe(2)
|
|
})
|
|
|
|
it('does not match "revert the <noun>" task phrasing even as a follow-up', () => {
|
|
const p = project([session('s1', [
|
|
turn({ userMessage: 'build the feature' }),
|
|
turn({ userMessage: 'now revert the migration we shipped last week' }),
|
|
])])
|
|
expect(scanUserCorrections([p]).corrections).toBe(0)
|
|
})
|
|
|
|
it('does not flag praise or ordinary requests (false-positive guards)', () => {
|
|
const phrases = [
|
|
'you were right about that',
|
|
"you're right, thanks",
|
|
"that's right, ship it",
|
|
'looks correct to me',
|
|
'undo the migration when done',
|
|
'the build is failing, can you fix it',
|
|
'what went wrong here',
|
|
]
|
|
const p = project([session('s1', phrases.map(m => turn({ userMessage: m })))])
|
|
const r = scanUserCorrections([p])
|
|
expect(r.corrections).toBe(0)
|
|
})
|
|
|
|
it('ignores continuation turns with no fresh prompt', () => {
|
|
const p = project([session('s1', [
|
|
turn({ userMessage: 'build the feature' }),
|
|
turn({ userMessage: '' }),
|
|
turn({ userMessage: ' ' }),
|
|
turn({ userMessage: 'that is wrong' }),
|
|
])])
|
|
const r = scanUserCorrections([p])
|
|
expect(r.userTurns).toBe(2)
|
|
expect(r.corrections).toBe(1)
|
|
expect(r.correctionRate).toBe(0.5)
|
|
})
|
|
|
|
it('returns a null rate for empty input', () => {
|
|
const r = scanUserCorrections([])
|
|
expect(r).toEqual({ corrections: 0, userTurns: 0, correctionRate: null })
|
|
})
|
|
|
|
it('every pattern is case-insensitive', () => {
|
|
for (const re of USER_CORRECTION_PATTERNS) expect(re.flags).toContain('i')
|
|
})
|
|
})
|
|
|
|
describe('medianTimeToFirstEditMs', () => {
|
|
const edit = (ts: string) => call({ tools: ['Edit'], timestamp: ts })
|
|
|
|
it('measures from the first turn to the first edit call', () => {
|
|
const s = session('s1', [
|
|
turn({ timestamp: '2026-05-05T00:00:00.000Z', calls: [call({ tools: ['Read'], timestamp: '2026-05-05T00:00:10.000Z' })] }),
|
|
turn({ timestamp: '2026-05-05T00:01:00.000Z', calls: [edit('2026-05-05T00:00:30.000Z')] }),
|
|
])
|
|
expect(medianTimeToFirstEditMs([project([s])])).toBe(30_000)
|
|
})
|
|
|
|
it('excludes sessions with no edits (not counted as zero)', () => {
|
|
const withEdit = session('s1', [turn({ timestamp: '2026-05-05T00:00:00.000Z', calls: [edit('2026-05-05T00:00:20.000Z')] })])
|
|
const noEdit = session('s2', [turn({ timestamp: '2026-05-05T00:00:00.000Z', calls: [call({ tools: ['Read'], timestamp: '2026-05-05T00:05:00.000Z' })] })])
|
|
expect(medianTimeToFirstEditMs([project([withEdit, noEdit])])).toBe(20_000)
|
|
})
|
|
|
|
it('takes the median across sessions (even count averages the middle two)', () => {
|
|
const mk = (id: string, secs: number) => session(id, [turn({ timestamp: '2026-05-05T00:00:00.000Z', calls: [edit(new Date(Date.parse('2026-05-05T00:00:00.000Z') + secs * 1000).toISOString())] })])
|
|
// deltas: 10, 20, 30, 40 -> median (20+30)/2 = 25s
|
|
expect(medianTimeToFirstEditMs([project([mk('a', 10), mk('b', 20), mk('c', 30), mk('d', 40)])])).toBe(25_000)
|
|
})
|
|
|
|
it('clamps out-of-order timestamps to zero', () => {
|
|
const s = session('s1', [turn({ timestamp: '2026-05-05T00:01:00.000Z', calls: [edit('2026-05-05T00:00:00.000Z')] })])
|
|
expect(medianTimeToFirstEditMs([project([s])])).toBe(0)
|
|
})
|
|
|
|
it('returns null when no session has an edit', () => {
|
|
const s = session('s1', [turn({ timestamp: '2026-05-05T00:00:00.000Z', calls: [call({ tools: ['Read'] })] })])
|
|
expect(medianTimeToFirstEditMs([project([s])])).toBeNull()
|
|
})
|
|
})
|
|
|
|
describe('aggregateFileChurn', () => {
|
|
const editSeq = (file: string): ToolCall[][] => [[{ tool: 'Edit', file }]]
|
|
|
|
it('ranks by distinct sessions then total edits, and relativizes paths', () => {
|
|
const s1 = session('s1', [
|
|
turn({ sessionId: 's1', calls: [call({ tools: ['Edit'], toolSequence: editSeq('/home/u/app/src/a.ts') })] }),
|
|
turn({ sessionId: 's1', calls: [call({ tools: ['Edit'], toolSequence: editSeq('/home/u/app/src/a.ts') })] }),
|
|
turn({ sessionId: 's1', calls: [call({ tools: ['Write'], toolSequence: [[{ tool: 'Write', file: '/home/u/app/src/b.ts' }]] })] }),
|
|
])
|
|
const s2 = session('s2', [
|
|
turn({ sessionId: 's2', calls: [call({ tools: ['Edit'], toolSequence: editSeq('/home/u/app/src/a.ts') })] }),
|
|
])
|
|
const files = aggregateFileChurn([project([s1, s2])])
|
|
expect(files[0]).toEqual({ path: 'src/a.ts', sessions: 2, edits: 3 })
|
|
expect(files[1]).toEqual({ path: 'src/b.ts', sessions: 1, edits: 1 })
|
|
})
|
|
|
|
it('breaks a session-count tie by edits then path', () => {
|
|
const s = session('s1', [
|
|
turn({ sessionId: 's1', calls: [call({ tools: ['Edit'], toolSequence: editSeq('/home/u/app/z.ts') })] }),
|
|
turn({ sessionId: 's1', calls: [call({ tools: ['Edit'], toolSequence: editSeq('/home/u/app/z.ts') })] }),
|
|
turn({ sessionId: 's1', calls: [call({ tools: ['Edit'], toolSequence: editSeq('/home/u/app/a.ts') })] }),
|
|
])
|
|
const files = aggregateFileChurn([project([s])])
|
|
// both touched by 1 session; z.ts has more edits so it ranks first
|
|
expect(files.map(f => f.path)).toEqual(['z.ts', 'a.ts'])
|
|
})
|
|
|
|
it('ignores non-edit tools and calls without a file path', () => {
|
|
const s = session('s1', [
|
|
turn({ sessionId: 's1', calls: [call({ tools: ['Read'], toolSequence: [[{ tool: 'Read', file: '/home/u/app/r.ts' }]] })] }),
|
|
turn({ sessionId: 's1', calls: [call({ tools: ['Edit'], toolSequence: [[{ tool: 'Edit' }]] })] }),
|
|
])
|
|
expect(aggregateFileChurn([project([s])])).toEqual([])
|
|
})
|
|
|
|
it('respects the limit', () => {
|
|
const turns = Array.from({ length: 5 }, (_, i) =>
|
|
turn({ sessionId: 's1', calls: [call({ tools: ['Edit'], toolSequence: editSeq(`/home/u/app/f${i}.ts`) })] }))
|
|
const files = aggregateFileChurn([project([session('s1', turns)])], 3)
|
|
expect(files).toHaveLength(3)
|
|
})
|
|
})
|
|
|
|
describe('computePricingCoverage', () => {
|
|
it('is 1 when everything is priced', () => {
|
|
expect(computePricingCoverage(100, 0)).toBe(1)
|
|
})
|
|
it('is the priced share when some calls are unpriced', () => {
|
|
expect(computePricingCoverage(100, 25)).toBe(0.75)
|
|
})
|
|
it('is 1 when there are no cost-bearing calls', () => {
|
|
expect(computePricingCoverage(0, 0)).toBe(1)
|
|
})
|
|
it('clamps to 0 if unpriced somehow exceeds the total', () => {
|
|
expect(computePricingCoverage(10, 20)).toBe(0)
|
|
})
|
|
})
|
|
|
|
describe('worstOneShotCategory', () => {
|
|
it('picks the lowest one-shot rate above the edit-turn floor', () => {
|
|
const s = session('s1', [], {
|
|
feature: cat(10, 3), // 30%
|
|
debugging: cat(8, 6), // 75%
|
|
} as SessionSummary['categoryBreakdown'])
|
|
const worst = worstOneShotCategory([project([s])])
|
|
expect(worst?.category).toBe('Feature Dev')
|
|
expect(worst?.rate).toBe(30)
|
|
expect(worst?.editTurns).toBe(10)
|
|
})
|
|
|
|
it('excludes categories below the edit-turn floor', () => {
|
|
const s = session('s1', [], {
|
|
feature: cat(2, 0), // 0% but only 2 edit turns
|
|
debugging: cat(10, 8), // 80%
|
|
} as SessionSummary['categoryBreakdown'])
|
|
const worst = worstOneShotCategory([project([s])])
|
|
expect(worst?.category).toBe('Debugging')
|
|
})
|
|
|
|
it('returns null when nothing qualifies', () => {
|
|
const s = session('s1', [], { feature: cat(1, 0) } as SessionSummary['categoryBreakdown'])
|
|
expect(worstOneShotCategory([project([s])])).toBeNull()
|
|
})
|
|
})
|
|
|
|
describe('buildCoachingNotes', () => {
|
|
it('emits a note per strong signal and never uses em-dashes', () => {
|
|
const notes = buildCoachingNotes({
|
|
worstOneShot: { category: 'Feature Dev', rate: 40, editTurns: 12 },
|
|
corrections: 6,
|
|
correctionRate: 0.2,
|
|
topReworkedFile: { path: 'src/a.ts', sessions: 4, edits: 20 },
|
|
medianTimeToFirstEditMs: 6 * 60 * 1000,
|
|
})
|
|
expect(notes.length).toBeGreaterThan(0)
|
|
expect(notes.length).toBeLessThanOrEqual(3)
|
|
for (const n of notes) expect(n).not.toContain('—')
|
|
})
|
|
|
|
it('caps at 3 notes even when all four signals fire', () => {
|
|
const notes = buildCoachingNotes({
|
|
worstOneShot: { category: 'Feature Dev', rate: 40, editTurns: 12 },
|
|
corrections: 6,
|
|
correctionRate: 0.2,
|
|
topReworkedFile: { path: 'src/a.ts', sessions: 4, edits: 20 },
|
|
medianTimeToFirstEditMs: 6 * 60 * 1000,
|
|
})
|
|
expect(notes).toHaveLength(3)
|
|
})
|
|
|
|
it('stays silent when every signal is below threshold', () => {
|
|
const notes = buildCoachingNotes({
|
|
worstOneShot: { category: 'Feature Dev', rate: 90, editTurns: 12 },
|
|
corrections: 1,
|
|
correctionRate: 0.02,
|
|
topReworkedFile: { path: 'src/a.ts', sessions: 1, edits: 2 },
|
|
medianTimeToFirstEditMs: 10 * 1000,
|
|
})
|
|
expect(notes).toEqual([])
|
|
})
|
|
|
|
it('requires both a high rate and a minimum count for the correction note', () => {
|
|
const highRateLowCount = buildCoachingNotes({ correctionRate: 0.5, corrections: 1 })
|
|
expect(highRateLowCount).toEqual([])
|
|
})
|
|
})
|
|
|
|
// ── Review-findings regressions (post-#756 review) ─────────────────────────
|
|
describe('review-findings regressions', () => {
|
|
const edit = (ts: string) => call({ tools: ['Edit'], timestamp: ts })
|
|
|
|
it('an unparseable FIRST edit timestamp yields null, never time-to-a-later-edit', () => {
|
|
const s = session('s1', [
|
|
turn({ timestamp: '2026-06-01T10:00:00Z', calls: [edit('garbage-timestamp')] }),
|
|
turn({ timestamp: '2026-06-01T10:30:00Z', calls: [edit('2026-06-01T10:30:00Z')] }),
|
|
])
|
|
expect(sessionTimeToFirstEditMs(s)).toBeNull()
|
|
})
|
|
|
|
it('normalizes backslashed Windows paths: one churn entry, relativized, no username leak', () => {
|
|
const p = project([session('s1', [
|
|
turn({ calls: [call({ tools: ['Edit'], toolSequence: [[{ tool: 'Edit', file: 'C:\\work\\proj\\src\\a.ts' }]] })] }),
|
|
turn({ calls: [call({ tools: ['Edit'], toolSequence: [[{ tool: 'Edit', file: 'C:/work/proj/src/a.ts' }]] })] }),
|
|
])], 'C:\\work\\proj')
|
|
const churn = aggregateFileChurn([p])
|
|
expect(churn).toHaveLength(1)
|
|
expect(churn[0]!.path).toBe('src/a.ts')
|
|
expect(churn[0]!.edits).toBe(2)
|
|
})
|
|
|
|
it('isExpectedFreeModel excludes local-style models from the coverage denominator', () => {
|
|
expect(isExpectedFreeModel('qwen3.6:35b-a3b-bf16')).toBe(true)
|
|
expect(isExpectedFreeModel('llama-3-8b-q4')).toBe(true)
|
|
expect(isExpectedFreeModel('claude-opus-4-8')).toBe(false)
|
|
// 95 local calls + 5 unpriced cloud calls: coverage must be 0, not 0.95.
|
|
expect(computePricingCoverage(5, 5)).toBe(0)
|
|
})
|
|
})
|