mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-08-14 19:14:28 +00:00
feat(compare): model comparison with planning rate fix
5-section compare view: Performance (one-shot, retry, self-correction), Efficiency (cost/call, cost/edit, output/call, cache hit), Category Head-to-Head bar charts, Working Style, and Context. Planning rate now detects TaskCreate/TaskUpdate/TodoWrite instead of only EnterPlanMode (which was never used, showing 0% for all models). Validated against raw JSONL with zero false positives. Responsive side-by-side layout at 90+ cols. Self-correction scanner with compact file skipping and model+timestamp dedup. 274 tests.
This commit is contained in:
iamtoruk
AgentSeal
parent
fb24eea186
commit
bd43b15342
4 changed files with 536 additions and 73 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -36,3 +36,6 @@ npm-debug.log*
|
|||
|
||||
# Local Discord brand / promo assets not yet ready to publish
|
||||
assets/discord-*.png
|
||||
|
||||
# Desktop app experiments
|
||||
desktop/
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ import { join } from 'path'
|
|||
|
||||
import type { ProjectSummary } from './types.js'
|
||||
|
||||
const PLANNING_TOOLS = new Set(['TaskCreate', 'TaskUpdate', 'TodoWrite', 'EnterPlanMode', 'ExitPlanMode'])
|
||||
|
||||
export type ModelStats = {
|
||||
model: string
|
||||
calls: number
|
||||
|
|
@ -16,6 +18,7 @@ export type ModelStats = {
|
|||
oneShotTurns: number
|
||||
retries: number
|
||||
selfCorrections: number
|
||||
editCost: number
|
||||
firstSeen: string
|
||||
lastSeen: string
|
||||
}
|
||||
|
|
@ -26,7 +29,7 @@ export function aggregateModelStats(projects: ProjectSummary[]): ModelStats[] {
|
|||
const ensure = (model: string): ModelStats => {
|
||||
let s = byModel.get(model)
|
||||
if (!s) {
|
||||
s = { model, calls: 0, cost: 0, outputTokens: 0, inputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, totalTurns: 0, editTurns: 0, oneShotTurns: 0, retries: 0, selfCorrections: 0, firstSeen: '', lastSeen: '' }
|
||||
s = { model, calls: 0, cost: 0, outputTokens: 0, inputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, totalTurns: 0, editTurns: 0, oneShotTurns: 0, retries: 0, selfCorrections: 0, editCost: 0, firstSeen: '', lastSeen: '' }
|
||||
byModel.set(model, s)
|
||||
}
|
||||
return s
|
||||
|
|
@ -41,8 +44,13 @@ export function aggregateModelStats(projects: ProjectSummary[]): ModelStats[] {
|
|||
|
||||
const ms = ensure(primaryModel)
|
||||
ms.totalTurns++
|
||||
if (turn.hasEdits) ms.editTurns++
|
||||
if (turn.hasEdits && turn.retries === 0) ms.oneShotTurns++
|
||||
if (turn.hasEdits) {
|
||||
ms.editTurns++
|
||||
if (turn.retries === 0) ms.oneShotTurns++
|
||||
for (const c of turn.assistantCalls) {
|
||||
if (c.model !== '<synthetic>') ms.editCost += c.costUSD
|
||||
}
|
||||
}
|
||||
ms.retries += turn.retries
|
||||
|
||||
for (const call of turn.assistantCalls) {
|
||||
|
|
@ -66,6 +74,7 @@ export function aggregateModelStats(projects: ProjectSummary[]): ModelStats[] {
|
|||
}
|
||||
|
||||
export type ComparisonRow = {
|
||||
section: string
|
||||
label: string
|
||||
valueA: number | null
|
||||
valueB: number | null
|
||||
|
|
@ -73,7 +82,26 @@ export type ComparisonRow = {
|
|||
winner: 'a' | 'b' | 'tie' | 'none'
|
||||
}
|
||||
|
||||
export type CategoryComparison = {
|
||||
category: string
|
||||
turnsA: number
|
||||
editTurnsA: number
|
||||
oneShotRateA: number | null
|
||||
turnsB: number
|
||||
editTurnsB: number
|
||||
oneShotRateB: number | null
|
||||
winner: 'a' | 'b' | 'tie' | 'none'
|
||||
}
|
||||
|
||||
export type WorkingStyleRow = {
|
||||
label: string
|
||||
valueA: number | null
|
||||
valueB: number | null
|
||||
formatFn: ComparisonRow['formatFn']
|
||||
}
|
||||
|
||||
type MetricDef = {
|
||||
section: string
|
||||
label: string
|
||||
formatFn: ComparisonRow['formatFn']
|
||||
higherIsBetter: boolean
|
||||
|
|
@ -82,18 +110,49 @@ type MetricDef = {
|
|||
|
||||
const METRICS: MetricDef[] = [
|
||||
{
|
||||
section: 'Performance',
|
||||
label: 'One-shot rate',
|
||||
formatFn: 'percent',
|
||||
higherIsBetter: true,
|
||||
compute: s => s.editTurns > 0 ? (s.oneShotTurns / s.editTurns) * 100 : null,
|
||||
},
|
||||
{
|
||||
section: 'Performance',
|
||||
label: 'Retry rate',
|
||||
formatFn: 'decimal',
|
||||
higherIsBetter: false,
|
||||
compute: s => s.editTurns > 0 ? s.retries / s.editTurns : null,
|
||||
},
|
||||
{
|
||||
section: 'Performance',
|
||||
label: 'Self-correction',
|
||||
formatFn: 'percent',
|
||||
higherIsBetter: false,
|
||||
compute: s => s.totalTurns > 0 ? (s.selfCorrections / s.totalTurns) * 100 : null,
|
||||
},
|
||||
{
|
||||
section: 'Efficiency',
|
||||
label: 'Cost / call',
|
||||
formatFn: 'cost',
|
||||
higherIsBetter: false,
|
||||
compute: s => s.calls > 0 ? s.cost / s.calls : null,
|
||||
},
|
||||
{
|
||||
section: 'Efficiency',
|
||||
label: 'Cost / edit',
|
||||
formatFn: 'cost',
|
||||
higherIsBetter: false,
|
||||
compute: s => s.editTurns > 0 ? s.editCost / s.editTurns : null,
|
||||
},
|
||||
{
|
||||
section: 'Efficiency',
|
||||
label: 'Output tok / call',
|
||||
formatFn: 'number',
|
||||
higherIsBetter: false,
|
||||
compute: s => s.calls > 0 ? Math.round(s.outputTokens / s.calls) : null,
|
||||
},
|
||||
{
|
||||
section: 'Efficiency',
|
||||
label: 'Cache hit rate',
|
||||
formatFn: 'percent',
|
||||
higherIsBetter: true,
|
||||
|
|
@ -102,24 +161,6 @@ const METRICS: MetricDef[] = [
|
|||
return total > 0 ? (s.cacheReadTokens / total) * 100 : null
|
||||
},
|
||||
},
|
||||
{
|
||||
label: 'One-shot rate',
|
||||
formatFn: 'percent',
|
||||
higherIsBetter: true,
|
||||
compute: s => s.editTurns > 0 ? (s.oneShotTurns / s.editTurns) * 100 : null,
|
||||
},
|
||||
{
|
||||
label: 'Retry rate',
|
||||
formatFn: 'decimal',
|
||||
higherIsBetter: false,
|
||||
compute: s => s.editTurns > 0 ? s.retries / s.editTurns : null,
|
||||
},
|
||||
{
|
||||
label: 'Self-correction',
|
||||
formatFn: 'percent',
|
||||
higherIsBetter: false,
|
||||
compute: s => s.totalTurns > 0 ? (s.selfCorrections / s.totalTurns) * 100 : null,
|
||||
},
|
||||
]
|
||||
|
||||
function pickWinner(valueA: number | null, valueB: number | null, higherIsBetter: boolean): ComparisonRow['winner'] {
|
||||
|
|
@ -134,6 +175,7 @@ export function computeComparison(a: ModelStats, b: ModelStats): ComparisonRow[]
|
|||
const valueA = m.compute(a)
|
||||
const valueB = m.compute(b)
|
||||
return {
|
||||
section: m.section,
|
||||
label: m.label,
|
||||
valueA,
|
||||
valueB,
|
||||
|
|
@ -143,6 +185,98 @@ export function computeComparison(a: ModelStats, b: ModelStats): ComparisonRow[]
|
|||
})
|
||||
}
|
||||
|
||||
export function computeCategoryComparison(projects: ProjectSummary[], modelA: string, modelB: string): CategoryComparison[] {
|
||||
type Accum = { turns: number; editTurns: number; oneShotTurns: number }
|
||||
const mapA = new Map<string, Accum>()
|
||||
const mapB = new Map<string, Accum>()
|
||||
|
||||
const ensure = (map: Map<string, Accum>, cat: string): Accum => {
|
||||
let a = map.get(cat)
|
||||
if (!a) { a = { turns: 0, editTurns: 0, oneShotTurns: 0 }; map.set(cat, a) }
|
||||
return a
|
||||
}
|
||||
|
||||
for (const project of projects) {
|
||||
for (const session of project.sessions) {
|
||||
for (const turn of session.turns) {
|
||||
if (turn.assistantCalls.length === 0) continue
|
||||
const primary = turn.assistantCalls[0]!.model
|
||||
if (primary !== modelA && primary !== modelB) continue
|
||||
|
||||
const acc = ensure(primary === modelA ? mapA : mapB, turn.category)
|
||||
acc.turns++
|
||||
if (turn.hasEdits) {
|
||||
acc.editTurns++
|
||||
if (turn.retries === 0) acc.oneShotTurns++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const allCats = new Set([...mapA.keys(), ...mapB.keys()])
|
||||
const result: CategoryComparison[] = []
|
||||
|
||||
for (const category of allCats) {
|
||||
const a = mapA.get(category)
|
||||
const b = mapB.get(category)
|
||||
if ((!a || a.editTurns === 0) && (!b || b.editTurns === 0)) continue
|
||||
|
||||
const rateA = a && a.editTurns > 0 ? (a.oneShotTurns / a.editTurns) * 100 : null
|
||||
const rateB = b && b.editTurns > 0 ? (b.oneShotTurns / b.editTurns) * 100 : null
|
||||
|
||||
result.push({
|
||||
category,
|
||||
turnsA: a?.turns ?? 0,
|
||||
editTurnsA: a?.editTurns ?? 0,
|
||||
oneShotRateA: rateA,
|
||||
turnsB: b?.turns ?? 0,
|
||||
editTurnsB: b?.editTurns ?? 0,
|
||||
oneShotRateB: rateB,
|
||||
winner: pickWinner(rateA, rateB, true),
|
||||
})
|
||||
}
|
||||
|
||||
return result.sort((a, b) => (b.turnsA + b.turnsB) - (a.turnsA + a.turnsB))
|
||||
}
|
||||
|
||||
export function computeWorkingStyle(projects: ProjectSummary[], modelA: string, modelB: string): WorkingStyleRow[] {
|
||||
type StyleAccum = { totalTurns: number; agentSpawns: number; planModeUses: number; totalToolCalls: number; fastModeCalls: number }
|
||||
const sA: StyleAccum = { totalTurns: 0, agentSpawns: 0, planModeUses: 0, totalToolCalls: 0, fastModeCalls: 0 }
|
||||
const sB: StyleAccum = { totalTurns: 0, agentSpawns: 0, planModeUses: 0, totalToolCalls: 0, fastModeCalls: 0 }
|
||||
|
||||
for (const project of projects) {
|
||||
for (const session of project.sessions) {
|
||||
for (const turn of session.turns) {
|
||||
if (turn.assistantCalls.length === 0) continue
|
||||
const primary = turn.assistantCalls[0]!.model
|
||||
if (primary !== modelA && primary !== modelB) continue
|
||||
|
||||
const s = primary === modelA ? sA : sB
|
||||
s.totalTurns++
|
||||
const turnTools = turn.assistantCalls.flatMap(c => c.tools)
|
||||
if (turnTools.some(t => PLANNING_TOOLS.has(t)) || turn.assistantCalls.some(c => c.hasPlanMode)) {
|
||||
s.planModeUses++
|
||||
}
|
||||
for (const call of turn.assistantCalls) {
|
||||
s.totalToolCalls += call.tools.length
|
||||
if (call.hasAgentSpawn) s.agentSpawns++
|
||||
if (call.speed === 'fast') s.fastModeCalls++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const pct = (num: number, den: number) => den > 0 ? (num / den) * 100 : null
|
||||
const avg = (num: number, den: number) => den > 0 ? num / den : null
|
||||
|
||||
return [
|
||||
{ label: 'Delegation rate', valueA: pct(sA.agentSpawns, sA.totalTurns), valueB: pct(sB.agentSpawns, sB.totalTurns), formatFn: 'percent' as const },
|
||||
{ label: 'Planning rate', valueA: pct(sA.planModeUses, sA.totalTurns), valueB: pct(sB.planModeUses, sB.totalTurns), formatFn: 'percent' as const },
|
||||
{ label: 'Avg tools / turn', valueA: avg(sA.totalToolCalls, sA.totalTurns), valueB: avg(sB.totalToolCalls, sB.totalTurns), formatFn: 'decimal' as const },
|
||||
{ label: 'Fast mode usage', valueA: pct(sA.fastModeCalls, sA.totalTurns), valueB: pct(sB.fastModeCalls, sB.totalTurns), formatFn: 'percent' as const },
|
||||
]
|
||||
}
|
||||
|
||||
const SELF_CORRECTION_PATTERNS = [
|
||||
/\bmy mistake\b/i,
|
||||
/\bmy bad\b/i,
|
||||
|
|
|
|||
204
src/compare.tsx
204
src/compare.tsx
|
|
@ -1,8 +1,8 @@
|
|||
import React, { useState, useEffect, useRef } from 'react'
|
||||
import { render, Box, Text, useInput, useApp } from 'ink'
|
||||
import { render, Box, Text, useInput, useApp, useStdout } from 'ink'
|
||||
|
||||
import type { ModelStats, ComparisonRow } from './compare-stats.js'
|
||||
import { aggregateModelStats, computeComparison, scanSelfCorrections } from './compare-stats.js'
|
||||
import type { ModelStats, ComparisonRow, CategoryComparison, WorkingStyleRow } from './compare-stats.js'
|
||||
import { aggregateModelStats, computeComparison, computeCategoryComparison, computeWorkingStyle, scanSelfCorrections } from './compare-stats.js'
|
||||
import { formatCost } from './format.js'
|
||||
import { parseAllSessions } from './parser.js'
|
||||
import { getAllProviders } from './providers/index.js'
|
||||
|
|
@ -10,13 +10,19 @@ import type { ProjectSummary, DateRange } from './types.js'
|
|||
|
||||
const ORANGE = '#FF8C42'
|
||||
const GREEN = '#5BF5A0'
|
||||
const DIM = '#555555'
|
||||
const DIM = '#888888'
|
||||
const GOLD = '#FFD700'
|
||||
const BAR_A = '#6495ED'
|
||||
const BAR_B = '#5BF5A0'
|
||||
const LOW_DATA_THRESHOLD = 20
|
||||
const LABEL_WIDTH = 20
|
||||
const VALUE_WIDTH = 14
|
||||
const MODEL_NAME_COL = 24
|
||||
const BAR_MAX_WIDTH = 30
|
||||
const MIN_WIDE = 90
|
||||
const PANEL_CHROME = 4
|
||||
const MS_PER_DAY = 24 * 60 * 60 * 1000
|
||||
const FULL_BLOCK = '\u2588'
|
||||
|
||||
function formatValue(value: number | null, fmt: ComparisonRow['formatFn']): string {
|
||||
if (value === null) return '-'
|
||||
|
|
@ -38,6 +44,10 @@ function daysOfData(first: string, last: string): number {
|
|||
return Math.max(1, Math.ceil(ms / MS_PER_DAY))
|
||||
}
|
||||
|
||||
function barWidth(rate: number): number {
|
||||
return Math.round((rate / 100) * BAR_MAX_WIDTH)
|
||||
}
|
||||
|
||||
type ModelSelectorProps = {
|
||||
models: ModelStats[]
|
||||
onSelect: (a: ModelStats, b: ModelStats) => void
|
||||
|
|
@ -86,7 +96,7 @@ function ModelSelector({ models, onSelect, onBack }: ModelSelectorProps) {
|
|||
<Box flexDirection="column" borderStyle="round" borderColor={ORANGE} paddingX={1}>
|
||||
<Text bold color={ORANGE}>Model Comparison</Text>
|
||||
<Text> </Text>
|
||||
<Text dimColor>Select two models to compare:</Text>
|
||||
<Text color={DIM}>Select two models to compare:</Text>
|
||||
<Text> </Text>
|
||||
{models.map((m, i) => {
|
||||
const isCursor = i === cursor
|
||||
|
|
@ -122,11 +132,64 @@ type ComparisonResultsProps = {
|
|||
modelA: ModelStats
|
||||
modelB: ModelStats
|
||||
rows: ComparisonRow[]
|
||||
categories: CategoryComparison[]
|
||||
workingStyle: WorkingStyleRow[]
|
||||
onBack: () => void
|
||||
}
|
||||
|
||||
function ComparisonResults({ modelA, modelB, rows, onBack }: ComparisonResultsProps) {
|
||||
function MetricPanel({ title, rows, nameA, nameB, pw }: { title: string; rows: ComparisonRow[]; nameA: string; nameB: string; pw: number }) {
|
||||
return (
|
||||
<Box flexDirection="column" borderStyle="round" borderColor={ORANGE} paddingX={1} width={pw}>
|
||||
<Text bold color={ORANGE}>{title}</Text>
|
||||
<Text>
|
||||
<Text>{''.padEnd(LABEL_WIDTH)}</Text>
|
||||
<Text bold>{nameA.padStart(VALUE_WIDTH)}</Text>
|
||||
<Text bold>{nameB.padStart(VALUE_WIDTH)}</Text>
|
||||
</Text>
|
||||
{rows.map(row => {
|
||||
const fmtA = formatValue(row.valueA, row.formatFn)
|
||||
const fmtB = formatValue(row.valueB, row.formatFn)
|
||||
return (
|
||||
<Text key={row.label}>
|
||||
<Text color={DIM}>{row.label.padEnd(LABEL_WIDTH)}</Text>
|
||||
<Text color={row.winner === 'a' ? GREEN : undefined}>{fmtA.padStart(VALUE_WIDTH)}</Text>
|
||||
<Text color={row.winner === 'b' ? GREEN : undefined}>{fmtB.padStart(VALUE_WIDTH)}</Text>
|
||||
</Text>
|
||||
)
|
||||
})}
|
||||
</Box>
|
||||
)
|
||||
}
|
||||
|
||||
function ContextPanel({ title, rows, nameA, nameB, pw, lowDataWarning }: { title: string; rows: { label: string; valueA: string; valueB: string }[]; nameA: string; nameB: string; pw: number; lowDataWarning?: string }) {
|
||||
return (
|
||||
<Box flexDirection="column" borderStyle="round" borderColor={ORANGE} paddingX={1} width={pw}>
|
||||
<Text bold color={ORANGE}>{title}</Text>
|
||||
<Text>
|
||||
<Text>{''.padEnd(LABEL_WIDTH)}</Text>
|
||||
<Text bold>{nameA.padStart(VALUE_WIDTH)}</Text>
|
||||
<Text bold>{nameB.padStart(VALUE_WIDTH)}</Text>
|
||||
</Text>
|
||||
{rows.map(row => (
|
||||
<Text key={row.label}>
|
||||
<Text color={DIM}>{row.label.padEnd(LABEL_WIDTH)}</Text>
|
||||
<Text color={DIM}>{row.valueA.padStart(VALUE_WIDTH)}</Text>
|
||||
<Text color={DIM}>{row.valueB.padStart(VALUE_WIDTH)}</Text>
|
||||
</Text>
|
||||
))}
|
||||
{lowDataWarning && <Text color={GOLD}>{lowDataWarning}</Text>}
|
||||
</Box>
|
||||
)
|
||||
}
|
||||
|
||||
function ComparisonResults({ modelA, modelB, rows, categories, workingStyle, onBack }: ComparisonResultsProps) {
|
||||
const { exit } = useApp()
|
||||
const { stdout } = useStdout()
|
||||
const termWidth = stdout?.columns || 80
|
||||
const dashWidth = Math.min(160, termWidth)
|
||||
const wide = dashWidth >= MIN_WIDE
|
||||
const halfWidth = wide ? Math.floor(dashWidth / 2) : dashWidth
|
||||
|
||||
const nameA = shortName(modelA.model)
|
||||
const nameB = shortName(modelB.model)
|
||||
const lowDataA = modelA.calls < LOW_DATA_THRESHOLD
|
||||
|
|
@ -137,59 +200,103 @@ function ComparisonResults({ modelA, modelB, rows, onBack }: ComparisonResultsPr
|
|||
if (key.escape) { onBack(); return }
|
||||
})
|
||||
|
||||
const sectionOrder: string[] = []
|
||||
const sectionRows = new Map<string, ComparisonRow[]>()
|
||||
for (const row of rows) {
|
||||
if (!sectionRows.has(row.section)) {
|
||||
sectionOrder.push(row.section)
|
||||
sectionRows.set(row.section, [])
|
||||
}
|
||||
sectionRows.get(row.section)!.push(row)
|
||||
}
|
||||
|
||||
const fmtTokens = (n: number) => {
|
||||
if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`
|
||||
if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K`
|
||||
return String(n)
|
||||
}
|
||||
|
||||
const contextRows: { label: string; valueA: string; valueB: string }[] = [
|
||||
{ label: 'Calls', valueA: modelA.calls.toLocaleString(), valueB: modelB.calls.toLocaleString() },
|
||||
{ label: 'Total cost', valueA: formatCost(modelA.cost), valueB: formatCost(modelB.cost) },
|
||||
{ label: 'Input tokens', valueA: fmtTokens(modelA.inputTokens), valueB: fmtTokens(modelB.inputTokens) },
|
||||
{ label: 'Output tokens', valueA: fmtTokens(modelA.outputTokens), valueB: fmtTokens(modelB.outputTokens) },
|
||||
{ label: 'Days of data', valueA: String(daysOfData(modelA.firstSeen, modelA.lastSeen)), valueB: String(daysOfData(modelB.firstSeen, modelB.lastSeen)) },
|
||||
{ label: 'Edit turns', valueA: modelA.editTurns.toLocaleString(), valueB: modelB.editTurns.toLocaleString() },
|
||||
{ label: 'Self-corrections', valueA: modelA.selfCorrections.toLocaleString(), valueB: modelB.selfCorrections.toLocaleString() },
|
||||
]
|
||||
|
||||
const lowDataWarning = (lowDataA || lowDataB)
|
||||
? `Note: ${[lowDataA && shortName(modelA.model), lowDataB && shortName(modelB.model)].filter(Boolean).join(' and ')} ha${lowDataA && lowDataB ? 've' : 's'} fewer than ${LOW_DATA_THRESHOLD} calls`
|
||||
: undefined
|
||||
|
||||
const pw = wide ? halfWidth : dashWidth
|
||||
|
||||
return (
|
||||
<Box flexDirection="column" paddingX={2} paddingY={1}>
|
||||
<Box flexDirection="column" borderStyle="round" borderColor={ORANGE} paddingX={1}>
|
||||
<Box flexDirection="column" borderStyle="round" borderColor={ORANGE} paddingX={1} width={dashWidth}>
|
||||
<Text>
|
||||
<Text bold color={ORANGE}>{nameA}</Text>
|
||||
<Text dimColor> vs </Text>
|
||||
<Text bold color={ORANGE}>{nameB}</Text>
|
||||
</Text>
|
||||
<Text> </Text>
|
||||
<Text>
|
||||
<Text>{''.padEnd(LABEL_WIDTH)}</Text>
|
||||
<Text bold>{nameA.padStart(VALUE_WIDTH)}</Text>
|
||||
<Text bold>{nameB.padStart(VALUE_WIDTH)}</Text>
|
||||
</Text>
|
||||
{rows.map(row => {
|
||||
const fmtA = formatValue(row.valueA, row.formatFn)
|
||||
const fmtB = formatValue(row.valueB, row.formatFn)
|
||||
|
||||
return (
|
||||
<Text key={row.label}>
|
||||
<Text dimColor>{row.label.padEnd(LABEL_WIDTH)}</Text>
|
||||
<Text color={row.winner === 'a' ? GREEN : undefined}>{fmtA.padStart(VALUE_WIDTH)}</Text>
|
||||
<Text color={row.winner === 'b' ? GREEN : undefined}>{fmtB.padStart(VALUE_WIDTH)}</Text>
|
||||
</Text>
|
||||
)
|
||||
})}
|
||||
<Text> </Text>
|
||||
<Text dimColor>{'-- Context '.padEnd(LABEL_WIDTH + VALUE_WIDTH * 2, '-')}</Text>
|
||||
{contextRows.map(row => (
|
||||
<Text key={row.label}>
|
||||
<Text color={DIM}>{row.label.padEnd(LABEL_WIDTH)}</Text>
|
||||
<Text color={DIM}>{row.valueA.padStart(VALUE_WIDTH)}</Text>
|
||||
<Text color={DIM}>{row.valueB.padStart(VALUE_WIDTH)}</Text>
|
||||
</Text>
|
||||
))}
|
||||
{(lowDataA || lowDataB) && (
|
||||
<>
|
||||
<Text> </Text>
|
||||
<Text color={GOLD}>
|
||||
Note: {[lowDataA && modelA.model, lowDataB && modelB.model].filter(Boolean).join(' and ')} ha{lowDataA && lowDataB ? 've' : 's'} fewer than {LOW_DATA_THRESHOLD} calls -- results may not be representative.
|
||||
</Text>
|
||||
</>
|
||||
)}
|
||||
</Box>
|
||||
<Text> </Text>
|
||||
|
||||
<Box width={dashWidth}>
|
||||
<MetricPanel title={sectionOrder[0] ?? 'Performance'} rows={sectionRows.get(sectionOrder[0] ?? '') ?? []} nameA={nameA} nameB={nameB} pw={pw} />
|
||||
<MetricPanel title={sectionOrder[1] ?? 'Efficiency'} rows={sectionRows.get(sectionOrder[1] ?? '') ?? []} nameA={nameA} nameB={nameB} pw={pw} />
|
||||
</Box>
|
||||
|
||||
{categories.length > 0 && (
|
||||
<Box flexDirection="column" borderStyle="round" borderColor={ORANGE} paddingX={1} width={dashWidth}>
|
||||
<Text bold color={ORANGE}>Category Head-to-Head</Text>
|
||||
<Text color={DIM}>one-shot rate per category</Text>
|
||||
<Text>
|
||||
<Text>{' '}</Text>
|
||||
<Text color={BAR_A}>{FULL_BLOCK + FULL_BLOCK}</Text>
|
||||
<Text> {nameA} </Text>
|
||||
<Text color={BAR_B}>{FULL_BLOCK + FULL_BLOCK}</Text>
|
||||
<Text> {nameB}</Text>
|
||||
</Text>
|
||||
{categories.map(cat => {
|
||||
const bwA = cat.oneShotRateA !== null ? barWidth(cat.oneShotRateA) : 0
|
||||
const bwB = cat.oneShotRateB !== null ? barWidth(cat.oneShotRateB) : 0
|
||||
const rateA = cat.oneShotRateA !== null ? `${cat.oneShotRateA.toFixed(1)}%` : '-'
|
||||
const rateB = cat.oneShotRateB !== null ? `${cat.oneShotRateB.toFixed(1)}%` : '-'
|
||||
const turnsA = cat.editTurnsA > 0 ? `(${cat.editTurnsA})` : ''
|
||||
const turnsB = cat.editTurnsB > 0 ? `(${cat.editTurnsB})` : ''
|
||||
|
||||
return (
|
||||
<React.Fragment key={cat.category}>
|
||||
<Text> </Text>
|
||||
<Text color={DIM}>{' '}{cat.category}</Text>
|
||||
<Text>
|
||||
<Text>{' '}</Text>
|
||||
<Text color={cat.winner === 'a' ? BAR_A : DIM}>{FULL_BLOCK.repeat(Math.max(bwA, 1))}</Text>
|
||||
<Text>{' '.repeat(Math.max(0, BAR_MAX_WIDTH - bwA))} </Text>
|
||||
<Text color={cat.winner === 'a' ? GREEN : undefined}>{rateA.padStart(6)}</Text>
|
||||
<Text color={DIM}> {turnsA}</Text>
|
||||
</Text>
|
||||
<Text>
|
||||
<Text>{' '}</Text>
|
||||
<Text color={cat.winner === 'b' ? BAR_B : DIM}>{FULL_BLOCK.repeat(Math.max(bwB, 1))}</Text>
|
||||
<Text>{' '.repeat(Math.max(0, BAR_MAX_WIDTH - bwB))} </Text>
|
||||
<Text color={cat.winner === 'b' ? GREEN : undefined}>{rateB.padStart(6)}</Text>
|
||||
<Text color={DIM}> {turnsB}</Text>
|
||||
</Text>
|
||||
</React.Fragment>
|
||||
)
|
||||
})}
|
||||
</Box>
|
||||
)}
|
||||
|
||||
<Box width={dashWidth}>
|
||||
{workingStyle.length > 0 && (
|
||||
<ContextPanel title="Working Style" rows={workingStyle.map(r => ({ label: r.label, valueA: formatValue(r.valueA, r.formatFn), valueB: formatValue(r.valueB, r.formatFn) }))} nameA={nameA} nameB={nameB} pw={pw} />
|
||||
)}
|
||||
<ContextPanel title="Context" rows={contextRows} nameA={nameA} nameB={nameB} pw={pw} lowDataWarning={lowDataWarning} />
|
||||
</Box>
|
||||
|
||||
<Text>
|
||||
<Text color={ORANGE} bold>[esc]</Text><Text dimColor> back </Text>
|
||||
<Text color={ORANGE} bold>[q]</Text><Text dimColor> quit</Text>
|
||||
|
|
@ -211,6 +318,8 @@ export function CompareView({ projects, onBack }: CompareViewProps) {
|
|||
const [selectedA, setSelectedA] = useState<ModelStats | null>(null)
|
||||
const [selectedB, setSelectedB] = useState<ModelStats | null>(null)
|
||||
const [rows, setRows] = useState<ComparisonRow[]>([])
|
||||
const [categories, setCategories] = useState<CategoryComparison[]>([])
|
||||
const [style, setStyle] = useState<WorkingStyleRow[]>([])
|
||||
const [loadTrigger, setLoadTrigger] = useState(0)
|
||||
const projectsRef = useRef(projects)
|
||||
projectsRef.current = projects
|
||||
|
|
@ -251,11 +360,14 @@ export function CompareView({ projects, onBack }: CompareViewProps) {
|
|||
const corrections = await scanSelfCorrections(dirs)
|
||||
if (cancelled) return
|
||||
|
||||
const currentProjects = projectsRef.current
|
||||
const aCopy = { ...a!, selfCorrections: corrections.get(a!.model) ?? 0 }
|
||||
const bCopy = { ...b!, selfCorrections: corrections.get(b!.model) ?? 0 }
|
||||
setSelectedA(aCopy)
|
||||
setSelectedB(bCopy)
|
||||
setRows(computeComparison(aCopy, bCopy))
|
||||
setCategories(computeCategoryComparison(currentProjects, a!.model, b!.model))
|
||||
setStyle(computeWorkingStyle(currentProjects, a!.model, b!.model))
|
||||
setPhase('results')
|
||||
}
|
||||
|
||||
|
|
@ -277,7 +389,7 @@ export function CompareView({ projects, onBack }: CompareViewProps) {
|
|||
<Box flexDirection="column" borderStyle="round" borderColor={ORANGE} paddingX={1}>
|
||||
<Text bold color={ORANGE}>Model Comparison</Text>
|
||||
<Text> </Text>
|
||||
<Text dimColor>Need at least 2 models to compare. Found {models.length}.</Text>
|
||||
<Text color={DIM}>Need at least 2 models to compare. Found {models.length}.</Text>
|
||||
</Box>
|
||||
<Text> </Text>
|
||||
<Text>
|
||||
|
|
@ -299,7 +411,7 @@ export function CompareView({ projects, onBack }: CompareViewProps) {
|
|||
<Box flexDirection="column" borderStyle="round" borderColor={ORANGE} paddingX={1}>
|
||||
<Text bold color={ORANGE}>Model Comparison</Text>
|
||||
<Text> </Text>
|
||||
<Text dimColor>Scanning self-corrections...</Text>
|
||||
<Text color={DIM}>Scanning self-corrections...</Text>
|
||||
</Box>
|
||||
</Box>
|
||||
)
|
||||
|
|
@ -311,6 +423,8 @@ export function CompareView({ projects, onBack }: CompareViewProps) {
|
|||
modelA={selectedA}
|
||||
modelB={selectedB}
|
||||
rows={rows}
|
||||
categories={categories}
|
||||
workingStyle={style}
|
||||
onBack={() => setPhase('select')}
|
||||
/>
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,13 +2,14 @@ import { mkdtemp, mkdir, rm, writeFile } from 'fs/promises'
|
|||
import { join } from 'path'
|
||||
import { tmpdir } from 'os'
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { aggregateModelStats, computeComparison, scanSelfCorrections, type ModelStats } from '../src/compare-stats.js'
|
||||
import { aggregateModelStats, computeComparison, computeCategoryComparison, computeWorkingStyle, scanSelfCorrections, type ModelStats } from '../src/compare-stats.js'
|
||||
import type { ProjectSummary, SessionSummary, ClassifiedTurn } from '../src/types.js'
|
||||
|
||||
function makeTurn(model: string, cost: number, opts: { hasEdits?: boolean; retries?: number; outputTokens?: number; inputTokens?: number; cacheRead?: number; cacheWrite?: number; timestamp?: string } = {}): ClassifiedTurn {
|
||||
function makeTurn(model: string, cost: number, opts: { hasEdits?: boolean; retries?: number; outputTokens?: number; inputTokens?: number; cacheRead?: number; cacheWrite?: number; timestamp?: string; category?: string; hasAgentSpawn?: boolean; hasPlanMode?: boolean; speed?: 'standard' | 'fast'; tools?: string[] } = {}): ClassifiedTurn {
|
||||
const defaultTools = opts.tools ?? (opts.hasEdits ? ['Edit'] : ['Read'])
|
||||
return {
|
||||
timestamp: opts.timestamp ?? '2026-04-15T10:00:00Z',
|
||||
category: 'coding',
|
||||
category: (opts.category ?? 'coding') as ClassifiedTurn['category'],
|
||||
retries: opts.retries ?? 0,
|
||||
hasEdits: opts.hasEdits ?? false,
|
||||
userMessage: '',
|
||||
|
|
@ -25,11 +26,11 @@ function makeTurn(model: string, cost: number, opts: { hasEdits?: boolean; retri
|
|||
webSearchRequests: 0,
|
||||
},
|
||||
costUSD: cost,
|
||||
tools: opts.hasEdits ? ['Edit'] : ['Read'],
|
||||
tools: defaultTools,
|
||||
mcpTools: [],
|
||||
hasAgentSpawn: false,
|
||||
hasPlanMode: false,
|
||||
speed: 'standard' as const,
|
||||
hasAgentSpawn: opts.hasAgentSpawn ?? false,
|
||||
hasPlanMode: opts.hasPlanMode ?? false,
|
||||
speed: opts.speed ?? 'standard' as const,
|
||||
timestamp: opts.timestamp ?? '2026-04-15T10:00:00Z',
|
||||
bashCommands: [],
|
||||
deduplicationKey: `key-${Math.random()}`,
|
||||
|
|
@ -128,6 +129,17 @@ describe('aggregateModelStats', () => {
|
|||
expect(aggregateModelStats([])).toEqual([])
|
||||
})
|
||||
|
||||
it('tracks editCost for edit turns', () => {
|
||||
const project = makeProject([
|
||||
makeTurn('opus-4-6', 0.10, { hasEdits: true }),
|
||||
makeTurn('opus-4-6', 0.20, { hasEdits: true }),
|
||||
makeTurn('opus-4-6', 0.50, { hasEdits: false }),
|
||||
])
|
||||
const stats = aggregateModelStats([project])
|
||||
const m = stats.find(s => s.model === 'opus-4-6')!
|
||||
expect(m.editCost).toBeCloseTo(0.30)
|
||||
})
|
||||
|
||||
it('sorts by cost descending', () => {
|
||||
const project = makeProject([
|
||||
makeTurn('cheap-model', 0.01),
|
||||
|
|
@ -153,6 +165,7 @@ function makeStats(overrides: Partial<ModelStats> = {}): ModelStats {
|
|||
oneShotTurns: 60,
|
||||
retries: 20,
|
||||
selfCorrections: 10,
|
||||
editCost: 8,
|
||||
firstSeen: '2026-04-01T00:00:00Z',
|
||||
lastSeen: '2026-04-15T00:00:00Z',
|
||||
...overrides,
|
||||
|
|
@ -199,6 +212,16 @@ describe('computeComparison', () => {
|
|||
expect(costRow.winner).toBe('tie')
|
||||
})
|
||||
|
||||
it('computes cost per edit correctly', () => {
|
||||
const a = makeStats({ editTurns: 40, editCost: 4 })
|
||||
const b = makeStats({ editTurns: 80, editCost: 4 })
|
||||
const rows = computeComparison(a, b)
|
||||
const editRow = rows.find(r => r.label === 'Cost / edit')!
|
||||
expect(editRow.valueA).toBeCloseTo(0.10)
|
||||
expect(editRow.valueB).toBeCloseTo(0.05)
|
||||
expect(editRow.winner).toBe('b')
|
||||
})
|
||||
|
||||
it('picks higher value as winner for cache hit rate', () => {
|
||||
const a = makeStats({ inputTokens: 5000, cacheReadTokens: 30000, cacheWriteTokens: 5000 })
|
||||
const b = makeStats({ inputTokens: 10000, cacheReadTokens: 10000, cacheWriteTokens: 5000 })
|
||||
|
|
@ -354,3 +377,192 @@ describe('scanSelfCorrections', () => {
|
|||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('computeCategoryComparison', () => {
|
||||
it('returns per-category one-shot rates for both models', () => {
|
||||
const project = makeProject([
|
||||
makeTurn('model-a', 0.10, { hasEdits: true, retries: 0, category: 'coding' }),
|
||||
makeTurn('model-a', 0.10, { hasEdits: true, retries: 1, category: 'coding' }),
|
||||
makeTurn('model-b', 0.10, { hasEdits: true, retries: 0, category: 'coding' }),
|
||||
makeTurn('model-b', 0.10, { hasEdits: true, retries: 0, category: 'coding' }),
|
||||
makeTurn('model-a', 0.10, { hasEdits: true, retries: 0, category: 'debugging' }),
|
||||
makeTurn('model-b', 0.10, { hasEdits: true, retries: 1, category: 'debugging' }),
|
||||
])
|
||||
const result = computeCategoryComparison([project], 'model-a', 'model-b')
|
||||
|
||||
const coding = result.find(r => r.category === 'coding')!
|
||||
expect(coding.editTurnsA).toBe(2)
|
||||
expect(coding.oneShotRateA).toBeCloseTo(50)
|
||||
expect(coding.editTurnsB).toBe(2)
|
||||
expect(coding.oneShotRateB).toBeCloseTo(100)
|
||||
expect(coding.winner).toBe('b')
|
||||
|
||||
const debugging = result.find(r => r.category === 'debugging')!
|
||||
expect(debugging.oneShotRateA).toBeCloseTo(100)
|
||||
expect(debugging.oneShotRateB).toBeCloseTo(0)
|
||||
expect(debugging.winner).toBe('a')
|
||||
})
|
||||
|
||||
it('skips categories with no edit turns', () => {
|
||||
const project = makeProject([
|
||||
makeTurn('model-a', 0.10, { hasEdits: false, category: 'conversation' }),
|
||||
makeTurn('model-b', 0.10, { hasEdits: false, category: 'conversation' }),
|
||||
makeTurn('model-a', 0.10, { hasEdits: true, category: 'coding' }),
|
||||
])
|
||||
const result = computeCategoryComparison([project], 'model-a', 'model-b')
|
||||
expect(result.find(r => r.category === 'conversation')).toBeUndefined()
|
||||
expect(result).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('sorts by total turns descending', () => {
|
||||
const project = makeProject([
|
||||
makeTurn('model-a', 0.10, { hasEdits: true, category: 'coding' }),
|
||||
makeTurn('model-a', 0.10, { hasEdits: true, category: 'coding' }),
|
||||
makeTurn('model-a', 0.10, { hasEdits: true, category: 'coding' }),
|
||||
makeTurn('model-b', 0.10, { hasEdits: true, category: 'coding' }),
|
||||
makeTurn('model-a', 0.10, { hasEdits: true, category: 'debugging' }),
|
||||
])
|
||||
const result = computeCategoryComparison([project], 'model-a', 'model-b')
|
||||
expect(result[0].category).toBe('coding')
|
||||
})
|
||||
|
||||
it('returns null one-shot rate when model has no edits in category', () => {
|
||||
const project = makeProject([
|
||||
makeTurn('model-a', 0.10, { hasEdits: true, category: 'coding' }),
|
||||
makeTurn('model-b', 0.10, { hasEdits: false, category: 'coding' }),
|
||||
])
|
||||
const result = computeCategoryComparison([project], 'model-a', 'model-b')
|
||||
const coding = result.find(r => r.category === 'coding')!
|
||||
expect(coding.oneShotRateA).toBeCloseTo(100)
|
||||
expect(coding.oneShotRateB).toBeNull()
|
||||
expect(coding.winner).toBe('none')
|
||||
})
|
||||
})
|
||||
|
||||
describe('computeWorkingStyle', () => {
|
||||
it('computes delegation and planning rates', () => {
|
||||
const project = makeProject([
|
||||
makeTurn('model-a', 0.10, { hasAgentSpawn: true }),
|
||||
makeTurn('model-a', 0.10, {}),
|
||||
makeTurn('model-a', 0.10, { hasPlanMode: true }),
|
||||
makeTurn('model-b', 0.10, {}),
|
||||
makeTurn('model-b', 0.10, {}),
|
||||
])
|
||||
const result = computeWorkingStyle([project], 'model-a', 'model-b')
|
||||
|
||||
const delegation = result.find(r => r.label === 'Delegation rate')!
|
||||
expect(delegation.valueA).toBeCloseTo(100 / 3)
|
||||
expect(delegation.valueB).toBeCloseTo(0)
|
||||
|
||||
const planning = result.find(r => r.label === 'Planning rate')!
|
||||
expect(planning.valueA).toBeCloseTo(100 / 3)
|
||||
expect(planning.valueB).toBeCloseTo(0)
|
||||
})
|
||||
|
||||
it('computes avg tools per turn', () => {
|
||||
const project = makeProject([
|
||||
makeTurn('model-a', 0.10, { hasEdits: true }),
|
||||
makeTurn('model-a', 0.10, {}),
|
||||
makeTurn('model-b', 0.10, { hasEdits: true }),
|
||||
])
|
||||
const result = computeWorkingStyle([project], 'model-a', 'model-b')
|
||||
const tools = result.find(r => r.label === 'Avg tools / turn')!
|
||||
expect(tools.valueA).toBeCloseTo(1)
|
||||
expect(tools.valueB).toBeCloseTo(1)
|
||||
})
|
||||
|
||||
it('computes fast mode usage', () => {
|
||||
const project = makeProject([
|
||||
makeTurn('model-a', 0.10, { speed: 'fast' }),
|
||||
makeTurn('model-a', 0.10, {}),
|
||||
makeTurn('model-b', 0.10, { speed: 'fast' }),
|
||||
makeTurn('model-b', 0.10, { speed: 'fast' }),
|
||||
])
|
||||
const result = computeWorkingStyle([project], 'model-a', 'model-b')
|
||||
const fast = result.find(r => r.label === 'Fast mode usage')!
|
||||
expect(fast.valueA).toBeCloseTo(50)
|
||||
expect(fast.valueB).toBeCloseTo(100)
|
||||
})
|
||||
|
||||
it('returns null for models with no turns', () => {
|
||||
const project = makeProject([
|
||||
makeTurn('model-a', 0.10, {}),
|
||||
])
|
||||
const result = computeWorkingStyle([project], 'model-a', 'model-b')
|
||||
const delegation = result.find(r => r.label === 'Delegation rate')!
|
||||
expect(delegation.valueA).toBeCloseTo(0)
|
||||
expect(delegation.valueB).toBeNull()
|
||||
})
|
||||
|
||||
it('counts TaskCreate as planning', () => {
|
||||
const project = makeProject([
|
||||
makeTurn('model-a', 0.10, { tools: ['TaskCreate'] }),
|
||||
makeTurn('model-a', 0.10, { tools: ['Read'] }),
|
||||
makeTurn('model-a', 0.10, { tools: ['Edit'] }),
|
||||
])
|
||||
const result = computeWorkingStyle([project], 'model-a', 'model-b')
|
||||
const planning = result.find(r => r.label === 'Planning rate')!
|
||||
expect(planning.valueA).toBeCloseTo(100 / 3)
|
||||
})
|
||||
|
||||
it('counts TaskUpdate as planning', () => {
|
||||
const project = makeProject([
|
||||
makeTurn('model-a', 0.10, { tools: ['TaskUpdate'] }),
|
||||
makeTurn('model-a', 0.10, { tools: ['Read'] }),
|
||||
])
|
||||
const result = computeWorkingStyle([project], 'model-a', 'model-b')
|
||||
const planning = result.find(r => r.label === 'Planning rate')!
|
||||
expect(planning.valueA).toBeCloseTo(50)
|
||||
})
|
||||
|
||||
it('counts TodoWrite as planning', () => {
|
||||
const project = makeProject([
|
||||
makeTurn('model-a', 0.10, { tools: ['TodoWrite', 'Read'] }),
|
||||
makeTurn('model-a', 0.10, { tools: ['Bash'] }),
|
||||
])
|
||||
const result = computeWorkingStyle([project], 'model-a', 'model-b')
|
||||
const planning = result.find(r => r.label === 'Planning rate')!
|
||||
expect(planning.valueA).toBeCloseTo(50)
|
||||
})
|
||||
|
||||
it('counts turn with planning tool + edits as planning', () => {
|
||||
const project = makeProject([
|
||||
makeTurn('model-a', 0.10, { tools: ['TaskCreate', 'Edit', 'Read'] }),
|
||||
makeTurn('model-a', 0.10, { tools: ['Edit'] }),
|
||||
])
|
||||
const result = computeWorkingStyle([project], 'model-a', 'model-b')
|
||||
const planning = result.find(r => r.label === 'Planning rate')!
|
||||
expect(planning.valueA).toBeCloseTo(50)
|
||||
})
|
||||
|
||||
it('does not count regular tools as planning', () => {
|
||||
const project = makeProject([
|
||||
makeTurn('model-a', 0.10, { tools: ['Read', 'Grep', 'Glob'] }),
|
||||
makeTurn('model-a', 0.10, { tools: ['Edit', 'Bash'] }),
|
||||
makeTurn('model-a', 0.10, { tools: ['Agent'] }),
|
||||
])
|
||||
const result = computeWorkingStyle([project], 'model-a', 'model-b')
|
||||
const planning = result.find(r => r.label === 'Planning rate')!
|
||||
expect(planning.valueA).toBeCloseTo(0)
|
||||
})
|
||||
|
||||
it('counts planning once per turn even with multiple planning tools', () => {
|
||||
const project = makeProject([
|
||||
makeTurn('model-a', 0.10, { tools: ['TaskCreate', 'TaskUpdate', 'TaskCreate'] }),
|
||||
makeTurn('model-a', 0.10, { tools: ['Read'] }),
|
||||
])
|
||||
const result = computeWorkingStyle([project], 'model-a', 'model-b')
|
||||
const planning = result.find(r => r.label === 'Planning rate')!
|
||||
expect(planning.valueA).toBeCloseTo(50)
|
||||
})
|
||||
|
||||
it('hasPlanMode still triggers planning rate', () => {
|
||||
const project = makeProject([
|
||||
makeTurn('model-a', 0.10, { hasPlanMode: true, tools: ['Read'] }),
|
||||
makeTurn('model-a', 0.10, { tools: ['Read'] }),
|
||||
])
|
||||
const result = computeWorkingStyle([project], 'model-a', 'model-b')
|
||||
const planning = result.find(r => r.label === 'Planning rate')!
|
||||
expect(planning.valueA).toBeCloseTo(50)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue