diff --git a/.gitignore b/.gitignore index 23a5028..695c7c3 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ npm-debug.log* # Local Discord brand / promo assets not yet ready to publish assets/discord-*.png + +# Desktop app experiments +desktop/ diff --git a/src/compare-stats.ts b/src/compare-stats.ts index 3406510..5e67781 100644 --- a/src/compare-stats.ts +++ b/src/compare-stats.ts @@ -3,6 +3,8 @@ import { join } from 'path' import type { ProjectSummary } from './types.js' +const PLANNING_TOOLS = new Set(['TaskCreate', 'TaskUpdate', 'TodoWrite', 'EnterPlanMode', 'ExitPlanMode']) + export type ModelStats = { model: string calls: number @@ -16,6 +18,7 @@ export type ModelStats = { oneShotTurns: number retries: number selfCorrections: number + editCost: number firstSeen: string lastSeen: string } @@ -26,7 +29,7 @@ export function aggregateModelStats(projects: ProjectSummary[]): ModelStats[] { const ensure = (model: string): ModelStats => { let s = byModel.get(model) if (!s) { - s = { model, calls: 0, cost: 0, outputTokens: 0, inputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, totalTurns: 0, editTurns: 0, oneShotTurns: 0, retries: 0, selfCorrections: 0, firstSeen: '', lastSeen: '' } + s = { model, calls: 0, cost: 0, outputTokens: 0, inputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, totalTurns: 0, editTurns: 0, oneShotTurns: 0, retries: 0, selfCorrections: 0, editCost: 0, firstSeen: '', lastSeen: '' } byModel.set(model, s) } return s @@ -41,8 +44,13 @@ export function aggregateModelStats(projects: ProjectSummary[]): ModelStats[] { const ms = ensure(primaryModel) ms.totalTurns++ - if (turn.hasEdits) ms.editTurns++ - if (turn.hasEdits && turn.retries === 0) ms.oneShotTurns++ + if (turn.hasEdits) { + ms.editTurns++ + if (turn.retries === 0) ms.oneShotTurns++ + for (const c of turn.assistantCalls) { + if (c.model !== '') ms.editCost += c.costUSD + } + } ms.retries += turn.retries for (const call of turn.assistantCalls) { @@ -66,6 +74,7 @@ export function aggregateModelStats(projects: ProjectSummary[]): ModelStats[] { } export type ComparisonRow = { + section: string label: string valueA: number | null valueB: number | null @@ -73,7 +82,26 @@ export type ComparisonRow = { winner: 'a' | 'b' | 'tie' | 'none' } +export type CategoryComparison = { + category: string + turnsA: number + editTurnsA: number + oneShotRateA: number | null + turnsB: number + editTurnsB: number + oneShotRateB: number | null + winner: 'a' | 'b' | 'tie' | 'none' +} + +export type WorkingStyleRow = { + label: string + valueA: number | null + valueB: number | null + formatFn: ComparisonRow['formatFn'] +} + type MetricDef = { + section: string label: string formatFn: ComparisonRow['formatFn'] higherIsBetter: boolean @@ -82,18 +110,49 @@ type MetricDef = { const METRICS: MetricDef[] = [ { + section: 'Performance', + label: 'One-shot rate', + formatFn: 'percent', + higherIsBetter: true, + compute: s => s.editTurns > 0 ? (s.oneShotTurns / s.editTurns) * 100 : null, + }, + { + section: 'Performance', + label: 'Retry rate', + formatFn: 'decimal', + higherIsBetter: false, + compute: s => s.editTurns > 0 ? s.retries / s.editTurns : null, + }, + { + section: 'Performance', + label: 'Self-correction', + formatFn: 'percent', + higherIsBetter: false, + compute: s => s.totalTurns > 0 ? (s.selfCorrections / s.totalTurns) * 100 : null, + }, + { + section: 'Efficiency', label: 'Cost / call', formatFn: 'cost', higherIsBetter: false, compute: s => s.calls > 0 ? s.cost / s.calls : null, }, { + section: 'Efficiency', + label: 'Cost / edit', + formatFn: 'cost', + higherIsBetter: false, + compute: s => s.editTurns > 0 ? s.editCost / s.editTurns : null, + }, + { + section: 'Efficiency', label: 'Output tok / call', formatFn: 'number', higherIsBetter: false, compute: s => s.calls > 0 ? Math.round(s.outputTokens / s.calls) : null, }, { + section: 'Efficiency', label: 'Cache hit rate', formatFn: 'percent', higherIsBetter: true, @@ -102,24 +161,6 @@ const METRICS: MetricDef[] = [ return total > 0 ? (s.cacheReadTokens / total) * 100 : null }, }, - { - label: 'One-shot rate', - formatFn: 'percent', - higherIsBetter: true, - compute: s => s.editTurns > 0 ? (s.oneShotTurns / s.editTurns) * 100 : null, - }, - { - label: 'Retry rate', - formatFn: 'decimal', - higherIsBetter: false, - compute: s => s.editTurns > 0 ? s.retries / s.editTurns : null, - }, - { - label: 'Self-correction', - formatFn: 'percent', - higherIsBetter: false, - compute: s => s.totalTurns > 0 ? (s.selfCorrections / s.totalTurns) * 100 : null, - }, ] function pickWinner(valueA: number | null, valueB: number | null, higherIsBetter: boolean): ComparisonRow['winner'] { @@ -134,6 +175,7 @@ export function computeComparison(a: ModelStats, b: ModelStats): ComparisonRow[] const valueA = m.compute(a) const valueB = m.compute(b) return { + section: m.section, label: m.label, valueA, valueB, @@ -143,6 +185,98 @@ export function computeComparison(a: ModelStats, b: ModelStats): ComparisonRow[] }) } +export function computeCategoryComparison(projects: ProjectSummary[], modelA: string, modelB: string): CategoryComparison[] { + type Accum = { turns: number; editTurns: number; oneShotTurns: number } + const mapA = new Map() + const mapB = new Map() + + const ensure = (map: Map, cat: string): Accum => { + let a = map.get(cat) + if (!a) { a = { turns: 0, editTurns: 0, oneShotTurns: 0 }; map.set(cat, a) } + return a + } + + for (const project of projects) { + for (const session of project.sessions) { + for (const turn of session.turns) { + if (turn.assistantCalls.length === 0) continue + const primary = turn.assistantCalls[0]!.model + if (primary !== modelA && primary !== modelB) continue + + const acc = ensure(primary === modelA ? mapA : mapB, turn.category) + acc.turns++ + if (turn.hasEdits) { + acc.editTurns++ + if (turn.retries === 0) acc.oneShotTurns++ + } + } + } + } + + const allCats = new Set([...mapA.keys(), ...mapB.keys()]) + const result: CategoryComparison[] = [] + + for (const category of allCats) { + const a = mapA.get(category) + const b = mapB.get(category) + if ((!a || a.editTurns === 0) && (!b || b.editTurns === 0)) continue + + const rateA = a && a.editTurns > 0 ? (a.oneShotTurns / a.editTurns) * 100 : null + const rateB = b && b.editTurns > 0 ? (b.oneShotTurns / b.editTurns) * 100 : null + + result.push({ + category, + turnsA: a?.turns ?? 0, + editTurnsA: a?.editTurns ?? 0, + oneShotRateA: rateA, + turnsB: b?.turns ?? 0, + editTurnsB: b?.editTurns ?? 0, + oneShotRateB: rateB, + winner: pickWinner(rateA, rateB, true), + }) + } + + return result.sort((a, b) => (b.turnsA + b.turnsB) - (a.turnsA + a.turnsB)) +} + +export function computeWorkingStyle(projects: ProjectSummary[], modelA: string, modelB: string): WorkingStyleRow[] { + type StyleAccum = { totalTurns: number; agentSpawns: number; planModeUses: number; totalToolCalls: number; fastModeCalls: number } + const sA: StyleAccum = { totalTurns: 0, agentSpawns: 0, planModeUses: 0, totalToolCalls: 0, fastModeCalls: 0 } + const sB: StyleAccum = { totalTurns: 0, agentSpawns: 0, planModeUses: 0, totalToolCalls: 0, fastModeCalls: 0 } + + for (const project of projects) { + for (const session of project.sessions) { + for (const turn of session.turns) { + if (turn.assistantCalls.length === 0) continue + const primary = turn.assistantCalls[0]!.model + if (primary !== modelA && primary !== modelB) continue + + const s = primary === modelA ? sA : sB + s.totalTurns++ + const turnTools = turn.assistantCalls.flatMap(c => c.tools) + if (turnTools.some(t => PLANNING_TOOLS.has(t)) || turn.assistantCalls.some(c => c.hasPlanMode)) { + s.planModeUses++ + } + for (const call of turn.assistantCalls) { + s.totalToolCalls += call.tools.length + if (call.hasAgentSpawn) s.agentSpawns++ + if (call.speed === 'fast') s.fastModeCalls++ + } + } + } + } + + const pct = (num: number, den: number) => den > 0 ? (num / den) * 100 : null + const avg = (num: number, den: number) => den > 0 ? num / den : null + + return [ + { label: 'Delegation rate', valueA: pct(sA.agentSpawns, sA.totalTurns), valueB: pct(sB.agentSpawns, sB.totalTurns), formatFn: 'percent' as const }, + { label: 'Planning rate', valueA: pct(sA.planModeUses, sA.totalTurns), valueB: pct(sB.planModeUses, sB.totalTurns), formatFn: 'percent' as const }, + { label: 'Avg tools / turn', valueA: avg(sA.totalToolCalls, sA.totalTurns), valueB: avg(sB.totalToolCalls, sB.totalTurns), formatFn: 'decimal' as const }, + { label: 'Fast mode usage', valueA: pct(sA.fastModeCalls, sA.totalTurns), valueB: pct(sB.fastModeCalls, sB.totalTurns), formatFn: 'percent' as const }, + ] +} + const SELF_CORRECTION_PATTERNS = [ /\bmy mistake\b/i, /\bmy bad\b/i, diff --git a/src/compare.tsx b/src/compare.tsx index 95b064b..d295468 100644 --- a/src/compare.tsx +++ b/src/compare.tsx @@ -1,8 +1,8 @@ import React, { useState, useEffect, useRef } from 'react' -import { render, Box, Text, useInput, useApp } from 'ink' +import { render, Box, Text, useInput, useApp, useStdout } from 'ink' -import type { ModelStats, ComparisonRow } from './compare-stats.js' -import { aggregateModelStats, computeComparison, scanSelfCorrections } from './compare-stats.js' +import type { ModelStats, ComparisonRow, CategoryComparison, WorkingStyleRow } from './compare-stats.js' +import { aggregateModelStats, computeComparison, computeCategoryComparison, computeWorkingStyle, scanSelfCorrections } from './compare-stats.js' import { formatCost } from './format.js' import { parseAllSessions } from './parser.js' import { getAllProviders } from './providers/index.js' @@ -10,13 +10,19 @@ import type { ProjectSummary, DateRange } from './types.js' const ORANGE = '#FF8C42' const GREEN = '#5BF5A0' -const DIM = '#555555' +const DIM = '#888888' const GOLD = '#FFD700' +const BAR_A = '#6495ED' +const BAR_B = '#5BF5A0' const LOW_DATA_THRESHOLD = 20 const LABEL_WIDTH = 20 const VALUE_WIDTH = 14 const MODEL_NAME_COL = 24 +const BAR_MAX_WIDTH = 30 +const MIN_WIDE = 90 +const PANEL_CHROME = 4 const MS_PER_DAY = 24 * 60 * 60 * 1000 +const FULL_BLOCK = '\u2588' function formatValue(value: number | null, fmt: ComparisonRow['formatFn']): string { if (value === null) return '-' @@ -38,6 +44,10 @@ function daysOfData(first: string, last: string): number { return Math.max(1, Math.ceil(ms / MS_PER_DAY)) } +function barWidth(rate: number): number { + return Math.round((rate / 100) * BAR_MAX_WIDTH) +} + type ModelSelectorProps = { models: ModelStats[] onSelect: (a: ModelStats, b: ModelStats) => void @@ -86,7 +96,7 @@ function ModelSelector({ models, onSelect, onBack }: ModelSelectorProps) { Model Comparison - Select two models to compare: + Select two models to compare: {models.map((m, i) => { const isCursor = i === cursor @@ -122,11 +132,64 @@ type ComparisonResultsProps = { modelA: ModelStats modelB: ModelStats rows: ComparisonRow[] + categories: CategoryComparison[] + workingStyle: WorkingStyleRow[] onBack: () => void } -function ComparisonResults({ modelA, modelB, rows, onBack }: ComparisonResultsProps) { +function MetricPanel({ title, rows, nameA, nameB, pw }: { title: string; rows: ComparisonRow[]; nameA: string; nameB: string; pw: number }) { + return ( + + {title} + + {''.padEnd(LABEL_WIDTH)} + {nameA.padStart(VALUE_WIDTH)} + {nameB.padStart(VALUE_WIDTH)} + + {rows.map(row => { + const fmtA = formatValue(row.valueA, row.formatFn) + const fmtB = formatValue(row.valueB, row.formatFn) + return ( + + {row.label.padEnd(LABEL_WIDTH)} + {fmtA.padStart(VALUE_WIDTH)} + {fmtB.padStart(VALUE_WIDTH)} + + ) + })} + + ) +} + +function ContextPanel({ title, rows, nameA, nameB, pw, lowDataWarning }: { title: string; rows: { label: string; valueA: string; valueB: string }[]; nameA: string; nameB: string; pw: number; lowDataWarning?: string }) { + return ( + + {title} + + {''.padEnd(LABEL_WIDTH)} + {nameA.padStart(VALUE_WIDTH)} + {nameB.padStart(VALUE_WIDTH)} + + {rows.map(row => ( + + {row.label.padEnd(LABEL_WIDTH)} + {row.valueA.padStart(VALUE_WIDTH)} + {row.valueB.padStart(VALUE_WIDTH)} + + ))} + {lowDataWarning && {lowDataWarning}} + + ) +} + +function ComparisonResults({ modelA, modelB, rows, categories, workingStyle, onBack }: ComparisonResultsProps) { const { exit } = useApp() + const { stdout } = useStdout() + const termWidth = stdout?.columns || 80 + const dashWidth = Math.min(160, termWidth) + const wide = dashWidth >= MIN_WIDE + const halfWidth = wide ? Math.floor(dashWidth / 2) : dashWidth + const nameA = shortName(modelA.model) const nameB = shortName(modelB.model) const lowDataA = modelA.calls < LOW_DATA_THRESHOLD @@ -137,59 +200,103 @@ function ComparisonResults({ modelA, modelB, rows, onBack }: ComparisonResultsPr if (key.escape) { onBack(); return } }) + const sectionOrder: string[] = [] + const sectionRows = new Map() + for (const row of rows) { + if (!sectionRows.has(row.section)) { + sectionOrder.push(row.section) + sectionRows.set(row.section, []) + } + sectionRows.get(row.section)!.push(row) + } + + const fmtTokens = (n: number) => { + if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M` + if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K` + return String(n) + } + const contextRows: { label: string; valueA: string; valueB: string }[] = [ { label: 'Calls', valueA: modelA.calls.toLocaleString(), valueB: modelB.calls.toLocaleString() }, { label: 'Total cost', valueA: formatCost(modelA.cost), valueB: formatCost(modelB.cost) }, + { label: 'Input tokens', valueA: fmtTokens(modelA.inputTokens), valueB: fmtTokens(modelB.inputTokens) }, + { label: 'Output tokens', valueA: fmtTokens(modelA.outputTokens), valueB: fmtTokens(modelB.outputTokens) }, { label: 'Days of data', valueA: String(daysOfData(modelA.firstSeen, modelA.lastSeen)), valueB: String(daysOfData(modelB.firstSeen, modelB.lastSeen)) }, { label: 'Edit turns', valueA: modelA.editTurns.toLocaleString(), valueB: modelB.editTurns.toLocaleString() }, { label: 'Self-corrections', valueA: modelA.selfCorrections.toLocaleString(), valueB: modelB.selfCorrections.toLocaleString() }, ] + const lowDataWarning = (lowDataA || lowDataB) + ? `Note: ${[lowDataA && shortName(modelA.model), lowDataB && shortName(modelB.model)].filter(Boolean).join(' and ')} ha${lowDataA && lowDataB ? 've' : 's'} fewer than ${LOW_DATA_THRESHOLD} calls` + : undefined + + const pw = wide ? halfWidth : dashWidth + return ( - + {nameA} vs {nameB} - - - {''.padEnd(LABEL_WIDTH)} - {nameA.padStart(VALUE_WIDTH)} - {nameB.padStart(VALUE_WIDTH)} - - {rows.map(row => { - const fmtA = formatValue(row.valueA, row.formatFn) - const fmtB = formatValue(row.valueB, row.formatFn) - - return ( - - {row.label.padEnd(LABEL_WIDTH)} - {fmtA.padStart(VALUE_WIDTH)} - {fmtB.padStart(VALUE_WIDTH)} - - ) - })} - - {'-- Context '.padEnd(LABEL_WIDTH + VALUE_WIDTH * 2, '-')} - {contextRows.map(row => ( - - {row.label.padEnd(LABEL_WIDTH)} - {row.valueA.padStart(VALUE_WIDTH)} - {row.valueB.padStart(VALUE_WIDTH)} - - ))} - {(lowDataA || lowDataB) && ( - <> - - - Note: {[lowDataA && modelA.model, lowDataB && modelB.model].filter(Boolean).join(' and ')} ha{lowDataA && lowDataB ? 've' : 's'} fewer than {LOW_DATA_THRESHOLD} calls -- results may not be representative. - - - )} - + + + + + + + {categories.length > 0 && ( + + Category Head-to-Head + one-shot rate per category + + {' '} + {FULL_BLOCK + FULL_BLOCK} + {nameA} + {FULL_BLOCK + FULL_BLOCK} + {nameB} + + {categories.map(cat => { + const bwA = cat.oneShotRateA !== null ? barWidth(cat.oneShotRateA) : 0 + const bwB = cat.oneShotRateB !== null ? barWidth(cat.oneShotRateB) : 0 + const rateA = cat.oneShotRateA !== null ? `${cat.oneShotRateA.toFixed(1)}%` : '-' + const rateB = cat.oneShotRateB !== null ? `${cat.oneShotRateB.toFixed(1)}%` : '-' + const turnsA = cat.editTurnsA > 0 ? `(${cat.editTurnsA})` : '' + const turnsB = cat.editTurnsB > 0 ? `(${cat.editTurnsB})` : '' + + return ( + + + {' '}{cat.category} + + {' '} + {FULL_BLOCK.repeat(Math.max(bwA, 1))} + {' '.repeat(Math.max(0, BAR_MAX_WIDTH - bwA))} + {rateA.padStart(6)} + {turnsA} + + + {' '} + {FULL_BLOCK.repeat(Math.max(bwB, 1))} + {' '.repeat(Math.max(0, BAR_MAX_WIDTH - bwB))} + {rateB.padStart(6)} + {turnsB} + + + ) + })} + + )} + + + {workingStyle.length > 0 && ( + ({ label: r.label, valueA: formatValue(r.valueA, r.formatFn), valueB: formatValue(r.valueB, r.formatFn) }))} nameA={nameA} nameB={nameB} pw={pw} /> + )} + + + [esc] back [q] quit @@ -211,6 +318,8 @@ export function CompareView({ projects, onBack }: CompareViewProps) { const [selectedA, setSelectedA] = useState(null) const [selectedB, setSelectedB] = useState(null) const [rows, setRows] = useState([]) + const [categories, setCategories] = useState([]) + const [style, setStyle] = useState([]) const [loadTrigger, setLoadTrigger] = useState(0) const projectsRef = useRef(projects) projectsRef.current = projects @@ -251,11 +360,14 @@ export function CompareView({ projects, onBack }: CompareViewProps) { const corrections = await scanSelfCorrections(dirs) if (cancelled) return + const currentProjects = projectsRef.current const aCopy = { ...a!, selfCorrections: corrections.get(a!.model) ?? 0 } const bCopy = { ...b!, selfCorrections: corrections.get(b!.model) ?? 0 } setSelectedA(aCopy) setSelectedB(bCopy) setRows(computeComparison(aCopy, bCopy)) + setCategories(computeCategoryComparison(currentProjects, a!.model, b!.model)) + setStyle(computeWorkingStyle(currentProjects, a!.model, b!.model)) setPhase('results') } @@ -277,7 +389,7 @@ export function CompareView({ projects, onBack }: CompareViewProps) { Model Comparison - Need at least 2 models to compare. Found {models.length}. + Need at least 2 models to compare. Found {models.length}. @@ -299,7 +411,7 @@ export function CompareView({ projects, onBack }: CompareViewProps) { Model Comparison - Scanning self-corrections... + Scanning self-corrections... ) @@ -311,6 +423,8 @@ export function CompareView({ projects, onBack }: CompareViewProps) { modelA={selectedA} modelB={selectedB} rows={rows} + categories={categories} + workingStyle={style} onBack={() => setPhase('select')} /> ) diff --git a/tests/compare-stats.test.ts b/tests/compare-stats.test.ts index d1441a4..a7ecb85 100644 --- a/tests/compare-stats.test.ts +++ b/tests/compare-stats.test.ts @@ -2,13 +2,14 @@ import { mkdtemp, mkdir, rm, writeFile } from 'fs/promises' import { join } from 'path' import { tmpdir } from 'os' import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { aggregateModelStats, computeComparison, scanSelfCorrections, type ModelStats } from '../src/compare-stats.js' +import { aggregateModelStats, computeComparison, computeCategoryComparison, computeWorkingStyle, scanSelfCorrections, type ModelStats } from '../src/compare-stats.js' import type { ProjectSummary, SessionSummary, ClassifiedTurn } from '../src/types.js' -function makeTurn(model: string, cost: number, opts: { hasEdits?: boolean; retries?: number; outputTokens?: number; inputTokens?: number; cacheRead?: number; cacheWrite?: number; timestamp?: string } = {}): ClassifiedTurn { +function makeTurn(model: string, cost: number, opts: { hasEdits?: boolean; retries?: number; outputTokens?: number; inputTokens?: number; cacheRead?: number; cacheWrite?: number; timestamp?: string; category?: string; hasAgentSpawn?: boolean; hasPlanMode?: boolean; speed?: 'standard' | 'fast'; tools?: string[] } = {}): ClassifiedTurn { + const defaultTools = opts.tools ?? (opts.hasEdits ? ['Edit'] : ['Read']) return { timestamp: opts.timestamp ?? '2026-04-15T10:00:00Z', - category: 'coding', + category: (opts.category ?? 'coding') as ClassifiedTurn['category'], retries: opts.retries ?? 0, hasEdits: opts.hasEdits ?? false, userMessage: '', @@ -25,11 +26,11 @@ function makeTurn(model: string, cost: number, opts: { hasEdits?: boolean; retri webSearchRequests: 0, }, costUSD: cost, - tools: opts.hasEdits ? ['Edit'] : ['Read'], + tools: defaultTools, mcpTools: [], - hasAgentSpawn: false, - hasPlanMode: false, - speed: 'standard' as const, + hasAgentSpawn: opts.hasAgentSpawn ?? false, + hasPlanMode: opts.hasPlanMode ?? false, + speed: opts.speed ?? 'standard' as const, timestamp: opts.timestamp ?? '2026-04-15T10:00:00Z', bashCommands: [], deduplicationKey: `key-${Math.random()}`, @@ -128,6 +129,17 @@ describe('aggregateModelStats', () => { expect(aggregateModelStats([])).toEqual([]) }) + it('tracks editCost for edit turns', () => { + const project = makeProject([ + makeTurn('opus-4-6', 0.10, { hasEdits: true }), + makeTurn('opus-4-6', 0.20, { hasEdits: true }), + makeTurn('opus-4-6', 0.50, { hasEdits: false }), + ]) + const stats = aggregateModelStats([project]) + const m = stats.find(s => s.model === 'opus-4-6')! + expect(m.editCost).toBeCloseTo(0.30) + }) + it('sorts by cost descending', () => { const project = makeProject([ makeTurn('cheap-model', 0.01), @@ -153,6 +165,7 @@ function makeStats(overrides: Partial = {}): ModelStats { oneShotTurns: 60, retries: 20, selfCorrections: 10, + editCost: 8, firstSeen: '2026-04-01T00:00:00Z', lastSeen: '2026-04-15T00:00:00Z', ...overrides, @@ -199,6 +212,16 @@ describe('computeComparison', () => { expect(costRow.winner).toBe('tie') }) + it('computes cost per edit correctly', () => { + const a = makeStats({ editTurns: 40, editCost: 4 }) + const b = makeStats({ editTurns: 80, editCost: 4 }) + const rows = computeComparison(a, b) + const editRow = rows.find(r => r.label === 'Cost / edit')! + expect(editRow.valueA).toBeCloseTo(0.10) + expect(editRow.valueB).toBeCloseTo(0.05) + expect(editRow.winner).toBe('b') + }) + it('picks higher value as winner for cache hit rate', () => { const a = makeStats({ inputTokens: 5000, cacheReadTokens: 30000, cacheWriteTokens: 5000 }) const b = makeStats({ inputTokens: 10000, cacheReadTokens: 10000, cacheWriteTokens: 5000 }) @@ -354,3 +377,192 @@ describe('scanSelfCorrections', () => { } }) }) + +describe('computeCategoryComparison', () => { + it('returns per-category one-shot rates for both models', () => { + const project = makeProject([ + makeTurn('model-a', 0.10, { hasEdits: true, retries: 0, category: 'coding' }), + makeTurn('model-a', 0.10, { hasEdits: true, retries: 1, category: 'coding' }), + makeTurn('model-b', 0.10, { hasEdits: true, retries: 0, category: 'coding' }), + makeTurn('model-b', 0.10, { hasEdits: true, retries: 0, category: 'coding' }), + makeTurn('model-a', 0.10, { hasEdits: true, retries: 0, category: 'debugging' }), + makeTurn('model-b', 0.10, { hasEdits: true, retries: 1, category: 'debugging' }), + ]) + const result = computeCategoryComparison([project], 'model-a', 'model-b') + + const coding = result.find(r => r.category === 'coding')! + expect(coding.editTurnsA).toBe(2) + expect(coding.oneShotRateA).toBeCloseTo(50) + expect(coding.editTurnsB).toBe(2) + expect(coding.oneShotRateB).toBeCloseTo(100) + expect(coding.winner).toBe('b') + + const debugging = result.find(r => r.category === 'debugging')! + expect(debugging.oneShotRateA).toBeCloseTo(100) + expect(debugging.oneShotRateB).toBeCloseTo(0) + expect(debugging.winner).toBe('a') + }) + + it('skips categories with no edit turns', () => { + const project = makeProject([ + makeTurn('model-a', 0.10, { hasEdits: false, category: 'conversation' }), + makeTurn('model-b', 0.10, { hasEdits: false, category: 'conversation' }), + makeTurn('model-a', 0.10, { hasEdits: true, category: 'coding' }), + ]) + const result = computeCategoryComparison([project], 'model-a', 'model-b') + expect(result.find(r => r.category === 'conversation')).toBeUndefined() + expect(result).toHaveLength(1) + }) + + it('sorts by total turns descending', () => { + const project = makeProject([ + makeTurn('model-a', 0.10, { hasEdits: true, category: 'coding' }), + makeTurn('model-a', 0.10, { hasEdits: true, category: 'coding' }), + makeTurn('model-a', 0.10, { hasEdits: true, category: 'coding' }), + makeTurn('model-b', 0.10, { hasEdits: true, category: 'coding' }), + makeTurn('model-a', 0.10, { hasEdits: true, category: 'debugging' }), + ]) + const result = computeCategoryComparison([project], 'model-a', 'model-b') + expect(result[0].category).toBe('coding') + }) + + it('returns null one-shot rate when model has no edits in category', () => { + const project = makeProject([ + makeTurn('model-a', 0.10, { hasEdits: true, category: 'coding' }), + makeTurn('model-b', 0.10, { hasEdits: false, category: 'coding' }), + ]) + const result = computeCategoryComparison([project], 'model-a', 'model-b') + const coding = result.find(r => r.category === 'coding')! + expect(coding.oneShotRateA).toBeCloseTo(100) + expect(coding.oneShotRateB).toBeNull() + expect(coding.winner).toBe('none') + }) +}) + +describe('computeWorkingStyle', () => { + it('computes delegation and planning rates', () => { + const project = makeProject([ + makeTurn('model-a', 0.10, { hasAgentSpawn: true }), + makeTurn('model-a', 0.10, {}), + makeTurn('model-a', 0.10, { hasPlanMode: true }), + makeTurn('model-b', 0.10, {}), + makeTurn('model-b', 0.10, {}), + ]) + const result = computeWorkingStyle([project], 'model-a', 'model-b') + + const delegation = result.find(r => r.label === 'Delegation rate')! + expect(delegation.valueA).toBeCloseTo(100 / 3) + expect(delegation.valueB).toBeCloseTo(0) + + const planning = result.find(r => r.label === 'Planning rate')! + expect(planning.valueA).toBeCloseTo(100 / 3) + expect(planning.valueB).toBeCloseTo(0) + }) + + it('computes avg tools per turn', () => { + const project = makeProject([ + makeTurn('model-a', 0.10, { hasEdits: true }), + makeTurn('model-a', 0.10, {}), + makeTurn('model-b', 0.10, { hasEdits: true }), + ]) + const result = computeWorkingStyle([project], 'model-a', 'model-b') + const tools = result.find(r => r.label === 'Avg tools / turn')! + expect(tools.valueA).toBeCloseTo(1) + expect(tools.valueB).toBeCloseTo(1) + }) + + it('computes fast mode usage', () => { + const project = makeProject([ + makeTurn('model-a', 0.10, { speed: 'fast' }), + makeTurn('model-a', 0.10, {}), + makeTurn('model-b', 0.10, { speed: 'fast' }), + makeTurn('model-b', 0.10, { speed: 'fast' }), + ]) + const result = computeWorkingStyle([project], 'model-a', 'model-b') + const fast = result.find(r => r.label === 'Fast mode usage')! + expect(fast.valueA).toBeCloseTo(50) + expect(fast.valueB).toBeCloseTo(100) + }) + + it('returns null for models with no turns', () => { + const project = makeProject([ + makeTurn('model-a', 0.10, {}), + ]) + const result = computeWorkingStyle([project], 'model-a', 'model-b') + const delegation = result.find(r => r.label === 'Delegation rate')! + expect(delegation.valueA).toBeCloseTo(0) + expect(delegation.valueB).toBeNull() + }) + + it('counts TaskCreate as planning', () => { + const project = makeProject([ + makeTurn('model-a', 0.10, { tools: ['TaskCreate'] }), + makeTurn('model-a', 0.10, { tools: ['Read'] }), + makeTurn('model-a', 0.10, { tools: ['Edit'] }), + ]) + const result = computeWorkingStyle([project], 'model-a', 'model-b') + const planning = result.find(r => r.label === 'Planning rate')! + expect(planning.valueA).toBeCloseTo(100 / 3) + }) + + it('counts TaskUpdate as planning', () => { + const project = makeProject([ + makeTurn('model-a', 0.10, { tools: ['TaskUpdate'] }), + makeTurn('model-a', 0.10, { tools: ['Read'] }), + ]) + const result = computeWorkingStyle([project], 'model-a', 'model-b') + const planning = result.find(r => r.label === 'Planning rate')! + expect(planning.valueA).toBeCloseTo(50) + }) + + it('counts TodoWrite as planning', () => { + const project = makeProject([ + makeTurn('model-a', 0.10, { tools: ['TodoWrite', 'Read'] }), + makeTurn('model-a', 0.10, { tools: ['Bash'] }), + ]) + const result = computeWorkingStyle([project], 'model-a', 'model-b') + const planning = result.find(r => r.label === 'Planning rate')! + expect(planning.valueA).toBeCloseTo(50) + }) + + it('counts turn with planning tool + edits as planning', () => { + const project = makeProject([ + makeTurn('model-a', 0.10, { tools: ['TaskCreate', 'Edit', 'Read'] }), + makeTurn('model-a', 0.10, { tools: ['Edit'] }), + ]) + const result = computeWorkingStyle([project], 'model-a', 'model-b') + const planning = result.find(r => r.label === 'Planning rate')! + expect(planning.valueA).toBeCloseTo(50) + }) + + it('does not count regular tools as planning', () => { + const project = makeProject([ + makeTurn('model-a', 0.10, { tools: ['Read', 'Grep', 'Glob'] }), + makeTurn('model-a', 0.10, { tools: ['Edit', 'Bash'] }), + makeTurn('model-a', 0.10, { tools: ['Agent'] }), + ]) + const result = computeWorkingStyle([project], 'model-a', 'model-b') + const planning = result.find(r => r.label === 'Planning rate')! + expect(planning.valueA).toBeCloseTo(0) + }) + + it('counts planning once per turn even with multiple planning tools', () => { + const project = makeProject([ + makeTurn('model-a', 0.10, { tools: ['TaskCreate', 'TaskUpdate', 'TaskCreate'] }), + makeTurn('model-a', 0.10, { tools: ['Read'] }), + ]) + const result = computeWorkingStyle([project], 'model-a', 'model-b') + const planning = result.find(r => r.label === 'Planning rate')! + expect(planning.valueA).toBeCloseTo(50) + }) + + it('hasPlanMode still triggers planning rate', () => { + const project = makeProject([ + makeTurn('model-a', 0.10, { hasPlanMode: true, tools: ['Read'] }), + makeTurn('model-a', 0.10, { tools: ['Read'] }), + ]) + const result = computeWorkingStyle([project], 'model-a', 'model-b') + const planning = result.find(r => r.label === 'Planning rate')! + expect(planning.valueA).toBeCloseTo(50) + }) +})