diff --git a/CHANGELOG.md b/CHANGELOG.md index 378017b..99eb76f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,24 @@ ## Unreleased +### Added (CLI) +- **`codeburn models` command.** Per-model breakdown across all providers, + one row per (provider, model), sorted by cost. Each row carries Input, + Output, Cache Write, Cache Read, Total, and Cost columns plus a Top Task + cell showing the dominant task category and its cost share (e.g. + `Coding (42%)`). Pass `--by-task` to explode each model into one row per + task type, with provider/model cells blanked on subsequent rows of the + same group and a horizontal divider between groups. Filters: `--period` + (default `30days`), `--from/--to`, `--provider`, `--task`, `--top`, + `--min-cost`, `--no-totals`. Output formats: `table` (Unicode box-drawn, + default), `markdown` (GitHub-flavored, copy-paste friendly), `json`, + `csv`. The table renderer auto-sizes every column to its content and + drops cache columns first, then input/output, then top-task when the + terminal is too narrow to fit the full set. Headers are cyan, totals row + is yellow, provider name is dim. Inspired by tokscale's per-model table + and ccusage's responsive cli-table3 layout, ported to plain Node with + no new runtime dependency. + ### Changed (CLI) - **`optimize` suggestions now declare their destination.** Every paste-style fix carries an explicit destination — `claude-md` (permanent project rule), diff --git a/README.md b/README.md index 8c09893..0dd68e0 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,12 @@ codeburn optimize -p week # scope the scan to last 7 days codeburn compare # side-by-side model comparison codeburn yield # track productive vs reverted/abandoned spend codeburn yield -p 30days # yield analysis for last 30 days +codeburn models # per-model token + cost table (last 30 days) +codeburn models --by-task # explode each model into per-task-type rows +codeburn models --top 10 # only the top 10 by cost +codeburn models --format markdown # paste-friendly markdown table +codeburn models --task feature # filter to feature-development work +codeburn models --provider claude # filter to one provider ``` Arrow keys switch between Today, 7 Days, 30 Days, Month, and 6 Months (use `--from` / `--to` for an exact historical window). Press `q` to quit, `1` `2` `3` `4` `5` as shortcuts, `c` to open model comparison, `o` to open optimize. The dashboard auto-refreshes every 30 seconds by default (`--refresh 0` to disable). It also shows average cost per session and the five most expensive sessions across all projects. diff --git a/src/cli.ts b/src/cli.ts index 080bdbd..fa44827 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -878,6 +878,63 @@ program await renderCompare(range, opts.provider) }) +program + .command('models') + .description('Per-model token + cost table, optionally exploded by task type') + .option('-p, --period ', 'Analysis period: today, week, 30days, month, all', '30days') + .option('--from ', 'Custom range start (YYYY-MM-DD)') + .option('--to ', 'Custom range end (YYYY-MM-DD)') + .option('--provider ', 'Filter by provider (e.g. claude, codex, cursor)', 'all') + .option('--task ', 'Filter to one task type (e.g. feature, debugging, refactoring)') + .option('--by-task', 'One row per (provider, model, task) instead of one row per (provider, model)') + .option('--top ', 'Show only the top N rows', (v: string) => parseInt(v, 10)) + .option('--min-cost ', 'Hide rows below this cost threshold', (v: string) => parseFloat(v)) + .option('--no-totals', 'Suppress the footer totals row') + .option('--format ', 'Output format: table, markdown, json, csv', 'table') + .action(async (opts) => { + const { aggregateModels, renderTable, renderMarkdown, renderJson, renderCsv } = await import('./models-report.js') + await loadPricing() + await hydrateCache() + + let range + if (opts.from || opts.to) { + const customRange = parseDateRangeFlags(opts.from, opts.to) + if (!customRange) { + process.stderr.write('codeburn: --from and --to must be valid YYYY-MM-DD dates\n') + process.exit(1) + } + range = customRange + } else { + range = getDateRange(opts.period).range + } + + const projects = await parseAllSessions(range, opts.provider) + const rows = await aggregateModels(projects, { + byTask: !!opts.byTask, + taskFilter: opts.task, + topN: typeof opts.top === 'number' && Number.isFinite(opts.top) ? opts.top : undefined, + minCost: typeof opts.minCost === 'number' && Number.isFinite(opts.minCost) ? opts.minCost : 0.01, + }) + + const fmt = (opts.format ?? 'table').toLowerCase() + if (rows.length === 0 && (fmt === 'table' || fmt === 'markdown')) { + process.stdout.write('No model usage found for the selected period.\n') + return + } + if (fmt === 'json') { + process.stdout.write(renderJson(rows) + '\n') + } else if (fmt === 'csv') { + process.stdout.write(renderCsv(rows, { byTask: !!opts.byTask }) + '\n') + } else if (fmt === 'markdown' || fmt === 'md') { + process.stdout.write(renderMarkdown(rows, { byTask: !!opts.byTask, showTotals: opts.totals !== false }) + '\n') + } else if (fmt === 'table') { + process.stdout.write(renderTable(rows, { byTask: !!opts.byTask, showTotals: opts.totals !== false }) + '\n') + } else { + process.stderr.write(`codeburn: unknown --format "${opts.format}". Choose table, markdown, json, or csv.\n`) + process.exit(1) + } + }) + program .command('yield') .description('Track which AI spend shipped to main vs reverted/abandoned (experimental)') diff --git a/src/models-report.ts b/src/models-report.ts new file mode 100644 index 0000000..ab70646 --- /dev/null +++ b/src/models-report.ts @@ -0,0 +1,645 @@ +import chalk from 'chalk' +import stripAnsi from 'strip-ansi' + +import { formatCost, formatTokens } from './format.js' +import { getProvider } from './providers/index.js' +import { CATEGORY_LABELS, type ProjectSummary, type TaskCategory } from './types.js' + +export type ModelReportRow = { + provider: string + providerDisplayName: string + model: string + modelDisplayName: string + category: TaskCategory | null + inputTokens: number + outputTokens: number + cacheWriteTokens: number + cacheReadTokens: number + totalTokens: number + costUSD: number + calls: number + topCategory?: TaskCategory + topCategoryCost?: number + topCategoryShare?: number +} + +export type AggregateOptions = { + byTask?: boolean + taskFilter?: TaskCategory + topN?: number + minCost?: number +} + +type Bucket = { + provider: string + model: string + category: TaskCategory | null + inputTokens: number + outputTokens: number + cacheWriteTokens: number + cacheReadTokens: number + costUSD: number + calls: number +} + +type ModelKey = string +type CategoryKey = string + +function bucketKey(provider: string, model: string, category: TaskCategory | null): string { + return `${provider} ${model} ${category ?? ''}` +} + +/// Walks every parsed turn, attributes each assistant call to a +/// (provider, model, category) bucket, and returns rows keyed by either +/// (provider, model) when `byTask` is false or (provider, model, category) when true. +/// +/// Default view: rows sorted by cost descending. +/// byTask view: rows grouped by (provider, model) so the renderer can blank +/// repeated provider/model cells. Group order follows total cost across that +/// model; within each group, rows go by cost descending. +export async function aggregateModels(projects: ProjectSummary[], opts: AggregateOptions = {}): Promise { + const buckets = new Map() + const perModelCategoryCost = new Map>() + const perModelTotalCost = new Map() + + for (const project of projects) { + for (const session of project.sessions) { + for (const turn of session.turns) { + if (opts.taskFilter && turn.category !== opts.taskFilter) continue + for (const call of turn.assistantCalls) { + const provider = call.provider || 'unknown' + const model = call.model || 'unknown' + const category: TaskCategory | null = opts.byTask ? turn.category : null + const key = bucketKey(provider, model, category) + let bucket = buckets.get(key) + if (!bucket) { + bucket = { + provider, + model, + category, + inputTokens: 0, + outputTokens: 0, + cacheWriteTokens: 0, + cacheReadTokens: 0, + costUSD: 0, + calls: 0, + } + buckets.set(key, bucket) + } + bucket.inputTokens += call.usage.inputTokens + bucket.outputTokens += call.usage.outputTokens + call.usage.reasoningTokens + bucket.cacheWriteTokens += call.usage.cacheCreationInputTokens + bucket.cacheReadTokens += call.usage.cacheReadInputTokens + call.usage.cachedInputTokens + bucket.costUSD += call.costUSD + bucket.calls += 1 + + const modelKey = `${provider} ${model}` + let perCat = perModelCategoryCost.get(modelKey) + if (!perCat) { + perCat = new Map() + perModelCategoryCost.set(modelKey, perCat) + } + perCat.set(turn.category, (perCat.get(turn.category) ?? 0) + call.costUSD) + perModelTotalCost.set(modelKey, (perModelTotalCost.get(modelKey) ?? 0) + call.costUSD) + } + } + } + } + + const providerCache = new Map string }>() + async function resolveProvider(name: string) { + const cached = providerCache.get(name) + if (cached) return cached + const p = await getProvider(name) + const entry = { + displayName: p?.displayName ?? name, + formatModel: p ? (m: string) => p.modelDisplayName(m) : (m: string) => m, + } + providerCache.set(name, entry) + return entry + } + + const rows: ModelReportRow[] = [] + for (const bucket of buckets.values()) { + const meta = await resolveProvider(bucket.provider) + const total = bucket.inputTokens + bucket.outputTokens + bucket.cacheWriteTokens + bucket.cacheReadTokens + const row: ModelReportRow = { + provider: bucket.provider, + providerDisplayName: meta.displayName, + model: bucket.model, + modelDisplayName: meta.formatModel(bucket.model), + category: bucket.category, + inputTokens: bucket.inputTokens, + outputTokens: bucket.outputTokens, + cacheWriteTokens: bucket.cacheWriteTokens, + cacheReadTokens: bucket.cacheReadTokens, + totalTokens: total, + costUSD: bucket.costUSD, + calls: bucket.calls, + } + + if (!opts.byTask) { + const perCat = perModelCategoryCost.get(`${bucket.provider} ${bucket.model}`) + if (perCat && perCat.size > 0) { + let topCat: TaskCategory = 'general' + let topCost = -1 + let totalCost = 0 + for (const [cat, cost] of perCat.entries()) { + totalCost += cost + if (cost > topCost) { + topCost = cost + topCat = cat + } + } + row.topCategory = topCat + row.topCategoryCost = topCost + row.topCategoryShare = totalCost > 0 ? topCost / totalCost : 0 + } + } + + rows.push(row) + } + + if (opts.byTask) { + rows.sort((a, b) => { + const aTotal = perModelTotalCost.get(`${a.provider} ${a.model}`) ?? 0 + const bTotal = perModelTotalCost.get(`${b.provider} ${b.model}`) ?? 0 + if (aTotal !== bTotal) return bTotal - aTotal + if (a.provider !== b.provider) return a.provider.localeCompare(b.provider) + if (a.model !== b.model) return a.model.localeCompare(b.model) + return b.costUSD - a.costUSD + }) + } else { + rows.sort((a, b) => b.costUSD - a.costUSD) + } + + let filtered = rows + if (opts.minCost !== undefined) { + filtered = filtered.filter(r => r.costUSD >= opts.minCost!) + } + if (opts.topN !== undefined) { + filtered = filtered.slice(0, opts.topN) + } + return filtered +} + +function visibleLength(text: string): number { + return stripAnsi(text).length +} + +function pad(text: string, width: number, align: 'left' | 'right' = 'left'): string { + const visible = visibleLength(text) + if (visible >= width) return text + const filler = ' '.repeat(width - visible) + return align === 'left' ? text + filler : filler + text +} + +function categoryLabel(c: TaskCategory): string { + return CATEGORY_LABELS[c] ?? c +} + +/// Box-drawing preset matching tokscale's comfy-table layout. Pure Unicode; +/// every modern terminal handles these. JSON / CSV / Markdown formats already +/// cover the no-Unicode case for downstream tooling. +const BOX = { + topLeft: '┌', + topRight: '┐', + bottomLeft: '└', + bottomRight: '┘', + topT: '┬', + bottomT: '┴', + leftT: '├', + rightT: '┤', + cross: '┼', + horizontal: '─', + vertical: '│', +} + +type Column = { + header: string + align: 'left' | 'right' + width: number + /// Drop priority. 0 = always shown; higher numbers get dropped first when + /// the terminal is narrow. + priority: number + key: 'provider' | 'model' | 'task' | 'input' | 'output' | 'cacheWrite' | 'cacheRead' | 'total' | 'cost' +} + +type TableRenderOptions = { + byTask?: boolean + showTotals?: boolean + terminalWidth?: number + fullWidth?: boolean +} + +const DROP_COLUMN_GROUPS: Array> = [ + ['cacheWrite', 'cacheRead'], + ['input', 'output'], + ['task'], +] + +function defaultColumns(byTask: boolean): Column[] { + // Higher priority numbers drop FIRST when the terminal is narrow. + // Cache columns are the cheapest to lose, then input/output, then top-task. + // Provider/Model/Total/Cost stay regardless. + // Widths are MINIMUMS; sizeColumnsToContent() expands them to fit cell text. + return [ + { key: 'provider', header: 'Provider', align: 'left', width: 8, priority: 0 }, + { key: 'model', header: 'Model', align: 'left', width: 8, priority: 0 }, + { key: 'task', header: byTask ? 'Task' : 'Top Task', align: 'left', width: 8, priority: 1 }, + { key: 'input', header: 'Input', align: 'right', width: 6, priority: 2 }, + { key: 'output', header: 'Output', align: 'right', width: 6, priority: 2 }, + { key: 'cacheWrite', header: 'Cache Write', align: 'right', width: 11, priority: 3 }, + { key: 'cacheRead', header: 'Cache Read', align: 'right', width: 10, priority: 3 }, + { key: 'total', header: 'Total', align: 'right', width: 6, priority: 0 }, + { key: 'cost', header: 'Cost', align: 'right', width: 6, priority: 0 }, + ] +} + +/// Expands each column's width to fit the widest cell in that column, so a +/// short header (e.g. "Task") in a fixed 18-wide cell does not leave 14 chars +/// of trailing whitespace. Mirrors cli-table3 / comfy-table auto-sizing. +function sizeColumnsToContent(columns: Column[], rows: string[][]): Column[] { + return columns.map((col, i) => { + let maxLen = visibleLength(col.header) + for (const row of rows) { + const cell = row[i] ?? '' + const len = visibleLength(cell) + if (len > maxLen) maxLen = len + } + return { ...col, width: Math.max(col.width, maxLen) } + }) +} + +function frameWidth(columns: Column[]): number { + if (columns.length === 0) return 0 + // 1 (left border) + sum(col + 2 padding) + (N-1) inner separators + 1 (right border) + return 2 + columns.reduce((acc, c) => acc + c.width + 2, 0) + (columns.length - 1) +} + +function chooseColumns(byTask: boolean, available: number): Column[] { + const all = defaultColumns(byTask) + if (frameWidth(all) <= available) return all + + // Drop in this order so the table degrades sensibly. Cache columns drop as + // a pair (showing only one of cache write / cache read looks broken). + const kept = new Set(all) + for (const group of DROP_COLUMN_GROUPS) { + for (const key of group) { + const col = all.find(c => c.key === key) + if (col) kept.delete(col) + } + const remaining = all.filter(c => kept.has(c)) + if (frameWidth(remaining) <= available) return remaining + } + return all.filter(c => c.priority === 0) +} + +function expandedColumnWeight(col: Column): number { + switch (col.key) { + case 'task': + case 'model': + return 3 + case 'provider': + return 2 + default: + return 1 + } +} + +/// Expands a fitted table to the available terminal width. The extra cells are +/// spread across all visible columns, weighted toward text columns so grouped +/// model/task rows breathe on wide terminals without turning numeric columns +/// into huge empty gutters. +function expandColumnsToWidth(columns: Column[], targetWidth: number): Column[] { + let remaining = targetWidth - frameWidth(columns) + if (remaining <= 0 || columns.length === 0) return columns + + const expanded = columns.map(c => ({ ...c })) + const weights = expanded.map(expandedColumnWeight) + const totalWeight = weights.reduce((sum, w) => sum + w, 0) + + for (let i = 0; i < expanded.length; i++) { + const add = Math.floor((targetWidth - frameWidth(columns)) * (weights[i]! / totalWeight)) + if (add <= 0) continue + expanded[i]!.width += add + remaining -= add + } + + // Hand out rounding leftovers in the same preference order. + const preferred: Column['key'][] = ['task', 'model', 'provider', 'total', 'cost', 'input', 'output', 'cacheRead', 'cacheWrite'] + while (remaining > 0) { + let changed = false + for (const key of preferred) { + const col = expanded.find(c => c.key === key) + if (!col) continue + col.width += 1 + remaining -= 1 + changed = true + if (remaining === 0) break + } + if (!changed) break + } + + return expanded +} + +function renderRow(cells: string[], columns: Column[]): string { + const padded = cells.map((c, i) => pad(c, columns[i]!.width, columns[i]!.align)) + return BOX.vertical + ' ' + padded.join(' ' + BOX.vertical + ' ') + ' ' + BOX.vertical +} + +function renderBorder(columns: Column[], left: string, mid: string, right: string): string { + const segments = columns.map(c => BOX.horizontal.repeat(c.width + 2)) + return left + segments.join(mid) + right +} + +function defaultTerminalWidth(): number { + const cols = process.stdout.columns + if (typeof cols === 'number' && cols > 0) return cols + // Honor $COLUMNS when stdout is not a TTY (piped, tee'd, etc.); some + // shells set it even when isTTY is false. + const envCols = process.env['COLUMNS'] ? parseInt(process.env['COLUMNS'], 10) : NaN + if (Number.isFinite(envCols) && envCols > 0) return envCols + // Conservative fallback. 100 keeps the table readable on the most common + // terminal sizes (80, 100, 120) without trying to fit cache columns into + // a window that cannot hold them. + return 100 +} + +/// Renders a Unicode box-drawn table. Columns are auto-sized to their content +/// (with declared `width` as a minimum). When the terminal is narrow, drops +/// the lowest-priority columns (cache first, then input/output, then top-task) +/// so the table fits without wrapping. +export function renderTable( + rows: ModelReportRow[], + opts: TableRenderOptions = {}, +): string { + const byTask = opts.byTask ?? false + const showTotals = opts.showTotals ?? true + const available = opts.terminalWidth ?? defaultTerminalWidth() + const fullWidth = opts.fullWidth ?? true + + const valueOf = (row: ModelReportRow, key: Column['key'], isNewGroup: boolean): string => { + switch (key) { + case 'provider': return isNewGroup ? row.providerDisplayName : '' + case 'model': return isNewGroup ? row.modelDisplayName : '' + case 'task': + if (byTask) return row.category ? categoryLabel(row.category) : '' + return row.topCategory + ? `${categoryLabel(row.topCategory)} ${chalk.dim(`(${Math.round((row.topCategoryShare ?? 0) * 100)}%)`)}` + : chalk.dim('-') + case 'input': return formatTokens(row.inputTokens) + case 'output': return formatTokens(row.outputTokens) + case 'cacheWrite': return formatTokens(row.cacheWriteTokens) + case 'cacheRead': return formatTokens(row.cacheReadTokens) + case 'total': return formatTokens(row.totalTokens) + case 'cost': return formatCost(row.costUSD) + } + } + + // Build all cell content first so we can size columns to fit. + type RowCells = { kind: 'data' | 'totals'; cells: string[]; isNewGroup: boolean } + const rowEntries: RowCells[] = [] + let prevProviderModel = '' + for (const row of rows) { + const groupKey = `${row.provider} ${row.model}` + const isNewGroup = !byTask || groupKey !== prevProviderModel + prevProviderModel = groupKey + const allCells = defaultColumns(byTask).map(col => { + const raw = valueOf(row, col.key, isNewGroup) + if (col.key === 'provider' && raw) return chalk.dim(raw) + return raw + }) + rowEntries.push({ kind: 'data', cells: allCells, isNewGroup }) + } + + let totalsEntry: RowCells | null = null + if (showTotals && rows.length > 0) { + const totals = rows.reduce( + (acc, r) => { + acc.input += r.inputTokens + acc.output += r.outputTokens + acc.cacheWrite += r.cacheWriteTokens + acc.cacheRead += r.cacheReadTokens + acc.total += r.totalTokens + acc.cost += r.costUSD + return acc + }, + { input: 0, output: 0, cacheWrite: 0, cacheRead: 0, total: 0, cost: 0 }, + ) + const cells = defaultColumns(byTask).map(col => { + switch (col.key) { + case 'provider': return '' + case 'model': return chalk.yellow.bold('Total') + case 'task': return '' + case 'input': return chalk.yellow(formatTokens(totals.input)) + case 'output': return chalk.yellow(formatTokens(totals.output)) + case 'cacheWrite': return chalk.yellow(formatTokens(totals.cacheWrite)) + case 'cacheRead': return chalk.yellow(formatTokens(totals.cacheRead)) + case 'total': return chalk.yellow.bold(formatTokens(totals.total)) + case 'cost': return chalk.yellow.bold(formatCost(totals.cost)) + } + }) + totalsEntry = { kind: 'totals', cells, isNewGroup: true } + } + + // Pick which columns to include based on terminal width, then size them. + // We index into `cells` by the column key to avoid object-identity pitfalls + // across defaultColumns() invocations. + const allKeys = defaultColumns(byTask).map(c => c.key) + const indexByKey = new Map(allKeys.map((k, i) => [k, i])) + const columns = chooseColumns(byTask, available) + const projectColumns = (cols: Column[], entry: RowCells) => + cols.map(c => entry.cells[indexByKey.get(c.key)!] ?? '') + const cellMatrix = [ + ...rowEntries.map(e => projectColumns(columns, e)), + ...(totalsEntry ? [projectColumns(columns, totalsEntry)] : []), + ] + const sized = sizeColumnsToContent(columns, cellMatrix) + + // If content sizing pushed the table back over budget, keep dropping the + // same low-value column groups until the rendered frame fits. + let final = sized + if (frameWidth(final) > available) { + let reduced = columns + for (const group of DROP_COLUMN_GROUPS) { + reduced = reduced.filter(c => !group.includes(c.key)) + const reducedMatrix = [ + ...rowEntries.map(e => projectColumns(reduced, e)), + ...(totalsEntry ? [projectColumns(reduced, totalsEntry)] : []), + ] + const candidate = sizeColumnsToContent(reduced, reducedMatrix) + final = candidate + if (frameWidth(candidate) <= available) break + } + } + + if (fullWidth && frameWidth(final) < available) { + final = expandColumnsToWidth(final, available) + } + + const lines: string[] = [] + lines.push(renderBorder(final, BOX.topLeft, BOX.topT, BOX.topRight)) + lines.push(renderRow(final.map(c => chalk.cyan.bold(c.header)), final)) + lines.push(renderBorder(final, BOX.leftT, BOX.cross, BOX.rightT)) + + let isFirstRow = true + for (const entry of rowEntries) { + if (byTask && entry.isNewGroup && !isFirstRow) { + lines.push(renderBorder(final, BOX.leftT, BOX.cross, BOX.rightT)) + } + isFirstRow = false + lines.push(renderRow(projectColumns(final, entry), final)) + } + + if (totalsEntry) { + lines.push(renderBorder(final, BOX.leftT, BOX.cross, BOX.rightT)) + lines.push(renderRow(projectColumns(final, totalsEntry), final)) + } + + lines.push(renderBorder(final, BOX.bottomLeft, BOX.bottomT, BOX.bottomRight)) + return lines.join('\n') +} + +export function renderJson(rows: ModelReportRow[]): string { + return JSON.stringify( + rows.map(r => ({ + provider: r.provider, + providerDisplayName: r.providerDisplayName, + model: r.model, + modelDisplayName: r.modelDisplayName, + category: r.category ?? r.topCategory ?? null, + topCategory: r.topCategory ?? null, + topCategoryShare: r.topCategoryShare ?? null, + inputTokens: r.inputTokens, + outputTokens: r.outputTokens, + cacheWriteTokens: r.cacheWriteTokens, + cacheReadTokens: r.cacheReadTokens, + totalTokens: r.totalTokens, + calls: r.calls, + costUSD: r.costUSD, + })), + null, + 2, + ) +} + +function csvEscape(value: string): string { + if (value.includes(',') || value.includes('"') || value.includes('\n')) { + return `"${value.replace(/"/g, '""')}"` + } + return value +} + +function mdEscape(value: string): string { + // Pipes break GitHub-flavored markdown tables; escape them. + return value.replace(/\|/g, '\\|') +} + +/// GitHub-flavored markdown table. Renders cleanly on GitHub, Notion, and most +/// chat platforms that understand markdown. Always shows provider/model on +/// every row (no blank-repeat trick) so the table remains useful when copied +/// into a context that loses whitespace alignment. +export function renderMarkdown(rows: ModelReportRow[], opts: { byTask?: boolean; showTotals?: boolean } = {}): string { + const byTask = opts.byTask ?? false + const showTotals = opts.showTotals ?? true + + const header = byTask + ? ['Provider', 'Model', 'Task', 'Input', 'Output', 'Cache Write', 'Cache Read', 'Total', 'Cost'] + : ['Provider', 'Model', 'Top Task', 'Input', 'Output', 'Cache Write', 'Cache Read', 'Total', 'Cost'] + const align = ['---', '---', '---', '---:', '---:', '---:', '---:', '---:', '---:'] + + const lines: string[] = [] + lines.push(`| ${header.join(' | ')} |`) + lines.push(`| ${align.join(' | ')} |`) + + for (const row of rows) { + const taskCell = byTask + ? row.category ? categoryLabel(row.category) : '' + : row.topCategory + ? `${categoryLabel(row.topCategory)} (${Math.round((row.topCategoryShare ?? 0) * 100)}%)` + : '-' + const cells = [ + mdEscape(row.providerDisplayName), + `\`${mdEscape(row.modelDisplayName)}\``, + taskCell, + formatTokens(row.inputTokens), + formatTokens(row.outputTokens), + formatTokens(row.cacheWriteTokens), + formatTokens(row.cacheReadTokens), + formatTokens(row.totalTokens), + formatCost(row.costUSD), + ] + lines.push(`| ${cells.join(' | ')} |`) + } + + if (showTotals && rows.length > 0) { + const totals = rows.reduce( + (acc, r) => { + acc.input += r.inputTokens + acc.output += r.outputTokens + acc.cacheWrite += r.cacheWriteTokens + acc.cacheRead += r.cacheReadTokens + acc.total += r.totalTokens + acc.cost += r.costUSD + return acc + }, + { input: 0, output: 0, cacheWrite: 0, cacheRead: 0, total: 0, cost: 0 }, + ) + const totalCells = [ + '', + '**Total**', + '', + `**${formatTokens(totals.input)}**`, + `**${formatTokens(totals.output)}**`, + `**${formatTokens(totals.cacheWrite)}**`, + `**${formatTokens(totals.cacheRead)}**`, + `**${formatTokens(totals.total)}**`, + `**${formatCost(totals.cost)}**`, + ] + lines.push(`| ${totalCells.join(' | ')} |`) + } + + return lines.join('\n') +} + +export function renderCsv(rows: ModelReportRow[], opts: { byTask?: boolean } = {}): string { + const byTask = opts.byTask ?? false + // CSV intentionally repeats provider/model on every row so downstream + // consumers can sort/filter without first reconstructing the grouping. + const header = byTask + ? ['provider', 'model', 'task', 'input_tokens', 'output_tokens', 'cache_write_tokens', 'cache_read_tokens', 'total_tokens', 'calls', 'cost_usd'] + : ['provider', 'model', 'top_task', 'top_task_share', 'input_tokens', 'output_tokens', 'cache_write_tokens', 'cache_read_tokens', 'total_tokens', 'calls', 'cost_usd'] + const lines: string[] = [header.join(',')] + for (const r of rows) { + const cells = byTask + ? [ + csvEscape(r.providerDisplayName), + csvEscape(r.modelDisplayName), + r.category ? categoryLabel(r.category) : '', + String(r.inputTokens), + String(r.outputTokens), + String(r.cacheWriteTokens), + String(r.cacheReadTokens), + String(r.totalTokens), + String(r.calls), + r.costUSD.toFixed(6), + ] + : [ + csvEscape(r.providerDisplayName), + csvEscape(r.modelDisplayName), + r.topCategory ? categoryLabel(r.topCategory) : '', + r.topCategoryShare !== undefined ? r.topCategoryShare.toFixed(4) : '', + String(r.inputTokens), + String(r.outputTokens), + String(r.cacheWriteTokens), + String(r.cacheReadTokens), + String(r.totalTokens), + String(r.calls), + r.costUSD.toFixed(6), + ] + lines.push(cells.join(',')) + } + return lines.join('\n') +} diff --git a/tests/models-report.test.ts b/tests/models-report.test.ts new file mode 100644 index 0000000..552673a --- /dev/null +++ b/tests/models-report.test.ts @@ -0,0 +1,466 @@ +import { describe, it, expect } from 'vitest' +import chalk from 'chalk' +import stripAnsi from 'strip-ansi' + +import { aggregateModels, renderTable, renderMarkdown, renderJson, renderCsv, type ModelReportRow } from '../src/models-report.js' +import type { + ProjectSummary, + SessionSummary, + ClassifiedTurn, + ParsedApiCall, + TokenUsage, + TaskCategory, +} from '../src/types.js' + +function emptyTokens(): TokenUsage { + return { + inputTokens: 0, + outputTokens: 0, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + } +} + +function makeCall(opts: { + provider: string + model: string + costUSD: number + input?: number + output?: number + cacheWrite?: number + cacheRead?: number +}): ParsedApiCall { + return { + provider: opts.provider, + model: opts.model, + usage: { + ...emptyTokens(), + inputTokens: opts.input ?? 0, + outputTokens: opts.output ?? 0, + cacheCreationInputTokens: opts.cacheWrite ?? 0, + cacheReadInputTokens: opts.cacheRead ?? 0, + }, + costUSD: opts.costUSD, + tools: [], + mcpTools: [], + skills: [], + hasAgentSpawn: false, + hasPlanMode: false, + speed: 'standard', + timestamp: '2026-05-09T00:00:00.000Z', + bashCommands: [], + deduplicationKey: `${opts.provider}-${opts.model}-${opts.costUSD}`, + } +} + +function makeTurn(category: TaskCategory, calls: ParsedApiCall[]): ClassifiedTurn { + return { + userMessage: 'test', + assistantCalls: calls, + timestamp: '2026-05-09T00:00:00.000Z', + sessionId: 's1', + category, + retries: 0, + hasEdits: false, + } +} + +function makeSession(turns: ClassifiedTurn[]): SessionSummary { + return { + sessionId: 's1', + project: 'p', + firstTimestamp: '2026-05-09T00:00:00.000Z', + lastTimestamp: '2026-05-09T00:00:00.000Z', + totalCostUSD: 0, + totalInputTokens: 0, + totalOutputTokens: 0, + totalCacheReadTokens: 0, + totalCacheWriteTokens: 0, + apiCalls: 0, + turns, + modelBreakdown: {}, + toolBreakdown: {}, + mcpBreakdown: {}, + bashBreakdown: {}, + categoryBreakdown: {} as SessionSummary['categoryBreakdown'], + skillBreakdown: {}, + } +} + +function makeProject(turns: ClassifiedTurn[]): ProjectSummary { + return { + project: 'p', + projectPath: '/tmp/p', + sessions: [makeSession(turns)], + totalCostUSD: 0, + totalApiCalls: 0, + } +} + +describe('aggregateModels', () => { + it('groups by (provider, model) and sorts by cost descending in default mode', async () => { + const project = makeProject([ + makeTurn('feature', [ + makeCall({ provider: 'claude', model: 'claude-sonnet-4-6', input: 1000, output: 200, cacheWrite: 500, cacheRead: 8000, costUSD: 5.0 }), + ]), + makeTurn('debugging', [ + makeCall({ provider: 'claude', model: 'claude-sonnet-4-6', input: 800, output: 100, cacheWrite: 300, cacheRead: 5000, costUSD: 3.5 }), + ]), + makeTurn('feature', [ + makeCall({ provider: 'codex', model: 'gpt-5', input: 600, output: 80, costUSD: 1.2 }), + ]), + ]) + const rows = await aggregateModels([project]) + expect(rows.map(r => `${r.provider}:${r.model}`)).toEqual(['claude:claude-sonnet-4-6', 'codex:gpt-5']) + const claudeRow = rows[0]! + expect(claudeRow.inputTokens).toBe(1800) + expect(claudeRow.outputTokens).toBe(300) + expect(claudeRow.cacheWriteTokens).toBe(800) + expect(claudeRow.cacheReadTokens).toBe(13000) + expect(claudeRow.costUSD).toBeCloseTo(8.5, 6) + expect(claudeRow.calls).toBe(2) + expect(claudeRow.totalTokens).toBe(1800 + 300 + 800 + 13000) + }) + + it('reports the dominant task type with its cost share in default mode', async () => { + const project = makeProject([ + makeTurn('feature', [makeCall({ provider: 'claude', model: 'claude-sonnet-4-6', costUSD: 6.0, input: 100, output: 20 })]), + makeTurn('debugging', [makeCall({ provider: 'claude', model: 'claude-sonnet-4-6', costUSD: 2.0, input: 50, output: 10 })]), + makeTurn('refactoring', [makeCall({ provider: 'claude', model: 'claude-sonnet-4-6', costUSD: 2.0, input: 50, output: 10 })]), + ]) + const rows = await aggregateModels([project]) + expect(rows[0]!.topCategory).toBe('feature') + expect(rows[0]!.topCategoryShare).toBeCloseTo(0.6, 3) + }) + + it('explodes rows by task in byTask mode and groups them so renderer can blank repeats', async () => { + const project = makeProject([ + makeTurn('feature', [makeCall({ provider: 'claude', model: 'claude-sonnet-4-6', costUSD: 6.0, input: 100, output: 20 })]), + makeTurn('debugging', [makeCall({ provider: 'claude', model: 'claude-sonnet-4-6', costUSD: 2.0, input: 50, output: 10 })]), + makeTurn('feature', [makeCall({ provider: 'codex', model: 'gpt-5', costUSD: 1.0, input: 60, output: 10 })]), + ]) + const rows = await aggregateModels([project], { byTask: true }) + expect(rows).toHaveLength(3) + // Group order: claude (8.0) before codex (1.0); within claude, feature (6.0) before debugging (2.0). + expect(rows.map(r => `${r.provider}:${r.model}:${r.category}`)).toEqual([ + 'claude:claude-sonnet-4-6:feature', + 'claude:claude-sonnet-4-6:debugging', + 'codex:gpt-5:feature', + ]) + }) + + it('respects taskFilter by excluding non-matching turns from every bucket', async () => { + const project = makeProject([ + makeTurn('feature', [makeCall({ provider: 'claude', model: 'claude-sonnet-4-6', costUSD: 5.0, input: 100, output: 20 })]), + makeTurn('debugging', [makeCall({ provider: 'claude', model: 'claude-sonnet-4-6', costUSD: 2.0, input: 50, output: 10 })]), + ]) + const rows = await aggregateModels([project], { taskFilter: 'feature' }) + expect(rows).toHaveLength(1) + expect(rows[0]!.costUSD).toBeCloseTo(5.0, 6) + }) + + it('applies topN and minCost filters', async () => { + const project = makeProject([ + makeTurn('feature', [makeCall({ provider: 'claude', model: 'claude-sonnet-4-6', costUSD: 5.0, input: 100, output: 20 })]), + makeTurn('feature', [makeCall({ provider: 'codex', model: 'gpt-5', costUSD: 0.5, input: 50, output: 10 })]), + makeTurn('feature', [makeCall({ provider: 'cursor', model: 'auto', costUSD: 0.001, input: 10, output: 1 })]), + ]) + const top = await aggregateModels([project], { topN: 1 }) + expect(top).toHaveLength(1) + const above = await aggregateModels([project], { minCost: 0.01 }) + expect(above.find(r => r.provider === 'cursor')).toBeUndefined() + }) + + it('counts reasoning tokens as output tokens', async () => { + const project = makeProject([ + makeTurn('feature', [ + { + provider: 'codex', + model: 'gpt-5', + usage: { ...emptyTokens(), inputTokens: 100, outputTokens: 50, reasoningTokens: 200 }, + costUSD: 1.0, + tools: [], + mcpTools: [], + skills: [], + hasAgentSpawn: false, + hasPlanMode: false, + speed: 'standard', + timestamp: '2026-05-09T00:00:00.000Z', + bashCommands: [], + deduplicationKey: 'k', + }, + ]), + ]) + const rows = await aggregateModels([project]) + expect(rows[0]!.outputTokens).toBe(250) + }) +}) + +describe('renderTable', () => { + function visibleWidth(line: string): number { + return stripAnsi(line).length + } + + function row(partial: Partial): ModelReportRow { + return { + provider: 'claude', + providerDisplayName: 'Claude', + model: 'claude-sonnet-4-6', + modelDisplayName: 'Sonnet 4.6', + category: null, + inputTokens: 0, + outputTokens: 0, + cacheWriteTokens: 0, + cacheReadTokens: 0, + totalTokens: 0, + costUSD: 0, + calls: 0, + ...partial, + } + } + + it('blanks repeated provider/model cells in byTask mode but keeps them in default mode', () => { + const rows: ModelReportRow[] = [ + row({ category: 'feature', costUSD: 7.78, inputTokens: 512_000, outputTokens: 98_000, cacheWriteTokens: 1_400_000, cacheReadTokens: 6_200_000, totalTokens: 8_210_000 }), + row({ category: 'debugging', costUSD: 5.31, inputTokens: 380_000, outputTokens: 71_000, cacheWriteTokens: 920_000, cacheReadTokens: 4_100_000, totalTokens: 5_471_000 }), + ] + const out = renderTable(rows, { byTask: true, showTotals: false, terminalWidth: 200 }) + const lines = out.split('\n') + // Layout: top border, header, header-separator, data..., bottom border. + const dataLines = lines.slice(3, -1) + expect(dataLines[0]).toContain('Sonnet 4.6') + expect(dataLines[0]).toContain('Feature Dev') + expect(dataLines[1]).not.toContain('Sonnet 4.6') + expect(dataLines[1]).not.toContain('Claude') + expect(dataLines[1]).toContain('Debugging') + }) + + it('keeps provider/model cells on every row in default mode', () => { + const rows: ModelReportRow[] = [ + row({ topCategory: 'feature', topCategoryShare: 0.6, costUSD: 5.0 }), + row({ provider: 'codex', providerDisplayName: 'Codex', model: 'gpt-5', modelDisplayName: 'GPT-5', topCategory: 'debugging', topCategoryShare: 0.4, costUSD: 1.2 }), + ] + const out = renderTable(rows, { byTask: false, showTotals: false, terminalWidth: 200 }) + const dataLines = out.split('\n').slice(3, -1) + expect(dataLines[0]).toContain('Sonnet 4.6') + expect(dataLines[1]).toContain('GPT-5') + }) + + it('drops cache columns when terminal is narrow', () => { + const rows: ModelReportRow[] = [row({ topCategory: 'feature', topCategoryShare: 1, costUSD: 1 })] + const wide = renderTable(rows, { showTotals: false, terminalWidth: 200 }) + const narrow = renderTable(rows, { showTotals: false, terminalWidth: 80 }) + expect(wide).toContain('Cache Write') + expect(narrow).not.toContain('Cache Write') + expect(narrow).not.toContain('Cache Read') + }) + + it('expands table borders to the available terminal width by default', () => { + const rows: ModelReportRow[] = [ + row({ category: 'coding', costUSD: 1.0, inputTokens: 46_300, outputTokens: 3_700_000, cacheWriteTokens: 16_300_000, cacheReadTokens: 1_569_800_000, totalTokens: 1_589_800_000 }), + row({ category: 'delegation', costUSD: 0.5, inputTokens: 44_200, outputTokens: 1_900_000, cacheWriteTokens: 9_400_000, cacheReadTokens: 499_600_000, totalTokens: 511_000_000 }), + ] + const out = renderTable(rows, { byTask: true, showTotals: false, terminalWidth: 132 }) + const lines = out.split('\n') + expect(visibleWidth(lines[0]!)).toBe(132) + expect(visibleWidth(lines[1]!)).toBe(132) + expect(visibleWidth(lines.at(-1)!)).toBe(132) + }) + + it('keeps every colored table row aligned to the same visible width', () => { + const originalLevel = chalk.level + chalk.level = 1 + try { + const rows: ModelReportRow[] = [ + row({ category: 'coding', costUSD: 978.89, inputTokens: 46_300, outputTokens: 3_700_000, cacheWriteTokens: 16_300_000, cacheReadTokens: 1_569_800_000, totalTokens: 1_589_800_000 }), + row({ category: 'delegation', costUSD: 357.0, inputTokens: 44_200, outputTokens: 1_900_000, cacheWriteTokens: 9_400_000, cacheReadTokens: 499_600_000, totalTokens: 511_000_000 }), + row({ category: 'exploration', costUSD: 324.86, inputTokens: 96_800, outputTokens: 1_600_000, cacheWriteTokens: 16_600_000, cacheReadTokens: 359_400_000, totalTokens: 377_800_000 }), + ] + const out = renderTable(rows, { byTask: true, terminalWidth: 160 }) + const widths = out.split('\n').map(visibleWidth) + expect(new Set(widths)).toEqual(new Set([160])) + } finally { + chalk.level = originalLevel + } + }) + + it('can render compact tables when fullWidth is disabled', () => { + const rows: ModelReportRow[] = [ + row({ category: 'coding', costUSD: 1.0, inputTokens: 46_300, outputTokens: 3_700_000, totalTokens: 1_589_800_000 }), + ] + const out = renderTable(rows, { byTask: true, showTotals: false, terminalWidth: 160, fullWidth: false }) + expect(visibleWidth(out.split('\n')[0]!)).toBeLessThan(160) + }) + + it('emits a footer totals row by default and suppresses it under showTotals=false', () => { + const rows: ModelReportRow[] = [row({ costUSD: 1.0, inputTokens: 100, totalTokens: 100 })] + expect(renderTable(rows, { showTotals: true })).toContain('Total') + expect(renderTable(rows, { showTotals: false })).not.toMatch(/^\s*Total/m) + }) +}) + +describe('renderMarkdown', () => { + it('produces a GitHub-flavored markdown table with right-aligned numeric columns', () => { + const rows: ModelReportRow[] = [ + { + provider: 'claude', + providerDisplayName: 'Claude', + model: 'claude-sonnet-4-6', + modelDisplayName: 'Sonnet 4.6', + category: null, + topCategory: 'feature', + topCategoryShare: 0.6, + inputTokens: 100, + outputTokens: 50, + cacheWriteTokens: 0, + cacheReadTokens: 0, + totalTokens: 150, + costUSD: 1.5, + calls: 1, + }, + ] + const md = renderMarkdown(rows, { showTotals: false }) + const lines = md.split('\n') + expect(lines[0]).toBe('| Provider | Model | Top Task | Input | Output | Cache Write | Cache Read | Total | Cost |') + expect(lines[1]).toBe('| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |') + expect(lines[2]).toContain('| Claude |') + expect(lines[2]).toContain('`Sonnet 4.6`') + expect(lines[2]).toContain('Feature Dev (60%)') + }) + + it('escapes pipe characters in provider/model names', () => { + const rows: ModelReportRow[] = [ + { + provider: 'odd', + providerDisplayName: 'A|B', + model: 'm|n', + modelDisplayName: 'M|N', + category: null, + topCategory: 'feature', + topCategoryShare: 1, + inputTokens: 0, + outputTokens: 0, + cacheWriteTokens: 0, + cacheReadTokens: 0, + totalTokens: 0, + costUSD: 0, + calls: 0, + }, + ] + const md = renderMarkdown(rows, { showTotals: false }) + expect(md).toContain('A\\|B') + expect(md).toContain('M\\|N') + }) + + it('emits a bold totals row when showTotals is true', () => { + const rows: ModelReportRow[] = [ + { + provider: 'p', + providerDisplayName: 'P', + model: 'm', + modelDisplayName: 'M', + category: null, + topCategory: 'feature', + topCategoryShare: 1, + inputTokens: 100, + outputTokens: 50, + cacheWriteTokens: 0, + cacheReadTokens: 0, + totalTokens: 150, + costUSD: 1.5, + calls: 1, + }, + ] + const md = renderMarkdown(rows) + expect(md).toContain('**Total**') + }) +}) + +describe('renderJson', () => { + it('emits a JSON array with the documented field shape', () => { + const rows: ModelReportRow[] = [ + { + provider: 'claude', + providerDisplayName: 'Claude', + model: 'claude-sonnet-4-6', + modelDisplayName: 'Sonnet 4.6', + category: null, + topCategory: 'feature', + topCategoryCost: 6.0, + topCategoryShare: 0.6, + inputTokens: 100, + outputTokens: 50, + cacheWriteTokens: 0, + cacheReadTokens: 0, + totalTokens: 150, + costUSD: 1.5, + calls: 1, + }, + ] + const parsed = JSON.parse(renderJson(rows)) as Array> + expect(parsed).toHaveLength(1) + expect(parsed[0]).toMatchObject({ + provider: 'claude', + model: 'claude-sonnet-4-6', + modelDisplayName: 'Sonnet 4.6', + topCategory: 'feature', + inputTokens: 100, + outputTokens: 50, + totalTokens: 150, + calls: 1, + }) + }) +}) + +describe('renderCsv', () => { + it('produces a header row followed by one row per ModelReportRow', () => { + const rows: ModelReportRow[] = [ + { + provider: 'claude', + providerDisplayName: 'Claude', + model: 'claude-sonnet-4-6', + modelDisplayName: 'Sonnet 4.6', + category: null, + topCategory: 'feature', + topCategoryShare: 0.6, + inputTokens: 100, + outputTokens: 50, + cacheWriteTokens: 0, + cacheReadTokens: 0, + totalTokens: 150, + costUSD: 1.5, + calls: 1, + }, + ] + const csv = renderCsv(rows) + const lines = csv.split('\n') + expect(lines[0]).toBe('provider,model,top_task,top_task_share,input_tokens,output_tokens,cache_write_tokens,cache_read_tokens,total_tokens,calls,cost_usd') + expect(lines[1]).toBe('Claude,Sonnet 4.6,Feature Dev,0.6000,100,50,0,0,150,1,1.500000') + }) + + it('escapes commas in provider/model cells', () => { + const rows: ModelReportRow[] = [ + { + provider: 'weird', + providerDisplayName: 'Weird, Co.', + model: 'm', + modelDisplayName: 'M', + category: null, + topCategory: 'feature', + topCategoryShare: 1.0, + inputTokens: 0, + outputTokens: 0, + cacheWriteTokens: 0, + cacheReadTokens: 0, + totalTokens: 0, + costUSD: 0, + calls: 0, + }, + ] + const csv = renderCsv(rows) + expect(csv.split('\n')[1]).toContain('"Weird, Co."') + }) +})