diff --git a/src/compare-stats.ts b/src/compare-stats.ts index 5e67781..1f7d46b 100644 --- a/src/compare-stats.ts +++ b/src/compare-stats.ts @@ -100,6 +100,18 @@ export type WorkingStyleRow = { formatFn: ComparisonRow['formatFn'] } +export type CompareJsonReport = { + period: { + label: string + provider: string + } + modelA: ModelStats + modelB: ModelStats + metrics: ComparisonRow[] + categories: CategoryComparison[] + workingStyle: WorkingStyleRow[] +} + type MetricDef = { section: string label: string @@ -277,6 +289,27 @@ export function computeWorkingStyle(projects: ProjectSummary[], modelA: string, ] } +export function buildCompareJson( + projects: ProjectSummary[], + modelA: ModelStats, + modelB: ModelStats, + label: string, + provider: string, +): CompareJsonReport { + return { + period: { label, provider }, + modelA, + modelB, + metrics: computeComparison(modelA, modelB), + categories: computeCategoryComparison(projects, modelA.model, modelB.model), + workingStyle: computeWorkingStyle(projects, modelA.model, modelB.model), + } +} + +export function renderCompareJson(report: CompareJsonReport): string { + return JSON.stringify(report, null, 2) +} + const SELF_CORRECTION_PATTERNS = [ /\bmy mistake\b/i, /\bmy bad\b/i, diff --git a/src/main.ts b/src/main.ts index e26d222..b63961f 100644 --- a/src/main.ts +++ b/src/main.ts @@ -4,7 +4,7 @@ import { installMenubarApp } from './menubar-installer.js' import { exportCsv, exportJson, type PeriodExport } from './export.js' import { findUnpricedModels, loadPricing, setModelAliases, setPriceOverrides, setLocalModelSavings, setProxyPaths, normalizeProxyPath } from './models.js' import { parseAllSessions, filterProjectsByName, filterProjectsByDateRange, clearSessionCache } from './parser.js' -import { allProviderNames } from './providers/index.js' +import { allProviderNames, getAllProviders } from './providers/index.js' import { convertCost } from './currency.js' import { renderStatusBar } from './format.js' import { toDateString } from './daily-cache.js' @@ -1496,10 +1496,51 @@ program .description('Compare two AI models side-by-side') .option('-p, --period ', 'Analysis period: today, week, 30days, month, all', 'all') .option('--provider ', 'Filter by provider (e.g. claude, gemini, cursor, copilot)', 'all') + .option('--format ', 'Output format: tui, json', 'tui') + .option('--model-a ', 'First model to compare') + .option('--model-b ', 'Second model to compare') .action(async (opts) => { assertProvider(opts.provider, 'compare') + assertFormat(opts.format, ['tui', 'json'], 'compare') await loadPricing() - const { range } = getDateRange(opts.period) + const { range, label } = getDateRange(opts.period) + if (opts.format === 'json') { + const { aggregateModelStats, buildCompareJson, renderCompareJson, scanSelfCorrections } = await import('./compare-stats.js') + const projects = await parseAllSessions(range, opts.provider) + const models = aggregateModelStats(projects) + + const providers = await getAllProviders() + const dirs: string[] = [] + for (const provider of providers) { + const sessions = await provider.discoverSessions() + for (const session of sessions) dirs.push(session.path) + } + const corrections = await scanSelfCorrections(dirs) + for (const model of models) { + model.selfCorrections = corrections.get(model.model) ?? 0 + } + + if (!opts.modelA && !opts.modelB) { + process.stdout.write(JSON.stringify(models, null, 2) + '\n') + return + } + if (!opts.modelA || !opts.modelB) { + process.stderr.write('codeburn compare: --model-a and --model-b must be provided together.\n') + process.exit(1) + } + const modelA = models.find(model => model.model === opts.modelA) + const modelB = models.find(model => model.model === opts.modelB) + if (!modelA) { + process.stderr.write(`codeburn compare: model not found: "${opts.modelA}".\n`) + process.exit(1) + } + if (!modelB) { + process.stderr.write(`codeburn compare: model not found: "${opts.modelB}".\n`) + process.exit(1) + } + process.stdout.write(renderCompareJson(buildCompareJson(projects, modelA, modelB, label, opts.provider)) + '\n') + return + } await renderCompare(range, opts.provider) }) @@ -1600,6 +1641,37 @@ program } }) +program + .command('sessions') + .description('Full per-session usage report') + .option('-p, --period ', 'Analysis period: today, week, 30days, month, all', '30days') + .option('--from ', 'Custom range start (YYYY-MM-DD)') + .option('--to ', 'Custom range end (YYYY-MM-DD)') + .option('--provider ', 'Filter by provider (e.g. claude, codex, cursor)', 'all') + .option('--format ', 'Output format: table, json', 'table') + .action(async (opts) => { + assertProvider(opts.provider, 'sessions') + assertFormat(opts.format, ['table', 'json'], 'sessions') + const { aggregateSessions, renderJson, renderTable } = await import('./sessions-report.js') + await loadPricing() + + let range + if (opts.from || opts.to) { + const customRange = parseDateRangeFlags(opts.from, opts.to) + if (!customRange) { + process.stderr.write('codeburn: --from and --to must be valid YYYY-MM-DD dates\n') + process.exit(1) + } + range = customRange + } else { + range = getDateRange(opts.period).range + } + + const rows = aggregateSessions(await parseAllSessions(range, opts.provider)) + const output = opts.format === 'json' ? renderJson(rows) : renderTable(rows) + process.stdout.write(output + '\n') + }) + program .command('yield') .description('Track which AI spend shipped to main vs reverted/abandoned (experimental)') diff --git a/src/sessions-report.ts b/src/sessions-report.ts new file mode 100644 index 0000000..4484ec3 --- /dev/null +++ b/src/sessions-report.ts @@ -0,0 +1,81 @@ +import type { ProjectSummary, SessionSummary } from './types.js' + +export type SessionRow = { + sessionId: string + project: string + provider: string + models: string[] + cost: number + savingsUSD: number + calls: number + turns: number + inputTokens: number + outputTokens: number + cacheReadTokens: number + cacheWriteTokens: number + startedAt: string + endedAt: string + durationMs: number +} + +function inferProvider(session: SessionSummary): string { + for (const turn of session.turns) { + const provider = turn.assistantCalls[0]?.provider + if (provider) return provider + } + + const models = Object.keys(session.modelBreakdown) + const model = models[0]?.toLowerCase() ?? '' + if (model.startsWith('claude')) return 'claude' + if (model.startsWith('gpt-') || model.startsWith('o1') || model.startsWith('o3') || model.startsWith('o4')) return 'codex' + if (model.startsWith('gemini')) return 'gemini' + if (model.includes('/')) return model.split('/', 1)[0] || 'unknown' + return 'unknown' +} + +function durationMs(startedAt: string, endedAt: string): number { + const duration = new Date(endedAt).getTime() - new Date(startedAt).getTime() + return Number.isFinite(duration) ? duration : 0 +} + +export function aggregateSessions(projects: ProjectSummary[]): SessionRow[] { + return projects.flatMap(project => project.sessions.map(session => ({ + sessionId: session.sessionId, + project: session.project || project.project, + provider: inferProvider(session), + models: Object.keys(session.modelBreakdown), + cost: session.totalCostUSD, + savingsUSD: session.totalSavingsUSD, + calls: session.apiCalls, + turns: session.turns.length, + inputTokens: session.totalInputTokens, + outputTokens: session.totalOutputTokens, + cacheReadTokens: session.totalCacheReadTokens, + cacheWriteTokens: session.totalCacheWriteTokens, + startedAt: session.firstTimestamp, + endedAt: session.lastTimestamp, + durationMs: durationMs(session.firstTimestamp, session.lastTimestamp), + }))) +} + +export function renderJson(rows: SessionRow[]): string { + return JSON.stringify(rows, null, 2) +} + +export function renderTable(rows: SessionRow[]): string { + const headers = ['SESSION', 'PROJECT', 'PROVIDER', 'MODELS', 'COST', 'SAVED', 'CALLS', 'TURNS', 'STARTED'] + const values = rows.map(row => [ + row.sessionId, + row.project, + row.provider, + row.models.join(', '), + `$${row.cost.toFixed(2)}`, + `$${row.savingsUSD.toFixed(2)}`, + String(row.calls), + String(row.turns), + row.startedAt, + ]) + const widths = headers.map((header, i) => Math.max(header.length, ...values.map(row => row[i]!.length))) + const format = (row: string[]) => row.map((value, i) => value.padEnd(widths[i]!)).join(' ').trimEnd() + return [format(headers), format(widths.map(width => '-'.repeat(width))), ...values.map(format)].join('\n') +} diff --git a/tests/cli-emitters.test.ts b/tests/cli-emitters.test.ts new file mode 100644 index 0000000..5f57448 --- /dev/null +++ b/tests/cli-emitters.test.ts @@ -0,0 +1,99 @@ +import { spawnSync } from 'node:child_process' +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { describe, expect, it } from 'vitest' + +function runCli(args: string[], home: string) { + return spawnSync(process.execPath, ['--import', 'tsx', 'src/cli.ts', ...args], { + cwd: process.cwd(), + env: { ...process.env, HOME: home, CLAUDE_CONFIG_DIR: join(home, '.claude'), TZ: 'UTC' }, + encoding: 'utf-8', + timeout: 30_000, + }) +} + +function userLine(sessionId: string, timestamp: string): string { + return JSON.stringify({ + type: 'user', + sessionId, + timestamp, + message: { role: 'user', content: 'implement feature' }, + }) +} + +function assistantLine(sessionId: string, timestamp: string, messageId: string, model: string): string { + return JSON.stringify({ + type: 'assistant', + sessionId, + timestamp, + message: { + id: messageId, + type: 'message', + role: 'assistant', + model, + content: [{ type: 'text', text: 'done' }], + usage: { input_tokens: 1000, output_tokens: 100 }, + }, + }) +} + +async function makeHome(): Promise { + const home = await mkdtemp(join(tmpdir(), 'codeburn-cli-emitters-')) + const projectDir = join(home, '.claude', 'projects', 'app') + await mkdir(projectDir, { recursive: true }) + await writeFile(join(projectDir, 'session-a.jsonl'), [ + userLine('session-a', '2026-04-10T09:00:00Z'), + assistantLine('session-a', '2026-04-10T09:01:00Z', 'msg-a', 'claude-sonnet-4-5'), + ].join('\n')) + await writeFile(join(projectDir, 'session-b.jsonl'), [ + userLine('session-b', '2026-04-10T10:00:00Z'), + assistantLine('session-b', '2026-04-10T10:01:00Z', 'msg-b', 'claude-opus-4-5'), + ].join('\n')) + return home +} + +describe('CLI JSON emitters', () => { + it('emits compare list mode and full mode as JSON', async () => { + const home = await makeHome() + try { + const listResult = runCli(['compare', '--format', 'json', '--period', 'all', '--provider', 'claude'], home) + expect(listResult.status, listResult.stderr).toBe(0) + const models = JSON.parse(listResult.stdout) as Array<{ model: string; selfCorrections: number }> + expect(models.map(model => model.model)).toEqual(expect.arrayContaining(['claude-sonnet-4-5', 'claude-opus-4-5'])) + expect(models.every(model => typeof model.selfCorrections === 'number')).toBe(true) + + const fullResult = runCli([ + 'compare', '--format', 'json', '--period', 'all', '--provider', 'claude', + '--model-a', 'claude-sonnet-4-5', '--model-b', 'claude-opus-4-5', + ], home) + expect(fullResult.status, fullResult.stderr).toBe(0) + const report = JSON.parse(fullResult.stdout) + expect(Object.keys(report)).toEqual(['period', 'modelA', 'modelB', 'metrics', 'categories', 'workingStyle']) + expect(report.period.provider).toBe('claude') + expect(report.modelA.model).toBe('claude-sonnet-4-5') + expect(report.modelB.model).toBe('claude-opus-4-5') + } finally { + await rm(home, { recursive: true, force: true }) + } + }) + + it('emits flat session rows as JSON', async () => { + const home = await makeHome() + try { + const result = runCli(['sessions', '--format', 'json', '--period', 'all', '--provider', 'claude'], home) + expect(result.status, result.stderr).toBe(0) + const rows = JSON.parse(result.stdout) as Array> + expect(rows).toHaveLength(2) + expect(Object.keys(rows[0]!)).toEqual([ + 'sessionId', 'project', 'provider', 'models', 'cost', 'savingsUSD', 'calls', 'turns', + 'inputTokens', 'outputTokens', 'cacheReadTokens', 'cacheWriteTokens', + 'startedAt', 'endedAt', 'durationMs', + ]) + expect(rows.every(row => row.provider === 'claude')).toBe(true) + } finally { + await rm(home, { recursive: true, force: true }) + } + }) +}) diff --git a/tests/compare-stats.test.ts b/tests/compare-stats.test.ts index 63d3534..ef9ec29 100644 --- a/tests/compare-stats.test.ts +++ b/tests/compare-stats.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, mkdir, rm, writeFile } from 'fs/promises' import { join } from 'path' import { tmpdir } from 'os' import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { aggregateModelStats, computeComparison, computeCategoryComparison, computeWorkingStyle, scanSelfCorrections, type ModelStats } from '../src/compare-stats.js' +import { aggregateModelStats, buildCompareJson, computeComparison, computeCategoryComparison, computeWorkingStyle, renderCompareJson, scanSelfCorrections, type ModelStats } from '../src/compare-stats.js' import type { ProjectSummary, SessionSummary, ClassifiedTurn } from '../src/types.js' function makeTurn(model: string, cost: number, opts: { hasEdits?: boolean; retries?: number; outputTokens?: number; inputTokens?: number; cacheRead?: number; cacheWrite?: number; timestamp?: string; category?: string; hasAgentSpawn?: boolean; hasPlanMode?: boolean; speed?: 'standard' | 'fast'; tools?: string[] } = {}): ClassifiedTurn { @@ -238,6 +238,32 @@ describe('computeComparison', () => { }) }) +describe('compare JSON emitter', () => { + it('builds and renders the full comparison shape', () => { + const project = makeProject([ + makeTurn('model-a', 0.10, { hasEdits: true, retries: 0 }), + makeTurn('model-b', 0.20, { hasEdits: true, retries: 1 }), + ]) + const models = aggregateModelStats([project]) + const modelA = models.find(model => model.model === 'model-a')! + const modelB = models.find(model => model.model === 'model-b')! + modelA.selfCorrections = 2 + modelB.selfCorrections = 1 + + const parsed = JSON.parse(renderCompareJson( + buildCompareJson([project], modelA, modelB, 'All Time', 'all'), + )) + + expect(Object.keys(parsed)).toEqual(['period', 'modelA', 'modelB', 'metrics', 'categories', 'workingStyle']) + expect(parsed.period).toEqual({ label: 'All Time', provider: 'all' }) + expect(parsed.modelA.selfCorrections).toBe(2) + expect(parsed.metrics.some((row: { section: string }) => row.section === 'Performance')).toBe(true) + expect(parsed.metrics.some((row: { section: string }) => row.section === 'Efficiency')).toBe(true) + expect(parsed.categories).toHaveLength(1) + expect(parsed.workingStyle).toHaveLength(4) + }) +}) + function jsonlLine(type: string, model: string, text: string, timestamp = '2026-04-15T10:00:00Z'): string { if (type === 'assistant') { return JSON.stringify({ diff --git a/tests/sessions-report.test.ts b/tests/sessions-report.test.ts new file mode 100644 index 0000000..0100a34 --- /dev/null +++ b/tests/sessions-report.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from 'vitest' + +import { aggregateSessions, renderJson, renderTable } from '../src/sessions-report.js' +import type { ClassifiedTurn, ProjectSummary, SessionSummary } from '../src/types.js' + +function makeProject(): ProjectSummary { + const turn: ClassifiedTurn = { + userMessage: 'build it', + timestamp: '2026-07-10T10:00:00.000Z', + sessionId: 'session-1', + category: 'feature', + retries: 0, + hasEdits: true, + assistantCalls: [{ + provider: 'claude', + model: 'claude-sonnet-4-5', + usage: { + inputTokens: 100, + outputTokens: 20, + cacheCreationInputTokens: 30, + cacheReadInputTokens: 400, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + }, + costUSD: 0.12, + tools: [], + mcpTools: [], + skills: [], + subagentTypes: [], + hasAgentSpawn: false, + hasPlanMode: false, + speed: 'standard', + timestamp: '2026-07-10T10:01:00.000Z', + bashCommands: [], + deduplicationKey: 'call-1', + }], + } + const session: SessionSummary = { + sessionId: 'session-1', + project: 'codeburn', + firstTimestamp: '2026-07-10T10:00:00.000Z', + lastTimestamp: '2026-07-10T10:05:00.000Z', + totalCostUSD: 0.12, + totalSavingsUSD: 0.03, + totalInputTokens: 100, + totalOutputTokens: 20, + totalReasoningTokens: 0, + totalCacheReadTokens: 400, + totalCacheWriteTokens: 30, + apiCalls: 1, + turns: [turn], + modelBreakdown: { + 'claude-sonnet-4-5': { calls: 1, costUSD: 0.12, tokens: turn.assistantCalls[0]!.usage, savingsUSD: 0.03 }, + }, + toolBreakdown: {}, + mcpBreakdown: {}, + bashBreakdown: {}, + categoryBreakdown: {} as SessionSummary['categoryBreakdown'], + skillBreakdown: {}, + subagentBreakdown: {}, + } + return { + project: 'codeburn', + projectPath: '/tmp/codeburn', + sessions: [session], + totalCostUSD: 0.12, + totalSavingsUSD: 0.03, + totalApiCalls: 1, + totalProxiedCostUSD: 0, + } +} + +describe('sessions JSON emitter', () => { + it('flattens SessionSummary fields into the exact JSON row shape', () => { + const rows = aggregateSessions([makeProject()]) + const parsed = JSON.parse(renderJson(rows)) + + expect(parsed).toEqual([{ + sessionId: 'session-1', + project: 'codeburn', + provider: 'claude', + models: ['claude-sonnet-4-5'], + cost: 0.12, + savingsUSD: 0.03, + calls: 1, + turns: 1, + inputTokens: 100, + outputTokens: 20, + cacheReadTokens: 400, + cacheWriteTokens: 30, + startedAt: '2026-07-10T10:00:00.000Z', + endedAt: '2026-07-10T10:05:00.000Z', + durationMs: 300_000, + }]) + }) + + it('renders a simple table', () => { + const output = renderTable(aggregateSessions([makeProject()])) + expect(output).toContain('SESSION') + expect(output).toContain('session-1') + expect(output).toContain('claude-sonnet-4-5') + }) +})