diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a912da6..d31565d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,24 @@ ## Unreleased +### Added (CLI) +- **Tooling breakdowns in dashboard and menubar.** New panels showing core + tools, MCP servers, and shell command usage per session and across periods. +- **File-aware retry detection with typed ToolCall.** One-shot rate now tracks + which file was edited, so editing file A then file B after a shell step no + longer counts as a retry. Claude and Codex extract file paths from tool + inputs; Codex also parses `patch_apply_end` changes and JSON-encoded + `function_call` arguments. Providers without file path data fall back to + tool-name-based detection. + +### Fixed (CLI) +- **Codex 100% one-shot rate.** Codex function_call arguments are JSON strings, + not objects, and `patch_apply_end` stores file paths in `changes` object keys. + Both are now parsed correctly. +- **Claude toolSequence missing from session cache.** `apiCallToCachedCall` was + not forwarding the `toolSequence` field, so all cached Claude sessions lost + their tool ordering data. + ## 0.9.10 - 2026-05-20 ### Added (CLI) diff --git a/README.md b/README.md index 234c5d5f..64ddd5d2 100644 --- a/README.md +++ b/README.md @@ -184,7 +184,7 @@ Daily cost chart, per-project, per-model (Opus, Sonnet, Haiku, GPT-5, GPT-4o, Ge ### One-Shot Rate -For categories that involve code edits, CodeBurn detects edit/test/fix retry cycles (Edit, Bash, Edit patterns). The one-shot column shows the percentage of edit turns that succeeded without retries. Coding at 90% means the AI got it right first try 9 out of 10 times. +For categories that involve code edits, CodeBurn tracks file-aware retry cycles. A retry is when the same file is re-edited after a shell command in between (Edit foo.ts, Bash, Edit foo.ts). Editing different files across shell steps is not a retry. The one-shot column shows the percentage of edit turns that succeeded without retries. Coding at 90% means the AI got it right first try 9 out of 10 times. File-level tracking is available for Claude, Codex, and Goose; other providers fall back to tool-name-based detection. ### Pricing diff --git a/src/classifier.ts b/src/classifier.ts index b0d97bd6..ca7854b0 100644 --- a/src/classifier.ts +++ b/src/classifier.ts @@ -1,4 +1,4 @@ -import type { ClassifiedTurn, ParsedTurn, TaskCategory } from './types.js' +import type { ClassifiedTurn, ParsedTurn, TaskCategory, ToolCall } from './types.js' const TEST_PATTERNS = /\b(test|pytest|vitest|jest|mocha|spec|coverage|npm\s+test|npx\s+vitest|npx\s+jest)\b/i const GIT_PATTERNS = /\bgit\s+(push|pull|commit|merge|rebase|checkout|branch|stash|log|diff|status|add|reset|cherry-pick|tag)\b/i @@ -15,7 +15,7 @@ const FILE_PATTERNS = /\.(py|js|ts|tsx|jsx|json|yaml|yml|toml|sql|sh|go|rs|java| const SCRIPT_PATTERNS = /\b(run\s+\S+\.\w+|execute|scrip?t|curl|api\s+\S+|endpoint|request\s+url|fetch\s+\S+|query|database|db\s+\S+)\b/i const URL_PATTERN = /https?:\/\/\S+/i -const EDIT_TOOLS = new Set(['Edit', 'Write', 'FileEditTool', 'FileWriteTool', 'NotebookEdit', 'cursor:edit']) +export const EDIT_TOOLS = new Set(['Edit', 'Write', 'FileEditTool', 'FileWriteTool', 'NotebookEdit', 'cursor:edit']) const READ_TOOLS = new Set(['Read', 'Grep', 'Glob', 'FileReadTool', 'GrepTool', 'GlobTool']) export const BASH_TOOLS = new Set(['Bash', 'BashTool', 'PowerShellTool']) const TASK_TOOLS = new Set(['TaskCreate', 'TaskUpdate', 'TaskGet', 'TaskList', 'TaskOutput', 'TaskStop', 'TodoWrite']) @@ -154,32 +154,34 @@ function classifyConversation(userMessage: string): TaskCategory { } function countRetries(turn: ParsedTurn): number { - const steps: string[][] = [] + const steps: ToolCall[][] = [] for (const call of turn.assistantCalls) { if (call.toolSequence && call.toolSequence.length > 0) { steps.push(...call.toolSequence) } else if (call.tools.length > 0) { - steps.push(call.tools) + steps.push(call.tools.map(t => ({ tool: t }))) } } - let sawEditBeforeBash = false - let sawBashAfterEdit = false + const lastEditStep = new Map() + let lastVerifyStep = -1 let retries = 0 - for (const tools of steps) { - const hasEdit = tools.some(t => EDIT_TOOLS.has(t)) - const hasBash = tools.some(t => BASH_TOOLS.has(t)) - - if (hasEdit) { - if (sawBashAfterEdit) retries++ - sawEditBeforeBash = true - sawBashAfterEdit = false + steps.forEach((step, i) => { + for (const call of step) { + if (BASH_TOOLS.has(call.tool)) { + lastVerifyStep = i + } + if (EDIT_TOOLS.has(call.tool)) { + const fileKey = call.file ?? '__no_file__' + const prevStep = lastEditStep.get(fileKey) + if (prevStep !== undefined && lastVerifyStep > prevStep && lastVerifyStep < i) { + retries++ + } + lastEditStep.set(fileKey, i) + } } - if (hasBash && sawEditBeforeBash) { - sawBashAfterEdit = true - } - } + }) return retries } diff --git a/src/codex-cache.ts b/src/codex-cache.ts index d408cb59..72708c4b 100644 --- a/src/codex-cache.ts +++ b/src/codex-cache.ts @@ -6,7 +6,7 @@ import { homedir } from 'os' import type { ParsedProviderCall } from './providers/types.js' -const CODEX_CACHE_VERSION = 1 +const CODEX_CACHE_VERSION = 3 const CACHE_FILE = 'codex-results.json' type FileFingerprint = { mtimeMs: number; sizeBytes: number } diff --git a/src/parser.ts b/src/parser.ts index bf0fd7b6..62e82685 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -31,9 +31,10 @@ import type { ProjectSummary, SessionSummary, TokenUsage, + ToolCall, ToolUseBlock, } from './types.js' -import { classifyTurn, BASH_TOOLS } from './classifier.js' +import { classifyTurn, BASH_TOOLS, EDIT_TOOLS } from './classifier.js' import { extractBashCommands } from './bash-utils.js' function unsanitizePath(dirName: string): string { @@ -263,7 +264,7 @@ function extractLargeToolBlocks(source: string, contentBounds: JsonValueBounds | const skillName = readJsonString(source, findObjectFieldValue(source, inputBounds.start, inputBounds.end, 'name'), 200) if (skill !== undefined) input['skill'] = skill if (skillName !== undefined) input['name'] = skillName - } else if (name === 'Read' || name === 'FileReadTool') { + } else if (name === 'Read' || name === 'FileReadTool' || EDIT_TOOLS.has(name)) { const filePath = readJsonString(source, findObjectFieldValue(source, inputBounds.start, inputBounds.end, 'file_path'), BASH_COMMAND_CAP) if (filePath !== undefined) input['file_path'] = filePath } else if (name === 'Agent' || name === 'Task') { @@ -608,7 +609,7 @@ function extractLargeToolBlocksBuffer(source: Buffer, contentBounds: BufferJsonV const skillName = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, inputBounds.start, inputBounds.end, 'name'), 200) if (skill !== undefined) input['skill'] = skill if (skillName !== undefined) input['name'] = skillName - } else if (name === 'Read' || name === 'FileReadTool') { + } else if (name === 'Read' || name === 'FileReadTool' || EDIT_TOOLS.has(name)) { const filePath = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, inputBounds.start, inputBounds.end, 'file_path'), BASH_COMMAND_CAP) if (filePath !== undefined) input['file_path'] = filePath } else if (name === 'Agent' || name === 'Task') { @@ -839,7 +840,7 @@ export function compactEntry(raw: JournalEntry): JournalEntry { const ri = (tb.input ?? {}) as Record if (typeof ri['skill'] === 'string') input['skill'] = (ri['skill'] as string).slice(0, 200) if (typeof ri['name'] === 'string') input['name'] = (ri['name'] as string).slice(0, 200) - } else if (tb.name === 'Read' || tb.name === 'FileReadTool') { + } else if (tb.name === 'Read' || tb.name === 'FileReadTool' || EDIT_TOOLS.has(tb.name)) { const ri = (tb.input ?? {}) as Record if (typeof ri['file_path'] === 'string') input['file_path'] = (ri['file_path'] as string).slice(0, BASH_COMMAND_CAP) } else if (tb.name === 'Agent' || tb.name === 'Task') { @@ -1006,6 +1007,16 @@ function parseApiCall(entry: JournalEntry): ParsedApiCall | null { const bashCmds = extractBashCommandsFromContent(msg.content ?? []) + const toolSeq: ToolCall[][] = (msg.content ?? []) + .filter((b): b is ToolUseBlock => b.type === 'tool_use') + .map(b => { + const call: ToolCall = { tool: b.name } + const inp = (b.input ?? {}) as Record + if (typeof inp['file_path'] === 'string') call.file = inp['file_path'] as string + if (typeof inp['command'] === 'string') call.command = inp['command'] as string + return [call] + }) + return { provider: 'claude', model: msg.model, @@ -1022,6 +1033,7 @@ function parseApiCall(entry: JournalEntry): ParsedApiCall | null { bashCommands: bashCmds, deduplicationKey: msg.id ?? `claude:${entry.timestamp}`, cacheCreationOneHourTokens: cacheCreation.oneHourTokens || undefined, + toolSequence: toolSeq.length > 0 ? toolSeq : undefined, } } @@ -1578,6 +1590,7 @@ function apiCallToCachedCall(call: ParsedApiCall): CachedCall { skills: call.skills, subagentTypes: call.subagentTypes, deduplicationKey: call.deduplicationKey, + toolSequence: call.toolSequence, } } diff --git a/src/providers/codex.ts b/src/providers/codex.ts index d68273a2..3e1f3274 100644 --- a/src/providers/codex.ts +++ b/src/providers/codex.ts @@ -7,6 +7,7 @@ import { homedir } from 'os' import { readSessionLines } from '../fs-utils.js' import { calculateCost } from '../models.js' import { readCachedCodexResults, writeCachedCodexResults, getCachedCodexProject, fingerprintFile } from '../codex-cache.js' +import type { ToolCall } from '../types.js' import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' const modelDisplayNames: Record = { @@ -331,9 +332,12 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars let prevOutput = 0 let prevReasoning = 0 let pendingTools: string[] = [] + let pendingToolSequence: ToolCall[][] = [] let pendingUserMessage = '' let pendingOutputChars = 0 let estCounter = 0 + let turnCounter = 0 + let currentTurnId = `${sessionId}:t0` let sawAnyLine = false const results: ParsedProviderCall[] = [] @@ -364,12 +368,35 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars if (entry.type === 'response_item' && entry.payload?.type === 'function_call') { const rawName = entry.payload.name ?? '' - pendingTools.push(toolNameMap[rawName] ?? rawName) + const mapped = toolNameMap[rawName] ?? rawName + pendingTools.push(mapped) + const call: ToolCall = { tool: mapped } + const rawArgs = (entry.payload as Record)['arguments'] + const args = typeof rawArgs === 'string' + ? (() => { try { return JSON.parse(rawArgs) as Record } catch { return null } })() + : typeof rawArgs === 'object' && rawArgs ? rawArgs as Record : null + if (args) { + const fp = args['file_path'] ?? args['path'] + if (typeof fp === 'string') call.file = fp + const cmd = args['command'] ?? args['cmd'] + if (typeof cmd === 'string') call.command = cmd + } + pendingToolSequence.push([call]) continue } if (entry.type === 'event_msg' && entry.payload?.type === 'patch_apply_end') { pendingTools.push('Edit') + const p = entry.payload as Record + const changes = p['changes'] + const filePaths = typeof changes === 'object' && changes ? Object.keys(changes as object) : [] + if (filePaths.length > 0) { + for (const fp of filePaths) { + pendingToolSequence.push([{ tool: 'Edit', file: fp }]) + } + } else { + pendingToolSequence.push([{ tool: 'Edit' }]) + } continue } @@ -378,7 +405,10 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars .filter(c => c.type === 'input_text') .map(c => c.text ?? '') .filter(Boolean) - if (texts.length > 0) pendingUserMessage = texts.join(' ').slice(0, 500) + if (texts.length > 0) { + pendingUserMessage = texts.join(' ').slice(0, 500) + currentTurnId = `${sessionId}:t${++turnCounter}` + } continue } @@ -406,7 +436,7 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars const timestamp = entry.timestamp ?? '' const dedupKey = `codex:${sessionId}:${timestamp}:est${estCounter++}` - if (seenKeys.has(dedupKey)) { pendingTools = []; pendingUserMessage = ''; pendingOutputChars = 0; continue } + if (seenKeys.has(dedupKey)) { pendingTools = []; pendingToolSequence = []; pendingUserMessage = ''; pendingOutputChars = 0; continue } seenKeys.add(dedupKey) const costUSD = calculateCost(model, estInput, estOutput, 0, 0, 0) @@ -428,11 +458,14 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars timestamp, speed: 'standard', deduplicationKey: dedupKey, + turnId: currentTurnId, + toolSequence: pendingToolSequence.length > 0 ? pendingToolSequence : undefined, userMessage: pendingUserMessage, sessionId, }) pendingTools = [] + pendingToolSequence = [] pendingUserMessage = '' pendingOutputChars = 0 continue @@ -521,11 +554,14 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars timestamp, speed: 'standard', deduplicationKey: dedupKey, + turnId: currentTurnId, + toolSequence: pendingToolSequence.length > 0 ? pendingToolSequence : undefined, userMessage: pendingUserMessage, sessionId, }) pendingTools = [] + pendingToolSequence = [] pendingUserMessage = '' pendingOutputChars = 0 } diff --git a/src/providers/goose.ts b/src/providers/goose.ts index 6a10d1df..71097097 100644 --- a/src/providers/goose.ts +++ b/src/providers/goose.ts @@ -4,6 +4,7 @@ import { homedir, platform } from 'os' import { calculateCost, getShortModelName } from '../models.js' import { extractBashCommands } from '../bash-utils.js' import { isSqliteAvailable, getSqliteLoadError, openDatabase, blobToText, type SqliteDatabase } from '../sqlite.js' +import type { ToolCall } from '../types.js' import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' type SessionRow = { @@ -80,11 +81,11 @@ function parseModelConfig(raw: string | null): ModelConfig { } } -function extractToolsFromMessages(db: SqliteDatabase, sessionId: string): { tools: string[]; bashCommands: string[]; toolSequence: string[][] } { +function extractToolsFromMessages(db: SqliteDatabase, sessionId: string): { tools: string[]; bashCommands: string[]; toolSequence: ToolCall[][] } { const tools: string[] = [] const bashCommands: string[] = [] const seen = new Set() - const toolSequence: string[][] = [] + const toolSequence: ToolCall[][] = [] try { const rows = db.query<{ content_json: Uint8Array | string }>( @@ -99,7 +100,7 @@ function extractToolsFromMessages(db: SqliteDatabase, sessionId: string): { tool } catch { continue } - const msgTools: string[] = [] + const msgCalls: ToolCall[] = [] for (const item of items) { if (item.type !== 'toolRequest') continue const rawName = item.toolCall?.value?.name ?? '' @@ -109,7 +110,15 @@ function extractToolsFromMessages(db: SqliteDatabase, sessionId: string): { tool seen.add(mapped) tools.push(mapped) } - msgTools.push(mapped) + const call: ToolCall = { tool: mapped } + const args = item.toolCall?.value?.arguments + if (args && typeof args === 'object') { + const fp = (args as Record)['file_path'] + if (typeof fp === 'string') call.file = fp + const cmd = (args as Record)['command'] + if (typeof cmd === 'string') call.command = cmd + } + msgCalls.push(call) if (mapped === 'Bash') { const cmd = item.toolCall?.value?.arguments?.command if (typeof cmd === 'string') { @@ -119,7 +128,7 @@ function extractToolsFromMessages(db: SqliteDatabase, sessionId: string): { tool } } } - if (msgTools.length > 0) toolSequence.push(msgTools) + if (msgCalls.length > 0) toolSequence.push(msgCalls) } } catch { /* best-effort */ } diff --git a/src/providers/kiro.ts b/src/providers/kiro.ts index 7d616d73..cc9445db 100644 --- a/src/providers/kiro.ts +++ b/src/providers/kiro.ts @@ -4,6 +4,7 @@ import { homedir } from 'os' import { readSessionFile } from '../fs-utils.js' import { calculateCost } from '../models.js' +import type { ToolCall } from '../types.js' import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' const CHARS_PER_TOKEN = 4 @@ -86,7 +87,7 @@ function parseChatFile(data: KiroChatFile, sessionId: string, project: string, s let pendingUserMessage = '' const allTools: string[] = [] - const toolSequence: string[][] = [] + const toolSequence: ToolCall[][] = [] for (const msg of chat) { if (msg.role === 'human') { @@ -96,7 +97,7 @@ function parseChatFile(data: KiroChatFile, sessionId: string, project: string, s if (msg.role === 'bot') { const msgTools = extractToolNames(msg.content) allTools.push(...msgTools) - if (msgTools.length > 0) toolSequence.push(msgTools) + if (msgTools.length > 0) toolSequence.push(msgTools.map(t => ({ tool: t }))) } } diff --git a/src/providers/types.ts b/src/providers/types.ts index 8e0c9372..d6e3230e 100644 --- a/src/providers/types.ts +++ b/src/providers/types.ts @@ -1,3 +1,5 @@ +import type { ToolCall } from '../types.js' + export type SessionSource = { path: string project: string @@ -26,7 +28,7 @@ export type ParsedProviderCall = { speed: 'standard' | 'fast' deduplicationKey: string turnId?: string - toolSequence?: string[][] + toolSequence?: ToolCall[][] userMessage: string sessionId: string project?: string diff --git a/src/session-cache.ts b/src/session-cache.ts index 7a860e3e..f82a7a17 100644 --- a/src/session-cache.ts +++ b/src/session-cache.ts @@ -4,6 +4,8 @@ import { createHash, randomBytes } from 'crypto' import { join } from 'path' import { homedir } from 'os' +import type { ToolCall } from './types.js' + // ── Types ────────────────────────────────────────────────────────────── export type CachedUsage = { @@ -31,7 +33,7 @@ export type CachedCall = { deduplicationKey: string project?: string projectPath?: string - toolSequence?: string[][] + toolSequence?: ToolCall[][] } export type CachedTurn = { @@ -68,7 +70,7 @@ export type SessionCache = { // ── Constants ────────────────────────────────────────────────────────── -export const CACHE_VERSION = 2 +export const CACHE_VERSION = 3 const CACHE_FILE = 'session-cache.json' const TEMP_FILE_MAX_AGE_MS = 5 * 60 * 1000 @@ -128,6 +130,18 @@ function isOptionalNum(v: unknown): boolean { return v === undefined || isNum(v) } +function isToolCall(v: unknown): boolean { + if (!v || typeof v !== 'object') return false + const o = v as Record + return typeof o['tool'] === 'string' + && isOptionalString(o['file']) + && isOptionalString(o['command']) +} + +function isToolCallArray(v: unknown): boolean { + return Array.isArray(v) && (v as unknown[]).every(isToolCall) +} + function validateFingerprint(fp: unknown): fp is FileFingerprint { if (!fp || typeof fp !== 'object') return false const f = fp as Record @@ -157,7 +171,7 @@ function validateCall(c: unknown): c is CachedCall { && isStringArray(o['skills']) && isOptionalString(o['project']) && isOptionalString(o['projectPath']) - && (o['toolSequence'] === undefined || (Array.isArray(o['toolSequence']) && (o['toolSequence'] as unknown[]).every(s => isStringArray(s)))) + && (o['toolSequence'] === undefined || (Array.isArray(o['toolSequence']) && (o['toolSequence'] as unknown[]).every(s => isToolCallArray(s)))) && validateUsage(o['usage']) } diff --git a/src/types.ts b/src/types.ts index bb0a204e..f35551ba 100644 --- a/src/types.ts +++ b/src/types.ts @@ -85,7 +85,13 @@ export type ParsedApiCall = { bashCommands: string[] deduplicationKey: string cacheCreationOneHourTokens?: number - toolSequence?: string[][] + toolSequence?: ToolCall[][] +} + +export type ToolCall = { + tool: string + file?: string + command?: string } export type TaskCategory = diff --git a/tests/classifier.test.ts b/tests/classifier.test.ts index 42c34417..3ef82b5d 100644 --- a/tests/classifier.test.ts +++ b/tests/classifier.test.ts @@ -165,7 +165,7 @@ describe('classifyTurn — retry detection via toolSequence', () => { it('detects retries from toolSequence on a single call (Kiro/Goose-style)', () => { const call = makeCall({ tools: ['Edit', 'Bash'] }) - call.toolSequence = [['Edit'], ['Bash'], ['Edit']] + call.toolSequence = [[{ tool: 'Edit' }], [{ tool: 'Bash' }], [{ tool: 'Edit' }]] const turn = makeTurn([call], 'fix the build') const c = classifyTurn(turn) expect(c.retries).toBe(1) @@ -180,7 +180,7 @@ describe('classifyTurn — retry detection via toolSequence', () => { it('counts multiple retries from toolSequence', () => { const call = makeCall({ tools: ['Edit', 'Bash'] }) - call.toolSequence = [['Edit'], ['Bash'], ['Edit'], ['Bash'], ['Edit']] + call.toolSequence = [[{ tool: 'Edit' }], [{ tool: 'Bash' }], [{ tool: 'Edit' }], [{ tool: 'Bash' }], [{ tool: 'Edit' }]] const turn = makeTurn([call], 'fix the build') const c = classifyTurn(turn) expect(c.retries).toBe(2) @@ -188,7 +188,7 @@ describe('classifyTurn — retry detection via toolSequence', () => { it('ignores toolSequence with only one step', () => { const call = makeCall({ tools: ['Edit', 'Bash'] }) - call.toolSequence = [['Edit', 'Bash']] + call.toolSequence = [[{ tool: 'Edit' }, { tool: 'Bash' }]] const turn = makeTurn([call], 'fix the build') const c = classifyTurn(turn) expect(c.retries).toBe(0)