diff --git a/src/classifier.ts b/src/classifier.ts index 44582db8..3599ec82 100644 --- a/src/classifier.ts +++ b/src/classifier.ts @@ -16,7 +16,7 @@ const FILE_PATTERNS = /\.(py|js|ts|tsx|jsx|json|yaml|yml|toml|sql|sh|go|rs|java| const SCRIPT_PATTERNS = /\b(run\s+\S+\.\w+|execute|scrip?t|curl|api\s+\S+|endpoint|request\s+url|fetch\s+\S+|query|database|db\s+\S+)\b/i const URL_PATTERN = /https?:\/\/\S+/i -export const EDIT_TOOLS = new Set(['Edit', 'Write', 'FileEditTool', 'FileWriteTool', 'NotebookEdit', 'cursor:edit']) +export const EDIT_TOOLS = new Set(['Edit', 'Write', 'FileEditTool', 'FileWriteTool', 'NotebookEdit', 'cursor:edit', 'cursor:strreplace', 'cursor:write', 'edit', 'write']) const READ_TOOLS = new Set(['Read', 'Grep', 'Glob', 'FileReadTool', 'GrepTool', 'GlobTool']) export const BASH_TOOLS = new Set(['Bash', 'BashTool', 'PowerShellTool']) const TASK_TOOLS = new Set(['TaskCreate', 'TaskUpdate', 'TaskGet', 'TaskList', 'TaskOutput', 'TaskStop', 'TodoWrite']) diff --git a/src/optimize.ts b/src/optimize.ts index 90cf4dcd..7345f19a 100644 --- a/src/optimize.ts +++ b/src/optimize.ts @@ -7,6 +7,7 @@ import { basename, join } from 'path' import { homedir } from 'os' import { isReadShapedBashCommand } from './bash-utils.js' +import { EDIT_TOOLS } from './classifier.js' import { readSessionLines, readSessionFileSync } from './fs-utils.js' import { discoverAllSessions, providerDisplayName } from './providers/index.js' import { parseJsonlLine, shouldSkipLine } from './parser.js' @@ -3455,15 +3456,26 @@ function formatContextRatio(ratio: number): string { // Use (\s|$|--) instead of \b after commit/push so `git commit-tree` and // `git commit-graph` are not treated as deliveries. The `--` clause keeps -// `git commit --amend` matching as a real delivery command. +// `git commit --amend` matching as a real delivery command. Git's global +// options (`git -C dir commit`, `git -c user.name=x commit`) may sit between +// `git` and the subcommand. const DELIVERY_COMMAND_PATTERNS = [ - /(?:^|[;&|]\s*)git\s+(?:commit|push)(?=\s|$|--)(?![^;&|]*--dry-run)/, - /(?:^|[;&|]\s*)gh\s+pr\s+(?:create|merge)(?=\s|$|--)(?![^;&|]*--dry-run)/, + /(?:^|[;&|(\n]\s*|\b(?:do|then)\s+)git(?:\s+-[Cc]\s+\S+|\s+--[\w-]+(?:=\S+)?)*\s+(?:commit|push)(?=\s|$|--)(?![^;&|\n]*--dry-run)/, + /(?:^|[;&|(\n]\s*|\b(?:do|then)\s+)gh\s+pr\s+(?:create|merge)(?=\s|$|--)(?![^;&|\n]*--dry-run)/, ] +// bashBreakdown keys are base command names (`git`), so delivery is read from +// the full command text each call keeps in its tool sequence. function sessionDeliveryCommand(session: ProjectSummary['sessions'][number]): string | null { - const commands = Object.keys(session.bashBreakdown) - return commands.find(command => DELIVERY_COMMAND_PATTERNS.some(pattern => pattern.test(command))) ?? null + for (const turn of session.turns) { + for (const call of turn.assistantCalls) { + for (const tool of call.toolSequence?.flat() ?? []) { + const command = tool.command + if (command && DELIVERY_COMMAND_PATTERNS.some(pattern => pattern.test(command))) return command + } + } + } + return null } function hasCategoryBreakdownData(session: ProjectSummary['sessions'][number]): boolean { @@ -3497,11 +3509,16 @@ function sessionRetryCount(session: ProjectSummary['sessions'][number]): number return session.turns.reduce((sum, turn) => sum + turn.retries, 0) } -function sessionTotalTurns(session: ProjectSummary['sessions'][number]): number { - if (hasCategoryBreakdownData(session)) { - return Object.values(session.categoryBreakdown).reduce((sum, c) => sum + c.turns, 0) +// Counted the way the classifier counts retries: one per edit tool call. +function sessionEditSteps(session: ProjectSummary['sessions'][number]): number { + let steps = 0 + for (const turn of session.turns) { + for (const call of turn.assistantCalls) { + const tools = call.toolSequence?.length ? call.toolSequence.flat().map(t => t.tool) : call.tools + steps += tools.filter(tool => EDIT_TOOLS.has(tool)).length + } } - return session.turns.length + return steps } // Token-savings estimate for a low-worth candidate. Two regimes: @@ -3511,8 +3528,8 @@ function sessionTotalTurns(session: ProjectSummary['sessions'][number]): number // - Sessions with edits but with retries / no one-shot: only the retry // fraction is counted as recoverable. Edits may still have been useful; // we credit the model with that and only flag the retry overhead. -// Ratio is bounded to [0, 1] so retry-heavy sessions with weird turn counts -// can't claim more than the full session token total. +// A retry is a re-edit, so the fraction is retries over edit steps, not over +// user prompts (an agent loop can make dozens of edits per prompt). function estimateLowWorthRecoverableTokens( session: ProjectSummary['sessions'][number], editTurns: number, @@ -3520,9 +3537,9 @@ function estimateLowWorthRecoverableTokens( ): number { const tokens = sessionTokenTotal(session) if (editTurns === 0) return Math.round(tokens * WORTH_IT_NO_EDIT_RECOVERY_FRACTION) - const totalTurns = sessionTotalTurns(session) - if (totalTurns === 0) return 0 - const fraction = Math.min(1, Math.max(0, retries / totalTurns)) + const editSteps = sessionEditSteps(session) + if (editSteps === 0) return 0 + const fraction = Math.min(1, Math.max(0, retries / editSteps)) return Math.round(tokens * fraction) } diff --git a/src/providers/kimicode.ts b/src/providers/kimicode.ts index fc50d825..c95cd52d 100644 --- a/src/providers/kimicode.ts +++ b/src/providers/kimicode.ts @@ -6,6 +6,7 @@ import { extractBashCommands } from '../bash-utils.js' import { calculateCost, pricingModelAt } from '../models.js' import { FS_SCAN_CONCURRENCY, mapWithConcurrency } from '../fs-utils.js' import type { ParsedProviderCall, ProbeRoot, Provider, SessionParser, SessionSource } from './types.js' +import type { ToolCall } from '../types.js' type JsonObject = Record @@ -249,7 +250,7 @@ function inputText(value: unknown): string { .join('\n') } -function toolDetails(value: unknown): { name: string; bashCommands: string[] } | null { +function toolDetails(value: unknown): { name: string; bashCommands: string[]; step: ToolCall } | null { const event = asObject(value) if (!event || stringValue(event['type']) !== 'tool.call') return null const rawName = stringValue(event['name']) @@ -265,9 +266,14 @@ function toolDetails(value: unknown): { name: string; bashCommands: string[] } | } } const command = stringValue(args?.['command']) + const path = stringValue(args?.['path']) + const step: ToolCall = { tool: name } + if (path) step.file = path + if (command) step.command = command return { name, bashCommands: name === 'Bash' && command ? extractBashCommands(command) : [], + step, } } @@ -293,6 +299,7 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars let currentRequest: RequestContext | null = null let pendingTools: string[] = [] let pendingBashCommands: string[] = [] + let pendingToolSequence: ToolCall[][] = [] let usageOrdinal = 0 const lines = contents.split(/\r?\n/) @@ -312,6 +319,7 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars if (type === 'turn.prompt') { pendingTools = [] pendingBashCommands = [] + pendingToolSequence = [] currentPrompt = inputText(record['input']) continue } @@ -336,6 +344,7 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars if (tool) { pendingTools.push(tool.name) pendingBashCommands.push(...tool.bashCommands) + pendingToolSequence.push([tool.step]) } continue } @@ -355,6 +364,7 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars if (!timestamp) { pendingTools = [] pendingBashCommands = [] + pendingToolSequence = [] continue } @@ -363,6 +373,7 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars if (seenKeys.has(deduplicationKey)) { pendingTools = [] pendingBashCommands = [] + pendingToolSequence = [] continue } seenKeys.add(deduplicationKey) @@ -388,6 +399,7 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars costIsEstimated: true, tools: pendingTools, bashCommands: pendingBashCommands, + toolSequence: pendingToolSequence.length > 0 ? pendingToolSequence : undefined, timestamp, speed: 'standard', deduplicationKey, @@ -400,6 +412,7 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars pendingTools = [] pendingBashCommands = [] + pendingToolSequence = [] } }, } diff --git a/src/session-cache.ts b/src/session-cache.ts index 5c343624..0a37abd4 100644 --- a/src/session-cache.ts +++ b/src/session-cache.ts @@ -598,7 +598,9 @@ export const PROVIDER_PARSE_VERSIONS: Record = { // that omits it. // cwd-project-v1: sessions whose state.json has `cwd` but no `workDir` land // on that folder instead of a path decoded from the wd_ directory name. - kimicode: 'wire-usage-v1-est-cost-session-lineage-capture-v1-cwd-project-v1', + // tool-sequence-v1: calls carry each tool call's path and shell command, so + // optimize sees commits/PRs and retries per file. Costs are unchanged. + kimicode: 'wire-usage-v1-est-cost-session-lineage-capture-v1-cwd-project-v1-tool-sequence-v1', // archived-subtree-v1: KiloCode shares the SQLite parser and the same schema. // billing-routes-v2: its warm cache must move with both shared route fields. 'kilo-code': 'worktree-project-grouping-v1-session-model-v1-archived-subtree-v1-billing-routes-v2-v2-legacy-union-v1-unknown-usage-v1-vertex-fallback-cost-v1', diff --git a/tests/classifier.test.ts b/tests/classifier.test.ts index a42c0570..6e62abdd 100644 --- a/tests/classifier.test.ts +++ b/tests/classifier.test.ts @@ -216,3 +216,11 @@ describe('classifyTurn — retry detection via toolSequence', () => { expect(c.retries).toBe(0) }) }) + +describe('classifyTurn — provider edit tool names', () => { + it('counts Devin and cursor-agent edit tools as edits', () => { + for (const tool of ['edit', 'write', 'cursor:strreplace', 'cursor:write']) { + expect(classifyTurn(makeTurn([makeCall({ tools: [tool] })])).hasEdits).toBe(true) + } + }) +}) diff --git a/tests/optimize-sidechains.test.ts b/tests/optimize-sidechains.test.ts index c54d9b45..64157b2e 100644 --- a/tests/optimize-sidechains.test.ts +++ b/tests/optimize-sidechains.test.ts @@ -295,11 +295,11 @@ describe('optimize sidechain population (issue #974)', () => { const projects = [project([ session('parent', { totalCostUSD: 1, - bashBreakdown: { 'git commit -m shipped': { calls: 1 } }, + bashBreakdown: { git: { calls: 1 } }, }), sidechain('agent-child', { totalCostUSD: 2, - bashBreakdown: { 'git commit -m irrelevant': { calls: 1 } }, + bashBreakdown: { git: { calls: 1 } }, }), ])] const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) diff --git a/tests/optimize.test.ts b/tests/optimize.test.ts index 6905614c..e6c1cb2e 100644 --- a/tests/optimize.test.ts +++ b/tests/optimize.test.ts @@ -89,7 +89,8 @@ function projectWithSessions(costs: number[], project = 'app'): ProjectSummary { function projectWithDeliveredSessions(costs: number[], project = 'app'): ProjectSummary { const summary = projectWithSessions(costs, project) for (const session of summary.sessions) { - session.bashBreakdown = { 'git commit -m test': { calls: 1 } } + session.bashBreakdown = { git: { calls: 1 } } + session.turns = [shellTurn('git commit -m test')] } return summary } @@ -659,6 +660,29 @@ function lowWorthTurn(overrides: Partial = {}): LowWorthTurn { } } +// Parsers key bashBreakdown by base command (`git`); the full command text +// lives only in each call's tool sequence. +function shellTurn(command: string): LowWorthTurn { + return lowWorthTurn({ + assistantCalls: [reliabilityCall({ tools: ['Bash'], toolSequence: [[{ tool: 'Bash', command }]] })], + }) +} + +function deliverySession(command: string): TestSession { + return lowWorthSession(8, 0, { turns: [shellTurn(command)], bashBreakdown: { [command.split(' ')[0]!]: { calls: 1 } } }) +} + +function editStepsTurn(steps: string[], retries: number): LowWorthTurn { + return lowWorthTurn({ + hasEdits: true, + retries, + assistantCalls: [reliabilityCall({ + tools: [...new Set(steps)], + toolSequence: steps.map(tool => [tool === 'Bash' ? { tool, command: 'npm test' } : { tool, file: '/tmp/a.ts' }]), + })], + }) +} + function lowWorthSession(cost: number, i: number, overrides: Partial = {}, project = 'app'): TestSession { const tokens = Math.round(cost * 1000) return { @@ -738,15 +762,14 @@ describe('detectLowWorthSessions', () => { }) it('estimates recoverable tokens by retry fraction for sessions with edits', () => { - // 4 turns, 2 retries spread across 2 edits, 0 one-shot edits → trips the - // 'no one-shot edit turns' reason. totalTurns=4, fraction=2/4=0.5, - // sessionTokenTotal=8K, so recoverable savings ceiling is 4K — half the - // session, not the full ceiling that no-edit sessions get. + // 2 retries over 4 edit steps, 0 one-shot edits → trips the 'no one-shot + // edit turns' reason. fraction=2/4=0.5. Billable output comes from the + // calls (0 here), so sessionTokenTotal is the 4K input and the estimate 2K. const project = projectWithLowWorthSessions([ lowWorthSession(4, 0, { turns: [ - lowWorthTurn({ hasEdits: true, retries: 1 }), - lowWorthTurn({ hasEdits: true, retries: 1 }), + editStepsTurn(['Edit', 'Bash', 'Edit'], 1), + editStepsTurn(['Edit', 'Bash', 'Edit'], 1), lowWorthTurn({ hasEdits: false }), lowWorthTurn({ hasEdits: false }), ], @@ -754,7 +777,20 @@ describe('detectLowWorthSessions', () => { ]) const finding = detectLowWorthSessions([project]) expect(finding).not.toBeNull() - expect(finding!.tokensSaved).toBe(4_000) + expect(finding!.tokensSaved).toBe(2_000) + }) + + it('divides retries by edit steps, not user prompts', () => { + // One prompt, an edit-test-fix loop: 3 retries over 6 edit steps. Per + // prompt this claimed the whole 4K session (3/1 capped at 1). + const project = projectWithLowWorthSessions([ + lowWorthSession(4, 0, { + turns: [editStepsTurn(['Edit', 'Edit', 'Edit', 'Bash', 'Edit', 'Edit', 'Edit'], 3)], + }), + ]) + const finding = detectLowWorthSessions([project]) + expect(finding!.explanation).toContain('3 retries') + expect(finding!.tokensSaved).toBe(2_000) }) it('uses the bounded recovery fraction for no-edit sessions', () => { @@ -795,42 +831,33 @@ describe('detectLowWorthSessions', () => { }) it('skips sessions with a git delivery command', () => { - const project = projectWithLowWorthSessions([ - lowWorthSession(8, 0, { - turns: [lowWorthTurn({ hasEdits: false })], - bashBreakdown: { 'cd /tmp/app && git commit -m "ship fix"': { calls: 1 } }, - }), - ]) + const project = projectWithLowWorthSessions([deliverySession('cd /tmp/app && git commit -m "ship fix"')]) expect(detectLowWorthSessions([project])).toBeNull() }) it('skips sessions with gh pr create', () => { - const project = projectWithLowWorthSessions([ - lowWorthSession(8, 0, { - turns: [lowWorthTurn({ hasEdits: false })], - bashBreakdown: { 'gh pr create --fill': { calls: 1 } }, - }), - ]) + const project = projectWithLowWorthSessions([deliverySession('gh pr create --fill')]) expect(detectLowWorthSessions([project])).toBeNull() }) + it('skips sessions that commit through git global options or on a later line', () => { + for (const command of [ + 'git -C /tmp/app commit -m "ship fix"', + 'git add -A && git -c user.name=dev -c user.email=dev@example.com commit -qm "ship fix"', + 'npm test\ngit push -u origin fix/ship', + 'for p in 1 2; do gh pr merge $p --squash; done', + ]) { + expect(detectLowWorthSessions([projectWithLowWorthSessions([deliverySession(command)])])).toBeNull() + } + }) + it('does not treat read-only git commands as delivery', () => { - const project = projectWithLowWorthSessions([ - lowWorthSession(8, 0, { - turns: [lowWorthTurn({ hasEdits: false })], - bashBreakdown: { 'git tag -l': { calls: 1 } }, - }), - ]) + const project = projectWithLowWorthSessions([deliverySession('git tag -l')]) expect(detectLowWorthSessions([project])).not.toBeNull() }) it('does not treat dry-run git commands as delivery', () => { - const project = projectWithLowWorthSessions([ - lowWorthSession(8, 0, { - turns: [lowWorthTurn({ hasEdits: false })], - bashBreakdown: { 'git push --dry-run origin main': { calls: 1 } }, - }), - ]) + const project = projectWithLowWorthSessions([deliverySession('git push --dry-run origin main')]) expect(detectLowWorthSessions([project])).not.toBeNull() }) @@ -838,22 +865,12 @@ describe('detectLowWorthSessions', () => { // Regex must match `git commit` only, not `git commit-tree` / // `git commit-graph`. Without the (?:\s|$|--) lookahead this would be a // false positive and the session would silently skip detection. - const project = projectWithLowWorthSessions([ - lowWorthSession(8, 0, { - turns: [lowWorthTurn({ hasEdits: false })], - bashBreakdown: { 'git commit-tree HEAD^{tree}': { calls: 1 } }, - }), - ]) + const project = projectWithLowWorthSessions([deliverySession('git commit-tree HEAD^{tree}')]) expect(detectLowWorthSessions([project])).not.toBeNull() }) it('still treats `git commit --amend` as a delivery command', () => { - const project = projectWithLowWorthSessions([ - lowWorthSession(8, 0, { - turns: [lowWorthTurn({ hasEdits: false })], - bashBreakdown: { 'git commit --amend --no-edit': { calls: 1 } }, - }), - ]) + const project = projectWithLowWorthSessions([deliverySession('git commit --amend --no-edit')]) expect(detectLowWorthSessions([project])).toBeNull() }) diff --git a/tests/providers/kimicode.test.ts b/tests/providers/kimicode.test.ts index 5d04ad9d..88767e26 100644 --- a/tests/providers/kimicode.test.ts +++ b/tests/providers/kimicode.test.ts @@ -299,6 +299,12 @@ describe('Kimi Code provider', () => { expect(calls).toHaveLength(1) expect(calls[0]!.tools).toEqual(['Write', 'Read', 'Bash', 'Grep']) expect(calls[0]!.bashCommands).toEqual(['npm', 'git']) + expect(calls[0]!.toolSequence).toEqual([ + [{ tool: 'Write', file: 'notes.txt' }], + [{ tool: 'Read', file: 'notes.txt' }], + [{ tool: 'Bash', command: 'npm test && git status' }], + [{ tool: 'Grep' }], + ]) expect(calls[0]!.model).toBe('glm-5.2') expect(calls[0]!.costUSD).toBeGreaterThan(0) }) @@ -320,6 +326,7 @@ describe('Kimi Code provider', () => { expect(calls).toHaveLength(1) expect(calls[0]!.tools).toEqual([]) expect(calls[0]!.bashCommands).toEqual([]) + expect(calls[0]!.toolSequence).toBeUndefined() }) it('keeps multiple turns in one session and attributes each prompt and model', async () => {