vrr/codeburn
2
0
Fork 0
mirror of https://github.com/AgentSeal/codeburn.git synced 2026-10-21 05:23:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

feat(claude): account for /advisor escalations under the advisor model (#659)

Browse source 
* feat(claude): account for /advisor escalations under the advisor model

Claude Code's advisor tool (/advisor) escalates hard decisions mid-turn
to a stronger model. Those tokens are recorded inside
message.usage.iterations as advisor_message entries with their own model
and are excluded from the top-level message.usage totals. The parser read
only the top-level totals, so advisor spend was silently dropped and never
attributed to the advisor model.

Emit a separate call per advisor_message iteration priced under the
advisor model, preserve those iterations through entry compaction and the
large-line (>32KB) byte-scanner path, and fold advisor cost into the guard
budget. Bump the claude parser and guard cache versions so existing caches
re-parse and pick up the advisor spend.

* test(guard): pin advisor cost fold in the live budget

The guard's advisor fold in computeSessionUsage had no test; dropping
advisorCost from the line cost left all 30 guard tests green. Pins the
fold (advisor line costs more than the same line without iterations)
and the per-message replace semantics under streaming rewrites.
This commit is contained in:
Resham Joshi 2026-07-16 07:42:49 -07:00 • committed by GitHub
parent ccd8764e5b
commit 4307542b62
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 366 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

16
src/guard/usage.ts
View file

@ -1,6 +1,6 @@
import { mkdir, readFile, stat, writeFile } from 'fs/promises'
import { readSessionLines } from '../fs-utils.js'
import { parseApiCall, parseJsonlLine } from '../parser.js'
import { parseApiCall, parseAdvisorCalls, parseJsonlLine } from '../parser.js'
import { EDIT_TOOLS } from '../classifier.js'
import { allowPath, sessionCachePath, sessionsDir } from './store.js'
@ -33,7 +33,9 @@ export type GuardCache = {
// v2: per-message-id replace fold (perMessage map, sawEdit boolean) and the
// guard/sessions/ cache location. v1 caches are ignored and cold-reparse once.
export const GUARD_CACHE_VERSION = 2
// v3: fold advisor (/advisor) cost into each message; v2 caches computed the
// per-message cost without it, so they must cold-reparse.
export const GUARD_CACHE_VERSION = 3
// `commit` must be the git subcommand: `git`, optionally flag tokens (long
// flags, or a short flag with an optional separate value like `-c k=v`), then
@ -121,14 +123,18 @@ export async function computeSessionUsage(
if (!entry) continue
const call = parseApiCall(entry)
if (!call) continue
// Advisor (/advisor) escalations are billed under a separate model and live
// outside the top-level usage, so add their cost to the message total.
const advisorCost = parseAdvisorCalls(entry).reduce((sum, a) => sum + a.costUSD, 0)
const lineCost = call.costUSD + advisorCost
// Last-wins per message id, matching the shipped dedupeStreamingMessageIds.
// Lines without an id (rare, and never streamed in copies) just add.
const msgId = (entry.message as { id?: string } | undefined)?.id
if (msgId) {
cache.costUSD += call.costUSD - (cache.perMessage[msgId] ?? 0)
cache.perMessage[msgId] = call.costUSD
cache.costUSD += lineCost - (cache.perMessage[msgId] ?? 0)
cache.perMessage[msgId] = lineCost
} else {
cache.costUSD += call.costUSD
cache.costUSD += lineCost
}
for (const tc of call.toolSequence?.flat() ?? []) {
if (!cache.sawEdit && EDIT_TOOLS.has(tc.tool)) cache.sawEdit = true

127
src/parser.ts
View file

@ -24,6 +24,7 @@ import {
} from './session-cache.js'
import type { ParsedProviderCall, SessionSource } from './providers/types.js'
import type {
ApiUsageIteration,
AssistantMessageContent,
ClassifiedTurn,
ContentBlock,
@ -276,6 +277,26 @@ function readJsonNumberField(source: string, objectBounds: JsonValueBounds | nul
return Number.isFinite(value) ? value : undefined
}
// The large-line parsers avoid JSON.parse on the whole (multi-KB) line, but the
// usage object itself is tiny; parse just that slice to recover advisor
// (/advisor) iterations, which the byte-scanner cannot cheaply extract. Without
// this, an advisor escalation on a large assistant turn would be dropped.
function extractAdvisorIterations(usageObjectJson: string): ApiUsageIteration[] | undefined {
let parsed: unknown
try {
parsed = JSON.parse(usageObjectJson)
} catch {
return undefined
}
const iterations = (parsed as { iterations?: unknown }).iterations
if (!Array.isArray(iterations)) return undefined
const advisor = iterations.filter(
(it): it is ApiUsageIteration =>
!!it && typeof it === 'object' && (it as { type?: unknown }).type === 'advisor_message',
)
return advisor.length > 0 ? advisor : undefined
}
function parseLargeUsage(source: string, usageBounds: JsonValueBounds | null) {
const usage: AssistantMessageContent['usage'] = {
input_tokens: readJsonNumberField(source, usageBounds, 'input_tokens') ?? 0,
@ -307,6 +328,9 @@ function parseLargeUsage(source: string, usageBounds: JsonValueBounds | null) {
const speed = readJsonString(source, findObjectFieldValue(source, usageBounds.start, usageBounds.end, 'speed'))
if (speed === 'standard' || speed === 'fast') usage.speed = speed
const advisor = extractAdvisorIterations(source.slice(usageBounds.start, usageBounds.end))
if (advisor) usage.iterations = advisor
}
return usage
@ -652,6 +676,9 @@ function parseLargeUsageBuffer(source: Buffer, usageBounds: BufferJsonValueBound
const speed = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, usageBounds.start, usageBounds.end, 'speed'))
if (speed === 'standard' || speed === 'fast') usage.speed = speed
const advisor = extractAdvisorIterations(source.subarray(usageBounds.start, usageBounds.end).toString('utf8'))
if (advisor) usage.iterations = advisor
}
return usage
@ -952,6 +979,33 @@ export function compactEntry(raw: JournalEntry): JournalEntry {
}
}
if (u.speed) compactUsage.speed = u.speed
// Preserve only advisor_message iterations (/advisor sub-usage) so
// parseAdvisorCalls can attribute the advisor model's spend; drop the rest to
// keep the cache small. Other iteration types (plain `message`, and the
// `fallback_message` written when a turn retries on another model) are not
// accounted here, a separate pre-existing gap, so they are not preserved.
if (Array.isArray(u.iterations)) {
const advisorIterations = u.iterations
.filter((it): it is ApiUsageIteration => !!it && it.type === 'advisor_message')
.map(it => {
const compact: ApiUsageIteration = { type: 'advisor_message' }
if (typeof it.model === 'string') compact.model = it.model
if (it.input_tokens) compact.input_tokens = it.input_tokens
if (it.output_tokens) compact.output_tokens = it.output_tokens
if (it.cache_creation_input_tokens) compact.cache_creation_input_tokens = it.cache_creation_input_tokens
if (it.cache_read_input_tokens) compact.cache_read_input_tokens = it.cache_read_input_tokens
if (it.cache_creation) {
compact.cache_creation = {
...(it.cache_creation.ephemeral_5m_input_tokens ? { ephemeral_5m_input_tokens: it.cache_creation.ephemeral_5m_input_tokens } : {}),
...(it.cache_creation.ephemeral_1h_input_tokens ? { ephemeral_1h_input_tokens: it.cache_creation.ephemeral_1h_input_tokens } : {}),
}
}
if (it.server_tool_use?.web_search_requests) compact.server_tool_use = { web_search_requests: it.server_tool_use.web_search_requests }
if (it.speed) compact.speed = it.speed
return compact
})
if (advisorIterations.length > 0) compactUsage.iterations = advisorIterations
}
entry.message = {
type: 'message',
@ -1037,7 +1091,10 @@ export function isPositiveNumber(value: unknown): value is number {
return typeof value === 'number' && Number.isFinite(value) && value > 0
}
function extractClaudeCacheCreation(usage: AssistantMessageContent['usage']): { totalTokens: number; oneHourTokens: number } {
function extractClaudeCacheCreation(usage: {
cache_creation_input_tokens?: number
cache_creation?: { ephemeral_5m_input_tokens?: number; ephemeral_1h_input_tokens?: number }
}): { totalTokens: number; oneHourTokens: number } {
const legacyTotal = safeNumber(usage.cache_creation_input_tokens)
const cacheCreation = usage.cache_creation
const fiveMinuteTokens = safeNumber(cacheCreation?.ephemeral_5m_input_tokens)
@ -1149,6 +1206,73 @@ export function parseApiCall(entry: JournalEntry): ParsedApiCall | null {
})
}
/// Claude Code's advisor tool (/advisor) escalates hard decisions to a stronger
/// advisor model mid-turn. Those tokens are recorded as `advisor_message`
/// records inside `message.usage.iterations` under the advisor's own model, and
/// are excluded from the top-level `message.usage` totals that `parseApiCall`
/// reads. Emit them as separate calls so the advisor's spend is counted and
/// attributed to the advisor model rather than silently dropped.
export function parseAdvisorCalls(entry: JournalEntry): ParsedApiCall[] {
if (entry.type !== 'assistant') return []
const msg = entry.message as AssistantMessageContent | undefined
const iterations = msg?.usage?.iterations
if (!msg?.usage || !Array.isArray(iterations)) return []
const calls: ParsedApiCall[] = []
const baseKey = msg.id ?? `claude:${entry.timestamp}`
// Ordinal among advisor entries (not the raw array index) so the dedup key is
// identical whether it is computed from the raw record (guard path) or the
// compacted record whose non-advisor iterations were dropped (report path).
let advisorOrdinal = 0
for (const it of iterations) {
if (!it || it.type !== 'advisor_message') continue
const model = typeof it.model === 'string' && it.model ? it.model : msg.model
if (!model) continue
const index = advisorOrdinal++
const cacheCreation = extractClaudeCacheCreation(it)
const tokens: TokenUsage = {
inputTokens: it.input_tokens ?? 0,
outputTokens: it.output_tokens ?? 0,
cacheCreationInputTokens: cacheCreation.totalTokens,
cacheReadInputTokens: it.cache_read_input_tokens ?? 0,
cachedInputTokens: 0,
reasoningTokens: 0,
webSearchRequests: it.server_tool_use?.web_search_requests ?? 0,
}
const speed = it.speed ?? msg.usage.speed ?? 'standard'
const costUSD = calculateCost(
model,
tokens.inputTokens,
tokens.outputTokens,
tokens.cacheCreationInputTokens,
tokens.cacheReadInputTokens,
tokens.webSearchRequests,
speed,
cacheCreation.oneHourTokens,
)
calls.push(applyLocalModelSavings({
provider: 'claude',
model,
usage: tokens,
costUSD,
tools: [],
mcpTools: [],
skills: [],
subagentTypes: [],
hasAgentSpawn: false,
hasPlanMode: false,
speed,
timestamp: entry.timestamp ?? '',
bashCommands: [],
deduplicationKey: `${baseKey}:advisor:${index}`,
cacheCreationOneHourTokens: cacheCreation.oneHourTokens || undefined,
}))
}
return calls
}
export function dedupeStreamingMessageIds(entries: JournalEntry[]): JournalEntry[] {
const firstIdxById = new Map<string, number>()
const lastIdxById = new Map<string, number>()
@ -1203,6 +1327,7 @@ function groupIntoTurns(entries: JournalEntry[], seenMsgIds: Set<string>): Parse
if (msgId) seenMsgIds.add(msgId)
const call = parseApiCall(entry)
if (call) currentCalls.push(call)
for (const advisorCall of parseAdvisorCalls(entry)) currentCalls.push(advisorCall)
}
}

2
src/session-cache.ts
View file

@ -114,7 +114,7 @@ const PROVIDER_ENV_VARS: Record<string, string[]> = {
export const DURABLE_PROVIDER_NAMES: ReadonlySet<string> = new Set(['copilot'])
const PROVIDER_PARSE_VERSIONS: Record<string, string> = {
claude: 'cowork-space-grouping-v1',
claude: 'advisor-usage-v1',
cline: 'worktree-project-grouping-v1',
codewhale: 'aggregate-session-v1',
// Bump when the Codex parser changes attribution so unchanged, already-cached

23
src/types.ts
View file

@ -35,6 +35,29 @@ export type ApiUsage = {
web_fetch_requests?: number
}
speed?: 'standard' | 'fast'
// Claude Code advisor tool (/advisor): per-turn sub-usage records. A record
// with type 'advisor_message' carries the advisor model's own tokens and is
// NOT included in the top-level totals above; type 'message' records mirror
// the main model and are already covered by the top-level totals.
iterations?: ApiUsageIteration[]
}
export type ApiUsageIteration = {
type?: string
model?: string
input_tokens?: number
output_tokens?: number
cache_creation_input_tokens?: number
cache_creation?: {
ephemeral_5m_input_tokens?: number
ephemeral_1h_input_tokens?: number
}
cache_read_input_tokens?: number
server_tool_use?: {
web_search_requests?: number
web_fetch_requests?: number
}
speed?: 'standard' | 'fast'
}
export type AssistantMessageContent = {

25
tests/guard-hooks.test.ts
View file

@ -88,6 +88,31 @@ describe('incremental session cache', () => {
expect(r2.cache.costUSD).toBeGreaterThan(r1.cache.costUSD)
})
it('folds advisor (/advisor) iteration cost into the session total', async () => {
const plain = await transcript([assistantLine('a')])
const plainCost = (await computeSessionUsage(emptyCache(SID), plain)).cache.costUSD
// Same line plus an advisor_message iteration: the advisor's tokens live
// outside the top-level usage, so the guard total must exceed the plain
// line by the advisor call's own cost.
const line = JSON.parse(assistantLine('a')) as { message: { usage: Record<string, unknown> } }
line.message.usage['iterations'] = [
{ type: 'message', input_tokens: 1_000_000, output_tokens: 200_000 },
{ type: 'advisor_message', model: 'claude-opus-4-20250514', input_tokens: 500_000, output_tokens: 50_000 },
]
const withAdvisor = await transcript([JSON.stringify(line) + '\n'])
const advisorTotal = (await computeSessionUsage(emptyCache(SID), withAdvisor)).cache.costUSD
expect(plainCost).toBeGreaterThan(0)
expect(advisorTotal).toBeGreaterThan(plainCost)
// Streaming rewrite of the same message id replaces, not adds: the fold
// stays stable when the advisor-carrying line is written twice.
const rewritten = await transcript([JSON.stringify(line) + '\n', JSON.stringify(line) + '\n'])
const rewrittenTotal = (await computeSessionUsage(emptyCache(SID), rewritten)).cache.costUSD
expect(rewrittenTotal).toBeCloseTo(advisorTotal, 10)
})
it('resets to a cold parse when the transcript shrinks (rotation)', async () => {
const base = await tmp()
const path = await transcript([assistantLine('a'), assistantLine('b')])

180
tests/parser-advisor-usage.test.ts Normal file
View file

@ -0,0 +1,180 @@
import { mkdtemp, mkdir, rm, writeFile } from 'fs/promises'
import { tmpdir } from 'os'
import { join } from 'path'
import { afterEach, beforeAll, describe, expect, it } from 'vitest'
import { parseApiCall, parseAdvisorCalls, compactEntry, parseAllSessions, clearSessionCache } from '../src/parser.js'
import { calculateCost, loadPricing } from '../src/models.js'
import type { JournalEntry } from '../src/types.js'
const MAIN_MODEL = 'claude-sonnet-4-20250514'
const ADVISOR_MODEL = 'claude-opus-4-20250514'
// Shape mirrors a real Claude Code /advisor turn: the top-level usage covers
// only the main model, and the advisor's tokens live in an `advisor_message`
// iteration under its own model. The two `message` iterations sum to the
// top-level usage (verified against real Claude Code session data).
function advisorEntry(): JournalEntry {
return {
type: 'assistant',
timestamp: '2026-07-10T10:00:00.000Z',
sessionId: 's1',
message: {
type: 'message',
role: 'assistant',
model: MAIN_MODEL,
id: 'msg-advisor-1',
content: [],
usage: {
input_tokens: 2,
output_tokens: 491,
cache_creation_input_tokens: 7853,
cache_read_input_tokens: 226584,
iterations: [
{ type: 'message', input_tokens: 1, output_tokens: 45, cache_creation_input_tokens: 7192, cache_read_input_tokens: 109696 },
{ type: 'advisor_message', model: ADVISOR_MODEL, input_tokens: 159419, output_tokens: 7805, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 },
{ type: 'message', input_tokens: 1, output_tokens: 446, cache_creation_input_tokens: 661, cache_read_input_tokens: 116888 },
],
},
},
} as unknown as JournalEntry
}
describe('advisor usage parsing', () => {
beforeAll(async () => {
await loadPricing()
})
it('leaves the main-model call attributed to the main model and its top-level totals', () => {
const call = parseApiCall(advisorEntry())
expect(call).not.toBeNull()
expect(call!.model).toBe(MAIN_MODEL)
expect(call!.usage.inputTokens).toBe(2)
expect(call!.usage.outputTokens).toBe(491)
expect(call!.usage.cacheReadInputTokens).toBe(226584)
expect(call!.deduplicationKey).toBe('msg-advisor-1')
})
it('emits a separate call for the advisor iteration, priced under the advisor model', () => {
const advisorCalls = parseAdvisorCalls(advisorEntry())
expect(advisorCalls).toHaveLength(1)
const a = advisorCalls[0]!
expect(a.model).toBe(ADVISOR_MODEL)
expect(a.usage.inputTokens).toBe(159419)
expect(a.usage.outputTokens).toBe(7805)
expect(a.deduplicationKey).toBe('msg-advisor-1:advisor:0')
const expectedCost = calculateCost(ADVISOR_MODEL, 159419, 7805, 0, 0, 0, 'standard', 0)
expect(a.costUSD).toBeCloseTo(expectedCost, 10)
expect(a.costUSD).toBeGreaterThan(0)
})
it('does not double-count: advisor tokens are absent from the main call', () => {
const call = parseApiCall(advisorEntry())!
// 159419 is the advisor input; it must never appear on the main call.
expect(call.usage.inputTokens).not.toBe(159419)
})
it('returns no advisor calls when iterations hold only main-model messages', () => {
const entry = advisorEntry()
// Strip the advisor_message iteration.
const usage = (entry.message as { usage: { iterations: unknown[] } }).usage
usage.iterations = usage.iterations.filter((it: unknown) => (it as { type?: string }).type !== 'advisor_message')
expect(parseAdvisorCalls(entry)).toHaveLength(0)
})
it('returns no advisor calls when there is no iterations array', () => {
const entry = advisorEntry()
delete (entry.message as { usage: { iterations?: unknown } }).usage.iterations
expect(parseAdvisorCalls(entry)).toHaveLength(0)
})
it('survives compaction: compactEntry keeps advisor iterations so the call is still emitted', () => {
const compacted = compactEntry(advisorEntry())
const advisorCalls = parseAdvisorCalls(compacted)
expect(advisorCalls).toHaveLength(1)
expect(advisorCalls[0]!.model).toBe(ADVISOR_MODEL)
expect(advisorCalls[0]!.usage.inputTokens).toBe(159419)
expect(advisorCalls[0]!.deduplicationKey).toBe('msg-advisor-1:advisor:0')
})
})
describe('advisor usage end-to-end through parseAllSessions', () => {
let tmpDir: string | null = null
const savedEnv = { config: process.env['CLAUDE_CONFIG_DIR'], cache: process.env['CODEBURN_CACHE_DIR'] }
beforeAll(async () => {
await loadPricing()
})
afterEach(async () => {
clearSessionCache()
if (savedEnv.config === undefined) delete process.env['CLAUDE_CONFIG_DIR']
else process.env['CLAUDE_CONFIG_DIR'] = savedEnv.config
if (savedEnv.cache === undefined) delete process.env['CODEBURN_CACHE_DIR']
else process.env['CODEBURN_CACHE_DIR'] = savedEnv.cache
if (tmpDir) {
await rm(tmpDir, { recursive: true, force: true })
tmpDir = null
}
})
it('attributes advisor spend to the advisor model in the session breakdown', async () => {
tmpDir = await mkdtemp(join(tmpdir(), 'codeburn-advisor-'))
process.env['CLAUDE_CONFIG_DIR'] = tmpDir
process.env['CODEBURN_CACHE_DIR'] = join(tmpDir, 'cache')
const proj = join(tmpDir, 'projects', 'p')
await mkdir(proj, { recursive: true })
const user = JSON.stringify({ type: 'user', timestamp: '2026-07-10T09:59:59.000Z', sessionId: 's1', message: { role: 'user', content: 'hi' } })
const assistant = JSON.stringify(advisorEntry())
await writeFile(join(proj, 's1.jsonl'), `${user}\n${assistant}\n`)
clearSessionCache()
const projects = await parseAllSessions(undefined, 'claude')
const inputByModel: Record<string, number> = {}
for (const p of projects) {
for (const s of p.sessions) {
for (const [model, b] of Object.entries(s.modelBreakdown ?? {})) {
inputByModel[model] = (inputByModel[model] ?? 0) + (b.tokens?.inputTokens ?? 0)
}
}
}
// Main model keeps only its top-level tokens; advisor model carries its own.
const advisorModelKey = Object.keys(inputByModel).find(m => m.toLowerCase().includes('opus'))
expect(advisorModelKey).toBeDefined()
expect(inputByModel[advisorModelKey!]).toBe(159419)
const mainModelKey = Object.keys(inputByModel).find(m => m.toLowerCase().includes('sonnet'))
expect(inputByModel[mainModelKey!]).toBe(2)
})
it('counts advisor spend even when the assistant line exceeds the large-line (32KB) threshold', async () => {
tmpDir = await mkdtemp(join(tmpdir(), 'codeburn-advisor-big-'))
process.env['CLAUDE_CONFIG_DIR'] = tmpDir
process.env['CODEBURN_CACHE_DIR'] = join(tmpDir, 'cache')
const proj = join(tmpDir, 'projects', 'p')
await mkdir(proj, { recursive: true })
// Pad the content past 32KB so parseJsonlLine routes to the large-line
// byte-scanner path (which previously dropped iterations entirely).
const entry = advisorEntry()
;(entry.message as { content: unknown[] }).content = [{ type: 'text', text: 'x'.repeat(40_000) }]
const assistant = JSON.stringify(entry)
expect(Buffer.byteLength(assistant, 'utf8')).toBeGreaterThan(32 * 1024)
const user = JSON.stringify({ type: 'user', timestamp: '2026-07-10T09:59:59.000Z', sessionId: 's1', message: { role: 'user', content: 'hi' } })
await writeFile(join(proj, 's1.jsonl'), `${user}\n${assistant}\n`)
clearSessionCache()
const projects = await parseAllSessions(undefined, 'claude')
const inputByModel: Record<string, number> = {}
for (const p of projects) {
for (const s of p.sessions) {
for (const [model, b] of Object.entries(s.modelBreakdown ?? {})) {
inputByModel[model] = (inputByModel[model] ?? 0) + (b.tokens?.inputTokens ?? 0)
}
}
}
const advisorModelKey = Object.keys(inputByModel).find(m => m.toLowerCase().includes('opus'))
expect(advisorModelKey).toBeDefined()
expect(inputByModel[advisorModelKey!]).toBe(159419)
})
})