codeburn/tests/parser-compact-entry.test.ts
Resham Joshi fdb9c185af
fix(claude): estimate the output of subagent messages that never got a final usage row (#1854)
* fix(claude): estimate the output of subagent messages that never got a final usage row

Claude Code before 2.1.307 wrote subagent (sidechain) assistant messages
without their final usage. Every row of such a message id keeps the
output_tokens of the first streamed chunk (usually 1 to 100) and a null
stop_reason, so the output was priced at that placeholder. Input and cache
tokens on those rows are correct. 2.1.307 fixed the writer.

For a sidechain row from a version below 2.1.307 with no stop_reason, the
parser now estimates the output from the message content (text plus
serialized tool_use input, 4 chars per token, summed over every row of the
id) and uses it when it exceeds the recorded count, flagging the call as
estimated. Rows with a stop_reason, main-thread rows and rows from 2.1.307
on are untouched. Large lines get the same estimate from the byte scanner.
The claude parse version gains sidechain-output-estimate-v1 so cached
sessions re-parse.

Measured on one machine with 327,711 Claude calls: 209,889 calls from
2.1.272 to 2.1.304 are re-estimated, output rises from 89.5M to 132.0M
tokens and cost from $22,947 to $23,687 (+$740). Call counts, input and
cache tokens do not change. On finished subagent messages from 2.1.307 on,
the same estimate recovers 24 to 48 percent of the real output, since
thinking is not visible in the content, so this is a lower bound.

* fix(daily-cache): bump to 78 so settled days carry the estimated subagent output
2026-11-04 19:48:32 +01:00

516 lines
19 KiB
TypeScript

import { describe, it, expect } from 'vitest'
import { compactEntry, dedupeStreamingMessageIds, parseApiCall, parseJsonlLine } from '../src/parser.js'
import type { JournalEntry } from '../src/types.js'
function entry(overrides: Partial<JournalEntry> & Record<string, unknown>): JournalEntry {
return { type: 'user', ...overrides } as JournalEntry
}
describe('compactEntry', () => {
it('preserves type, timestamp, sessionId, cwd', () => {
const raw = entry({ type: 'user', timestamp: 't1', sessionId: 's1', cwd: '/foo' })
const c = compactEntry(raw)
expect(c.type).toBe('user')
expect(c.timestamp).toBe('t1')
expect(c.sessionId).toBe('s1')
expect(c.cwd).toBe('/foo')
})
it('strips unknown catch-all fields', () => {
const raw = entry({
type: 'assistant',
toolResult: { type: 'tool_result', content: 'x'.repeat(10_000) },
someHugeField: 'y'.repeat(10_000),
})
const c = compactEntry(raw)
expect((c as Record<string, unknown>)['toolResult']).toBeUndefined()
expect((c as Record<string, unknown>)['someHugeField']).toBeUndefined()
})
it('preserves prUrl on a pr-link entry for per-turn PR attribution', () => {
const raw = entry({ type: 'pr-link', prUrl: 'https://github.com/o/r/pull/1', extra: 'x'.repeat(10_000) })
const c = compactEntry(raw)
expect((c as Record<string, unknown>)['prUrl']).toBe('https://github.com/o/r/pull/1')
expect((c as Record<string, unknown>)['extra']).toBeUndefined()
})
it('does not copy prUrl onto non-pr-link entries', () => {
const raw = entry({ type: 'user', prUrl: 'https://github.com/o/r/pull/1' })
const c = compactEntry(raw)
expect((c as Record<string, unknown>)['prUrl']).toBeUndefined()
})
it('preserves deferred_tools_delta attachment with copied names', () => {
const raw = entry({
type: 'attachment',
attachment: {
type: 'deferred_tools_delta',
addedNames: ['mcp__svc__t1', 'Bash'],
extraData: 'should be dropped',
},
})
const c = compactEntry(raw)
const att = (c as Record<string, unknown>)['attachment'] as Record<string, unknown>
expect(att['type']).toBe('deferred_tools_delta')
expect(att['addedNames']).toEqual(['mcp__svc__t1', 'Bash'])
expect(att['extraData']).toBeUndefined()
})
it('copies addedNames into a new array (not by reference)', () => {
const originalNames = ['mcp__a__b', 'Bash']
const raw = entry({
type: 'attachment',
attachment: { type: 'deferred_tools_delta', addedNames: originalNames },
})
const c = compactEntry(raw)
const att = (c as Record<string, unknown>)['attachment'] as { addedNames: string[] }
expect(att.addedNames).not.toBe(originalNames)
expect(att.addedNames).toEqual(originalNames)
})
it('caps addedNames at 1000 entries', () => {
const names = Array.from({ length: 2000 }, (_, i) => `mcp__svc__t${i}`)
const raw = entry({
type: 'attachment',
attachment: { type: 'deferred_tools_delta', addedNames: names },
})
const c = compactEntry(raw)
const att = (c as Record<string, unknown>)['attachment'] as { addedNames: string[] }
expect(att.addedNames).toHaveLength(1000)
})
it('filters non-string entries from addedNames', () => {
const raw = entry({
type: 'attachment',
attachment: { type: 'deferred_tools_delta', addedNames: [42, null, 'mcp__a__b', undefined] },
})
const c = compactEntry(raw)
const att = (c as Record<string, unknown>)['attachment'] as { addedNames: string[] }
expect(att.addedNames).toEqual(['mcp__a__b'])
})
it('drops non-deferred_tools_delta attachments', () => {
const raw = entry({
type: 'attachment',
attachment: { type: 'other', data: 'x'.repeat(10_000) },
})
const c = compactEntry(raw)
expect((c as Record<string, unknown>)['attachment']).toBeUndefined()
})
it('caps user message string content at 2000', () => {
const longText = 'a'.repeat(5000)
const raw = entry({
type: 'user',
message: { role: 'user' as const, content: longText },
})
const c = compactEntry(raw)
expect(c.message!.role).toBe('user')
const content = (c.message as { content: string }).content
expect(content.length).toBe(2000)
})
it('caps total user text across all blocks at 2000', () => {
const raw = entry({
type: 'user',
message: {
role: 'user' as const,
content: [
{ type: 'text' as const, text: 'a'.repeat(1500) },
{ type: 'text' as const, text: 'b'.repeat(1500) },
{ type: 'text' as const, text: 'c'.repeat(1500) },
{ type: 'image' as const, source: 'big data' },
],
},
})
const c = compactEntry(raw)
const content = (c.message as { content: Array<{ type: string; text: string }> }).content
expect(content).toHaveLength(2)
expect(content[0]!.text.length).toBe(1500)
expect(content[1]!.text.length).toBe(500)
})
it('compacts assistant tool_use blocks, dropping text and thinking, preserving id', () => {
const raw = entry({
type: 'assistant',
timestamp: 't1',
message: {
type: 'message' as const,
role: 'assistant' as const,
model: 'claude-opus-4-6',
id: 'msg_123',
usage: { input_tokens: 100, output_tokens: 200 },
content: [
{ type: 'text', text: 'x'.repeat(50_000) },
{ type: 'thinking', thinking: 'y'.repeat(50_000) },
{ type: 'tool_use', id: 'tu1', name: 'Read', input: { file_path: '/foo', huge: 'z'.repeat(10_000) } },
{ type: 'tool_use', id: 'tu2', name: 'Edit', input: { old_string: 'a'.repeat(5000), new_string: 'b'.repeat(5000) } },
],
},
})
const c = compactEntry(raw)
const msg = c.message as { content: Array<{ type: string; id?: string; name?: string; input?: Record<string, unknown> }> }
expect(msg.content).toHaveLength(2)
expect(msg.content[0]!.name).toBe('Read')
expect(msg.content[0]!.id).toBe('tu1')
expect(msg.content[0]!.input).toEqual({ file_path: '/foo' })
expect(msg.content[1]!.name).toBe('Edit')
expect(msg.content[1]!.id).toBe('tu2')
expect(msg.content[1]!.input).toEqual({})
})
it('caps tool_use blocks at 500 per message', () => {
const blocks = Array.from({ length: 600 }, (_, i) => ({
type: 'tool_use' as const,
id: `tu${i}`,
name: `Tool${i}`,
input: {},
}))
const raw = entry({
type: 'assistant',
message: {
type: 'message' as const,
role: 'assistant' as const,
model: 'claude-opus-4-6',
usage: { input_tokens: 10, output_tokens: 10 },
content: blocks,
},
})
const c = compactEntry(raw)
const msg = c.message as { content: unknown[] }
expect(msg.content).toHaveLength(500)
})
it('preserves model, usage (destructured), and id on assistant messages', () => {
const raw = entry({
type: 'assistant',
message: {
type: 'message' as const,
role: 'assistant' as const,
model: 'claude-opus-4-6',
id: 'msg_abc',
usage: {
input_tokens: 50,
output_tokens: 100,
cache_read_input_tokens: 25,
extraGarbage: 'should not survive',
},
content: [],
},
})
const c = compactEntry(raw)
const msg = c.message as { model: string; id: string; usage: Record<string, unknown> }
expect(msg.model).toBe('claude-opus-4-6')
expect(msg.id).toBe('msg_abc')
expect(msg.usage['input_tokens']).toBe(50)
expect(msg.usage['output_tokens']).toBe(100)
expect(msg.usage['cache_read_input_tokens']).toBe(25)
expect(msg.usage['extraGarbage']).toBeUndefined()
})
it('deep-copies usage nested objects, stripping extra keys', () => {
const cacheCreation = { ephemeral_5m_input_tokens: 100, ephemeral_1h_input_tokens: 200, extraJunk: 'big' }
const serverToolUse = { web_search_requests: 3, web_fetch_requests: 1, extraJunk: 'big' }
const raw = entry({
type: 'assistant',
message: {
type: 'message' as const,
role: 'assistant' as const,
model: 'claude-opus-4-6',
usage: {
input_tokens: 10,
output_tokens: 10,
speed: 'fast',
cache_creation: cacheCreation,
server_tool_use: serverToolUse,
},
content: [],
},
})
const c = compactEntry(raw)
const msg = c.message as { usage: Record<string, unknown> }
expect(msg.usage['speed']).toBe('fast')
const cc = msg.usage['cache_creation'] as Record<string, unknown>
expect(cc['ephemeral_5m_input_tokens']).toBe(100)
expect(cc['ephemeral_1h_input_tokens']).toBe(200)
expect(cc['extraJunk']).toBeUndefined()
expect(cc).not.toBe(cacheCreation)
const stu = msg.usage['server_tool_use'] as Record<string, unknown>
expect(stu['web_search_requests']).toBe(3)
expect(stu['web_fetch_requests']).toBe(1)
expect(stu['extraJunk']).toBeUndefined()
expect(stu).not.toBe(serverToolUse)
})
it('keeps Skill input.skill and input.name, type-checked and capped', () => {
const raw = entry({
type: 'assistant',
message: {
type: 'message' as const,
role: 'assistant' as const,
model: 'claude-opus-4-6',
usage: { input_tokens: 10, output_tokens: 10 },
content: [
{ type: 'tool_use', id: 'tu', name: 'Skill', input: { skill: 'graphify', args: 'huge arg data' } },
],
},
})
const c = compactEntry(raw)
const msg = c.message as { content: Array<{ input: Record<string, unknown> }> }
expect(msg.content[0]!.input['skill']).toBe('graphify')
expect(msg.content[0]!.input['args']).toBeUndefined()
})
it('rejects non-string Skill input.skill and caps long names', () => {
const raw = entry({
type: 'assistant',
message: {
type: 'message' as const,
role: 'assistant' as const,
model: 'claude-opus-4-6',
usage: { input_tokens: 10, output_tokens: 10 },
content: [
{ type: 'tool_use', id: 'tu1', name: 'Skill', input: { skill: { malicious: 'x'.repeat(10_000) } } },
{ type: 'tool_use', id: 'tu2', name: 'Skill', input: { skill: 'a'.repeat(500) } },
],
},
})
const c = compactEntry(raw)
const msg = c.message as { content: Array<{ input: Record<string, unknown> }> }
expect(msg.content[0]!.input['skill']).toBeUndefined()
expect((msg.content[1]!.input['skill'] as string).length).toBe(200)
})
it('keeps Bash input.command capped at 2000 for bash command extraction', () => {
const longCmd = 'npm run build && '.repeat(200)
const raw = entry({
type: 'assistant',
message: {
type: 'message' as const,
role: 'assistant' as const,
model: 'claude-opus-4-6',
usage: { input_tokens: 10, output_tokens: 10 },
content: [
{ type: 'tool_use', id: 'tu', name: 'Bash', input: { command: longCmd, description: 'big desc' } },
],
},
})
const c = compactEntry(raw)
const msg = c.message as { content: Array<{ input: Record<string, unknown> }> }
const cmd = msg.content[0]!.input['command'] as string
expect(cmd.length).toBe(2000)
expect(msg.content[0]!.input['description']).toBeUndefined()
})
it('keeps Read file_path capped and drops unrelated input fields', () => {
const raw = entry({
type: 'assistant',
message: {
type: 'message' as const,
role: 'assistant' as const,
model: 'claude-opus-4-6',
usage: { input_tokens: 10, output_tokens: 10 },
content: [
{ type: 'tool_use', id: 'tu', name: 'Read', input: { file_path: '/tmp/' + 'x'.repeat(3000), content: 'big' } },
],
},
})
const c = compactEntry(raw)
const msg = c.message as { content: Array<{ input: Record<string, unknown> }> }
expect((msg.content[0]!.input['file_path'] as string).length).toBe(2000)
expect(msg.content[0]!.input['content']).toBeUndefined()
})
it('keeps Agent subagent_type capped and drops prompt text', () => {
const raw = entry({
type: 'assistant',
message: {
type: 'message' as const,
role: 'assistant' as const,
model: 'claude-opus-4-6',
usage: { input_tokens: 10, output_tokens: 10 },
content: [
{ type: 'tool_use', id: 'tu', name: 'Agent', input: { subagent_type: 'reviewer'.repeat(50), prompt: 'big' } },
],
},
})
const c = compactEntry(raw)
const msg = c.message as { content: Array<{ input: Record<string, unknown> }> }
expect((msg.content[0]!.input['subagent_type'] as string).length).toBe(200)
expect(msg.content[0]!.input['prompt']).toBeUndefined()
})
it('handles entry with no message field', () => {
const raw = entry({ type: 'system', timestamp: 't1', cwd: '/x' })
const c = compactEntry(raw)
expect(c.type).toBe('system')
expect(c.timestamp).toBe('t1')
expect(c.message).toBeUndefined()
})
it('handles assistant message with no usage (non-standard)', () => {
const raw = entry({
type: 'assistant',
message: {
type: 'message' as const,
role: 'assistant' as const,
model: 'claude-opus-4-6',
content: [{ type: 'text', text: 'response' }],
},
})
const c = compactEntry(raw)
expect(c.message).toBeUndefined()
})
it('handles unexpected message role (neither user nor assistant)', () => {
const raw = entry({
type: 'system',
message: { role: 'system' as never, content: 'sys prompt' },
})
const c = compactEntry(raw)
expect(c.message).toBeUndefined()
})
it('tolerates null elements in user content array', () => {
const raw = entry({
type: 'user',
message: {
role: 'user' as const,
content: [null, undefined, { type: 'text', text: 'ok' }, 42, { type: 'text' }] as never,
},
})
const c = compactEntry(raw)
const content = (c.message as { content: Array<{ text: string }> }).content
expect(content).toHaveLength(1)
expect(content[0]!.text).toBe('ok')
})
it('tolerates assistant content that is not an array', () => {
const raw = entry({
type: 'assistant',
message: {
type: 'message' as const,
role: 'assistant' as const,
model: 'claude-opus-4-6',
usage: { input_tokens: 10, output_tokens: 10 },
content: 'not an array' as never,
},
})
const c = compactEntry(raw)
const msg = c.message as { content: unknown[] }
expect(msg.content).toEqual([])
})
it('tolerates null elements in assistant content array', () => {
const raw = entry({
type: 'assistant',
message: {
type: 'message' as const,
role: 'assistant' as const,
model: 'claude-opus-4-6',
usage: { input_tokens: 10, output_tokens: 10 },
content: [null, { type: 'tool_use', id: 'tu1', name: 'Read', input: {} }, undefined] as never,
},
})
const c = compactEntry(raw)
const msg = c.message as { content: Array<{ name: string }> }
expect(msg.content).toHaveLength(1)
expect(msg.content[0]!.name).toBe('Read')
})
it('memory reduction: compacted entry is much smaller than raw', () => {
const hugeContent = Array.from({ length: 20 }, (_, i) => ({
type: i % 2 === 0 ? 'text' : 'tool_result',
text: 'x'.repeat(100_000),
content: 'y'.repeat(100_000),
}))
const raw = entry({
type: 'assistant',
timestamp: '2026-01-01T00:00:00',
message: {
type: 'message' as const,
role: 'assistant' as const,
model: 'claude-opus-4-6',
id: 'msg_1',
usage: { input_tokens: 1000, output_tokens: 500 },
content: hugeContent as never,
},
toolResult: { content: 'z'.repeat(500_000) },
})
const rawSize = JSON.stringify(raw).length
const compacted = compactEntry(raw)
const compactedSize = JSON.stringify(compacted).length
expect(rawSize).toBeGreaterThan(2_000_000)
expect(compactedSize).toBeLessThan(500)
})
})
// Claude Code before 2.1.307 left subagent messages on the first streamed
// chunk's output_tokens with a null stop_reason; the output is estimated from
// the content instead.
describe('placeholder usage on pre-2.1.307 subagent messages', () => {
const text = 'x'.repeat(3_000)
function row(opts: { id?: string; version?: string; isSidechain?: boolean; stop_reason?: string | null; output?: number; content?: unknown[] } = {}): string {
return JSON.stringify({
type: 'assistant',
timestamp: '2026-06-01T10:00:00.000Z',
sessionId: 's1',
isSidechain: opts.isSidechain ?? true,
version: opts.version ?? '2.1.300',
message: {
id: opts.id ?? 'msg-1',
type: 'message',
role: 'assistant',
model: 'claude-sonnet-4-5',
content: opts.content ?? [{ type: 'text', text }],
stop_reason: opts.stop_reason ?? null,
usage: { input_tokens: 3, output_tokens: opts.output ?? 8, cache_read_input_tokens: 20_000 },
},
})
}
const calls = (...lines: string[]) =>
dedupeStreamingMessageIds(lines.map(l => compactEntry(parseJsonlLine(l)!))).map(e => parseApiCall(e)!)
it('estimates the output of a row that never got a stop_reason', () => {
const [call] = calls(row())
expect(call!.usage.outputTokens).toBe(750)
expect(call!.usage.inputTokens).toBe(3)
expect(call!.usage.cacheReadInputTokens).toBe(20_000)
expect(call!.isEstimated).toBe(true)
})
it('keeps the recorded output when the row has a stop_reason', () => {
const [call] = calls(row({ stop_reason: 'end_turn' }))
expect(call!.usage.outputTokens).toBe(8)
expect(call!.isEstimated).toBeUndefined()
})
it('keeps the recorded output from 2.1.307 on', () => {
expect(calls(row({ version: '2.1.307' }))[0]!.usage.outputTokens).toBe(8)
})
it('keeps the recorded output of a main-thread row', () => {
expect(calls(row({ isSidechain: false }))[0]!.usage.outputTokens).toBe(8)
})
it('resolves a placeholder row followed by a final row to the final row', () => {
const [call] = calls(row(), row({ stop_reason: 'end_turn', output: 40 }))
expect(call!.usage.outputTokens).toBe(40)
expect(call!.isEstimated).toBeUndefined()
})
it('sums the content of every row of a placeholder message', () => {
const write = { type: 'tool_use', id: 't1', name: 'Write', input: { file_path: '/a.ts', content: 'y'.repeat(4_000) } }
const [call] = calls(row(), row({ content: [write] }))
expect(call!.usage.outputTokens).toBe(Math.ceil((3_000 + JSON.stringify(write.input).length) / 4))
expect(call!.tools).toEqual(['Write'])
})
it('estimates a large row the same way as a small one', () => {
const write = { type: 'tool_use', id: 't1', name: 'Write', input: { file_path: '/a.ts', content: 'y'.repeat(40_000) } }
const [call] = calls(row({ content: [write] }))
expect(call!.usage.outputTokens).toBe(Math.ceil(JSON.stringify(write.input).length / 4))
expect(call!.isEstimated).toBe(true)
})
})