mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-08-25 00:14:40 +00:00
fda7e8024d
Reasoning tokens are a subset of output_tokens for OpenAI models, not an extra bucket: on a 1,396-rollout corpus all 134,316 token_count events carrying a total satisfy input + output == total. CodeBurn added reasoning_output_tokens on top when pricing a codex call, in the cache-rehydration re-price, and in the models/audit display sums. That overstated codex cost by $166.03 (3.5%) and displayed output tokens by 34.6% on that corpus. Both cost sites and the display sums now go through one shared billableOutputTokens() so a cold parse and a warm read cannot drift apart. cache_write_input_tokens (codex PR #33454) was never read and cacheCreationInputTokens was hardcoded to 0. It is now carved out of the uncached-input bucket and clamped to it, but routed to the cache-write bucket ONLY when the pricing source publishes an explicit cache-write rate. buildCosts fabricates 1.25x input when a source omits one, which is correct for Anthropic and would have invented a surcharge OpenAI never charged on gpt-5.5 / 5.4 / 5.3-codex / gpt-5. ModelCosts now carries cacheWriteCostIsExplicit so that distinction survives getModelCosts. A cost change invalidates persisted output: codex-results.json v10 -> v11 (stores costUSD verbatim), the codex parse version moves (the token-bucket change does not self-heal on read), and the daily cache goes 20 -> 23 (21 is claimed by the #946 landing branch and 22 by PR #1056). The upgrade-path corpus asserts codex tokens and calls exactly and reports the repricing. Closes #1075
115 lines
3.8 KiB
TypeScript
115 lines
3.8 KiB
TypeScript
import { describe, it, expect } from 'vitest'
|
|
|
|
import { aggregateAudit } from '../src/audit-report.js'
|
|
import type {
|
|
ProjectSummary,
|
|
SessionSummary,
|
|
ClassifiedTurn,
|
|
ParsedApiCall,
|
|
TokenUsage,
|
|
TaskCategory,
|
|
} from '../src/types.js'
|
|
|
|
function emptyTokens(): TokenUsage {
|
|
return {
|
|
inputTokens: 0,
|
|
outputTokens: 0,
|
|
cacheCreationInputTokens: 0,
|
|
cacheReadInputTokens: 0,
|
|
cachedInputTokens: 0,
|
|
reasoningTokens: 0,
|
|
webSearchRequests: 0,
|
|
}
|
|
}
|
|
|
|
function makeCall(usage: Partial<TokenUsage>, costUSD: number, model = 'unknown-model-xyz', provider = 'claude'): ParsedApiCall {
|
|
return {
|
|
provider,
|
|
model,
|
|
usage: { ...emptyTokens(), ...usage },
|
|
costUSD,
|
|
tools: [],
|
|
mcpTools: [],
|
|
skills: [],
|
|
hasAgentSpawn: false,
|
|
hasPlanMode: false,
|
|
speed: 'standard',
|
|
timestamp: '2026-05-09T00:00:00.000Z',
|
|
bashCommands: [],
|
|
deduplicationKey: `${provider}-${model}-${costUSD}-${usage.inputTokens ?? 0}-${usage.cachedInputTokens ?? 0}`,
|
|
}
|
|
}
|
|
|
|
function makeProject(calls: ParsedApiCall[]): ProjectSummary {
|
|
const turn: ClassifiedTurn = {
|
|
userMessage: 't',
|
|
assistantCalls: calls,
|
|
timestamp: '2026-05-09T00:00:00.000Z',
|
|
sessionId: 's1',
|
|
category: 'feature' as TaskCategory,
|
|
retries: 0,
|
|
hasEdits: false,
|
|
}
|
|
const session: SessionSummary = {
|
|
sessionId: 's1',
|
|
project: 'p',
|
|
firstTimestamp: '2026-05-09T00:00:00.000Z',
|
|
lastTimestamp: '2026-05-09T00:00:00.000Z',
|
|
totalCostUSD: 0,
|
|
totalInputTokens: 0,
|
|
totalOutputTokens: 0,
|
|
totalCacheReadTokens: 0,
|
|
totalCacheWriteTokens: 0,
|
|
apiCalls: 0,
|
|
turns: [turn],
|
|
modelBreakdown: {},
|
|
toolBreakdown: {},
|
|
mcpBreakdown: {},
|
|
bashBreakdown: {},
|
|
categoryBreakdown: {} as SessionSummary['categoryBreakdown'],
|
|
skillBreakdown: {},
|
|
}
|
|
return { project: 'p', projectPath: 'p', sessions: [session], totalCostUSD: 0, totalApiCalls: 0 }
|
|
}
|
|
|
|
describe('aggregateAudit', () => {
|
|
it('keeps raw fields and exposes codeburn normalizations', async () => {
|
|
const anthropicCall = makeCall({ inputTokens: 100, outputTokens: 50, reasoningTokens: 10, cacheReadInputTokens: 200 }, 0.5)
|
|
const openaiCall = makeCall({ inputTokens: 100, outputTokens: 50, cachedInputTokens: 300 }, 0.5)
|
|
const rows = await aggregateAudit([makeProject([anthropicCall, openaiCall])])
|
|
|
|
expect(rows).toHaveLength(1)
|
|
const r = rows[0]!
|
|
// raw fields are summed untouched
|
|
expect(r.raw.inputTokens).toBe(200)
|
|
expect(r.raw.outputTokens).toBe(100)
|
|
expect(r.raw.reasoningTokens).toBe(10)
|
|
expect(r.raw.cacheReadInputTokens).toBe(200)
|
|
expect(r.raw.cachedInputTokens).toBe(300)
|
|
// Reasoning does NOT fold into output for claude or codex: both bill it
|
|
// as part of output_tokens already, so adding it would double-count
|
|
// (#1075). Providers that report reasoning as a separate bucket still get
|
|
// the additive treatment - see tests/codex-pricing-1075.test.ts.
|
|
expect(r.displayed.outputTokens).toBe(100)
|
|
// cache read is the SUM of per-call max(anthropic, openai), not max of sums
|
|
expect(r.displayed.cacheReadTokens).toBe(500)
|
|
// attributed cost is preserved exactly
|
|
expect(r.attributedCostUSD).toBeCloseTo(1.0)
|
|
})
|
|
|
|
it('returns null rates and zero component cost for an unpriced model', async () => {
|
|
const rows = await aggregateAudit([makeProject([makeCall({ inputTokens: 1000 }, 0, 'definitely-not-a-real-model-zzz')])])
|
|
expect(rows).toHaveLength(1)
|
|
expect(rows[0]!.rates).toBeNull()
|
|
expect(rows[0]!.cost.recomputedTotalUSD).toBe(0)
|
|
})
|
|
|
|
it('splits buckets by (provider, model)', async () => {
|
|
const rows = await aggregateAudit([makeProject([
|
|
makeCall({ inputTokens: 10 }, 0.1, 'model-a', 'claude'),
|
|
makeCall({ inputTokens: 20 }, 0.2, 'model-b', 'claude'),
|
|
makeCall({ inputTokens: 30 }, 0.3, 'model-a', 'codex'),
|
|
])])
|
|
expect(rows).toHaveLength(3)
|
|
})
|
|
})
|