vrr/codeburn
2
0
Fork 0
mirror of https://github.com/AgentSeal/codeburn.git synced 2026-08-25 08:22:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

fix(models): merge models report rows by canonical id, not display name

Browse source 
Maintainer review on #1053: display-name keys collide for distinct
SKUs (GPT-5 / GPT-5.3 Codex / Kimi K2 Thinking / Opus 4.6). Key on
the alias-resolved canonical id, keep the first raw id, and partial-sum
Codex credits when a merge mixes rated and unrated buckets.
This commit is contained in:
Aditya Vikram Singh 2026-08-21 17:26:40 +05:30
parent 9e361175bb
commit 912281e9b6
4 changed files with 122 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

46
src/models-report.ts
View file

@ -3,7 +3,7 @@ import stripAnsi from 'strip-ansi'
import { codexCredits } from './codex-credits.js'
import { formatCost, formatTokens } from './format.js'
import { fallbackRawModelDisplayName, getShortModelName, sanitizeModelForDisplay } from './models.js'
import { fallbackRawModelDisplayName, getShortModelName, resolveCanonicalModelId, sanitizeModelForDisplay } from './models.js'
import { getProvider } from './providers/index.js'
import { CATEGORY_LABELS, type ProjectSummary, type TaskCategory } from './types.js'
@ -28,8 +28,12 @@ export type ModelReportRow = {
savingsBaselineModel: string
calls: number
/// Codex credit consumption (issues #408/#495). null for non-Codex models or
/// Codex models without a known credit rate.
/// Codex models without a known credit rate. A merged row that mixed rated
/// and unrated buckets stores the partial sum of the rated ones.
credits: number | null
/// True when `credits` is a partial sum because some contributing buckets
/// had no known credit rate.
creditsIncomplete?: boolean
topCategory?: TaskCategory
topCategoryCost?: number
topCategoryShare?: number
@ -74,9 +78,9 @@ function bucketKey(provider: string, model: string, category: TaskCategory | nul
/// Walks every parsed turn, attributes each assistant call to a
/// (provider, raw-model, category, agent) bucket, then merges buckets that
/// resolve to the same provider + display name. Returned rows are keyed by
/// (provider, display name) by default, plus category under `byTask` or agent
/// under `byAgent`.
/// resolve to the same provider + alias-resolved canonical id. Display names
/// stay cosmetic. Returned rows are keyed by (provider, canonical id) by
/// default, plus category under `byTask` or agent under `byAgent`.
///
/// Default view: rows sorted by cost descending.
/// byTask / byAgent view: rows grouped by (provider, model) so the renderer can
@ -175,11 +179,13 @@ export async function aggregateModels(projects: ProjectSummary[], opts: Aggregat
for (const bucket of buckets.values()) {
const meta = await resolveProvider(bucket.provider)
const modelDisplayName = meta.formatModel(bucket.model)
const resolvedKey = bucketKey(bucket.provider, modelDisplayName, bucket.category, bucket.agentType)
const displayModelKey = `${bucket.provider} ${modelDisplayName}`
const canonicalId = resolveCanonicalModelId(bucket.model)
const resolvedKey = bucketKey(bucket.provider, canonicalId, bucket.category, bucket.agentType)
const foldKey = `${bucket.provider} ${canonicalId}`
const total = bucket.inputTokens + bucket.outputTokens + bucket.cacheWriteTokens + bucket.cacheReadTokens
// Credits are per raw id (aliases can have different rates). Sum only when
// every contributing bucket has a known rate; otherwise the merged row is null.
// Credits are per raw id (aliases can have different rates). Sum the
// rated buckets and flag the row incomplete when any contributor is
// unrated — nulling the whole merge would zero a real menubar total.
const bucketCredits = bucket.provider === 'codex'
? codexCredits(bucket.model, {
inputTokens: bucket.inputTokens,
@ -203,10 +209,11 @@ export async function aggregateModels(projects: ProjectSummary[], opts: Aggregat
existing.costUSD += bucket.costUSD
existing.savingsUSD += bucket.savingsUSD
existing.calls += bucket.calls
if (bucket.model < existing.model) existing.model = bucket.model
existing.savingsBaselineModel = resolvedBaseline
if (existing.credits === null || bucketCredits === null) existing.credits = null
else existing.credits += bucketCredits
const existingRated = existing.credits !== null
const incomingRated = bucketCredits !== null
if (incomingRated) existing.credits = (existing.credits ?? 0) + bucketCredits
if (existingRated !== incomingRated) existing.creditsIncomplete = true
} else {
rowsByKey.set(resolvedKey, {
provider: bucket.provider,
@ -233,18 +240,18 @@ export async function aggregateModels(projects: ProjectSummary[], opts: Aggregat
foldedRawSeen.add(rawKey)
const rawCat = perModelCategoryCost.get(rawKey)
if (rawCat) {
let folded = foldedCategoryCost.get(displayModelKey)
let folded = foldedCategoryCost.get(foldKey)
if (!folded) {
folded = new Map()
foldedCategoryCost.set(displayModelKey, folded)
foldedCategoryCost.set(foldKey, folded)
}
for (const [cat, cost] of rawCat) {
folded.set(cat, (folded.get(cat) ?? 0) + cost)
}
}
foldedTotalCost.set(
displayModelKey,
(foldedTotalCost.get(displayModelKey) ?? 0) + (perModelTotalCost.get(rawKey) ?? 0),
foldKey,
(foldedTotalCost.get(foldKey) ?? 0) + (perModelTotalCost.get(rawKey) ?? 0),
)
}
}
@ -252,7 +259,7 @@ export async function aggregateModels(projects: ProjectSummary[], opts: Aggregat
const rows = [...rowsByKey.values()]
for (const row of rows) {
if (opts.byTask || opts.byAgent) continue
const perCat = foldedCategoryCost.get(`${row.provider} ${row.modelDisplayName}`)
const perCat = foldedCategoryCost.get(`${row.provider} ${resolveCanonicalModelId(row.model)}`)
if (!perCat || perCat.size === 0) continue
let topCat: TaskCategory = 'general'
let topCost = -1
@ -271,8 +278,8 @@ export async function aggregateModels(projects: ProjectSummary[], opts: Aggregat
if (opts.byTask || opts.byAgent) {
rows.sort((a, b) => {
const aTotal = foldedTotalCost.get(`${a.provider} ${a.modelDisplayName}`) ?? 0
const bTotal = foldedTotalCost.get(`${b.provider} ${b.modelDisplayName}`) ?? 0
const aTotal = foldedTotalCost.get(`${a.provider} ${resolveCanonicalModelId(a.model)}`) ?? 0
const bTotal = foldedTotalCost.get(`${b.provider} ${resolveCanonicalModelId(b.model)}`) ?? 0
if (aTotal !== bTotal) return bTotal - aTotal
if (a.provider !== b.provider) return a.provider.localeCompare(b.provider)
if (a.modelDisplayName !== b.modelDisplayName) return a.modelDisplayName.localeCompare(b.modelDisplayName)
@ -657,6 +664,7 @@ export function renderJson(rows: ModelReportRow[]): string {
savingsUSD: r.savingsUSD,
savingsBaselineModel: r.savingsBaselineModel,
credits: r.credits,
creditsIncomplete: r.creditsIncomplete === true,
})),
null,
2,

13
src/models.ts
View file

@ -616,6 +616,19 @@ function getCanonicalName(model: string): string {
)
}
/// Alias-resolved identity for report merge. Display names stay cosmetic —
/// prefix matches in SHORT_NAMES must not fold distinct SKUs into one row.
/// Path-form ids (`accounts/fireworks/models/<slug>`, `cline-pass/<slug>`)
/// peel to the leaf so they share a bucket with the bare slug.
export function resolveCanonicalModelId(model: string): string {
const viaUser = Object.hasOwn(userAliases, model) ? userAliases[model]! : model
const aliased = resolveAlias(getCanonicalName(viaUser))
if (!aliased.includes('/')) return aliased
const leaf = aliased.slice(aliased.lastIndexOf('/') + 1)
if (!leaf) return aliased
return resolveAlias(getCanonicalName(leaf))
}
// Namespaces the pricing catalog itself uses, plus the ones below. An unknown
// `provider/model` must stay unpriced — do not treat `/` as authority. Derived
// rather than hand-listed so a vendor LiteLLM already knows (`x-ai/`, `qwen/`,

72
tests/models-report.test.ts
View file

@ -8,6 +8,7 @@ import chalk from 'chalk'
import stripAnsi from 'strip-ansi'
import { aggregateModels, renderTable, renderMarkdown, renderJson, renderCsv, type ModelReportRow } from '../src/models-report.js'
import { setModelAliases } from '../src/models.js'
import type {
ProjectSummary,
SessionSummary,
@ -169,6 +170,32 @@ describe('aggregateModels', () => {
expect(byKey['codex:gpt-5.5']!.credits).toBeCloseTo(887.5, 6)
expect(byKey['codex:gpt-5']!.credits).toBeNull()
expect(byKey['claude:claude-sonnet-4-6']!.credits).toBeNull()
expect(byKey['codex:gpt-5.5']!.creditsIncomplete).toBeFalsy()
})
it('partial-sums Codex credits when a merge mixes rated and unrated ids', async () => {
setModelAliases({ 'codex-house-sku': 'gpt-5.5' })
try {
const rows = await aggregateModels([makeProject([
makeTurn('feature', [
makeCall({ provider: 'codex', model: 'gpt-5.5', input: 0, output: 1_000_000, costUSD: 9 }),
]),
makeTurn('feature', [
makeCall({ provider: 'codex', model: 'codex-house-sku', input: 0, output: 1_000_000, costUSD: 9 }),
]),
])])
expect(rows).toHaveLength(1)
expect(rows[0]!.model).toBe('gpt-5.5')
// gpt-5.5 output is 750 credits/M; the aliased house SKU has no rate.
expect(rows[0]!.credits).toBeCloseTo(750, 6)
expect(rows[0]!.creditsIncomplete).toBe(true)
expect(rows[0]!.calls).toBe(2)
const parsed = JSON.parse(renderJson(rows))
expect(parsed[0].credits).toBeCloseTo(750, 6)
expect(parsed[0].creditsIncomplete).toBe(true)
} finally {
setModelAliases({})
}
})
it('includes credits in the JSON output', async () => {
@ -210,20 +237,22 @@ describe('aggregateModels', () => {
expect(rows[0]!.model).toBe('accounts/fireworks/models/kimi-k2p6')
})
it('merges two raw ids that resolve to the same provider + display name', async () => {
it('merges two raw ids that resolve to the same alias-resolved canonical id', async () => {
const rows = await aggregateModels([makeProject([
makeTurn('testing', [makeCall({
provider: 'cline-cli', model: 'accounts/fireworks/models/glm-5p2',
input: 800, output: 67, costUSD: 0.246,
})]),
makeTurn('conversation', [makeCall({
provider: 'cline-cli', model: 'GLM-5.2',
provider: 'cline-cli', model: 'glm-5p2',
input: 15, output: 1, costUSD: 0.019,
})]),
])])
expect(rows).toHaveLength(1)
expect(rows[0]!.provider).toBe('cline-cli')
expect(rows[0]!.modelDisplayName).toBe('GLM-5.2')
// First-seen raw id, not the lexicographically-smallest of the merge.
expect(rows[0]!.model).toBe('accounts/fireworks/models/glm-5p2')
expect(rows[0]!.inputTokens).toBe(815)
expect(rows[0]!.outputTokens).toBe(68)
expect(rows[0]!.costUSD).toBeCloseTo(0.265, 6)
@ -232,24 +261,53 @@ describe('aggregateModels', () => {
expect(rows[0]!.topCategoryShare).toBeCloseTo(0.246 / 0.265, 3)
})
it('does not pick the lexicographically-smallest raw id after a merge', async () => {
const rows = await aggregateModels([makeProject([
makeTurn('feature', [makeCall({ provider: 'codex', model: 'kimi-k3', costUSD: 2 })]),
makeTurn('feature', [makeCall({ provider: 'codex', model: 'k3', costUSD: 1 })]),
])])
expect(rows).toHaveLength(1)
expect(rows[0]!.model).toBe('kimi-k3')
expect(rows[0]!.modelDisplayName).toBe('Kimi K3')
expect(rows[0]!.calls).toBe(2)
})
it('does not merge same-provider ids that only share a display name', async () => {
const rows = await aggregateModels([makeProject([
makeTurn('feature', [makeCall({ provider: 'codex', model: 'gpt-5', costUSD: 3 })]),
makeTurn('feature', [makeCall({ provider: 'codex', model: 'gpt-5-untracked-xyz', costUSD: 1 })]),
makeTurn('feature', [makeCall({ provider: 'cline-cli', model: 'glm-5p2', costUSD: 1 })]),
makeTurn('feature', [makeCall({ provider: 'cline-cli', model: 'GLM-5.2', costUSD: 2 })]),
])])
const codex = rows.filter(r => r.provider === 'codex')
expect(codex).toHaveLength(2)
expect(codex.every(r => r.modelDisplayName === 'GPT-5')).toBe(true)
expect(new Set(codex.map(r => r.model))).toEqual(new Set(['gpt-5', 'gpt-5-untracked-xyz']))
const cline = rows.filter(r => r.provider === 'cline-cli')
expect(cline).toHaveLength(2)
expect(cline.every(r => r.modelDisplayName === 'GLM-5.2')).toBe(true)
expect(new Set(cline.map(r => r.model))).toEqual(new Set(['glm-5p2', 'GLM-5.2']))
})
it('clears a merged savings baseline when three raw ids disagree', async () => {
const rows = await aggregateModels([makeProject([
makeTurn('feature', [makeCall({
provider: 'cline-cli', model: 'glm-5p2',
provider: 'codex', model: 'k3',
costUSD: 1, savingsUSD: 2, savingsBaselineModel: 'gpt-4o',
})]),
makeTurn('feature', [makeCall({
provider: 'cline-cli', model: 'GLM-5.2',
provider: 'codex', model: 'kimi-k3',
costUSD: 1, savingsUSD: 2, savingsBaselineModel: 'claude-sonnet-4-6',
})]),
makeTurn('feature', [makeCall({
provider: 'cline-cli', model: 'accounts/fireworks/models/glm-5p2',
provider: 'codex', model: 'k3-agent',
costUSD: 1, savingsUSD: 2, savingsBaselineModel: 'gpt-5',
})]),
])])
expect(rows).toHaveLength(1)
expect(rows[0]!.provider).toBe('cline-cli')
expect(rows[0]!.modelDisplayName).toBe('GLM-5.2')
expect(rows[0]!.provider).toBe('codex')
expect(rows[0]!.modelDisplayName).toBe('Kimi K3')
expect(rows[0]!.savingsUSD).toBe(6)
expect(rows[0]!.savingsBaselineModel).toBe('')
expect(rows[0]!.calls).toBe(3)

17
tests/models.test.ts
View file

@ -7,6 +7,7 @@ import {
findUnpricedModels,
getModelCosts,
getShortModelName,
resolveCanonicalModelId,
calculateCost,
loadPricing,
setModelAliases,
@ -123,6 +124,22 @@ describe('getModelCosts', () => {
})
})
describe('resolveCanonicalModelId', () => {
it('aliases, peels path-form ids, and leaves display-only collisions distinct', () => {
expect(resolveCanonicalModelId('k3')).toBe('kimi-k3')
expect(resolveCanonicalModelId('k3-agent')).toBe('kimi-k3')
expect(resolveCanonicalModelId('kimi-k3')).toBe('kimi-k3')
expect(resolveCanonicalModelId('accounts/fireworks/models/glm-5p2')).toBe('glm-5p2')
expect(resolveCanonicalModelId('glm-5p2')).toBe('glm-5p2')
expect(resolveCanonicalModelId('GLM-5.2')).toBe('glm-5p1')
expect(resolveCanonicalModelId('gpt-5-fast')).toBe('gpt-5')
expect(resolveCanonicalModelId('gpt-5-untracked-xyz')).toBe('gpt-5-untracked-xyz')
expect(resolveCanonicalModelId('claude-opus-4.6')).toBe('claude-opus-4-6')
expect(resolveCanonicalModelId('kimi-code')).toBe('kimi-k2-thinking')
expect(resolveCanonicalModelId('cline-pass/kimi-k3')).toBe('kimi-k3')
})
})
describe('getShortModelName', () => {
it('maps gpt-4o-mini correctly (not gpt-4o)', () => {
expect(getShortModelName('gpt-4o-mini-2024-07-18')).toBe('GPT-4o Mini')