mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-08-25 08:22:42 +00:00
fix(models): resolve and merge raw ids in models report
codeburn models asked each provider for a label and bucketed by raw id, so gpt-5.6-* and Fireworks path-form ids stayed raw and two ids that share a display name became two rows. Keep provider-first labels. Fall back to the global short-name table on a local miss, then merge by provider + display name.
This commit is contained in:
parent
7862aabd47
commit
28be2472ce
8 changed files with 174 additions and 60 deletions
|
|
@ -1,4 +1,4 @@
|
|||
import { getModelCosts, sanitizeModelForDisplay, type ModelCosts } from './models.js'
|
||||
import { fallbackRawModelDisplayName, getModelCosts, getShortModelName, sanitizeModelForDisplay, type ModelCosts } from './models.js'
|
||||
import { getProvider } from './providers/index.js'
|
||||
import { formatCost, formatTokens } from './format.js'
|
||||
import { renderTable, type TableColumn } from './text-table.js'
|
||||
|
|
@ -112,8 +112,8 @@ export async function aggregateAudit(projects: ProjectSummary[]): Promise<AuditR
|
|||
const entry = {
|
||||
displayName: p?.displayName ?? name,
|
||||
formatModel: p
|
||||
? (m: string) => sanitizeModelForDisplay(p.modelDisplayName(m))
|
||||
: sanitizeModelForDisplay,
|
||||
? (m: string) => sanitizeModelForDisplay(fallbackRawModelDisplayName(p.modelDisplayName(m), m))
|
||||
: (m: string) => sanitizeModelForDisplay(getShortModelName(m)),
|
||||
}
|
||||
providerCache.set(name, entry)
|
||||
return entry
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import stripAnsi from 'strip-ansi'
|
|||
|
||||
import { codexCredits } from './codex-credits.js'
|
||||
import { formatCost, formatTokens } from './format.js'
|
||||
import { sanitizeModelForDisplay } from './models.js'
|
||||
import { fallbackRawModelDisplayName, getShortModelName, sanitizeModelForDisplay } from './models.js'
|
||||
import { getProvider } from './providers/index.js'
|
||||
import { CATEGORY_LABELS, type ProjectSummary, type TaskCategory } from './types.js'
|
||||
|
||||
|
|
@ -73,9 +73,10 @@ function bucketKey(provider: string, model: string, category: TaskCategory | nul
|
|||
}
|
||||
|
||||
/// Walks every parsed turn, attributes each assistant call to a
|
||||
/// (provider, model, category, agent) bucket, and returns rows keyed by
|
||||
/// (provider, model) by default, (provider, model, category) under `byTask`, or
|
||||
/// (provider, model, agent) under `byAgent`.
|
||||
/// (provider, raw-model, category, agent) bucket, then merges buckets that
|
||||
/// resolve to the same provider + display name. Returned rows are keyed by
|
||||
/// (provider, display name) by default, plus category under `byTask` or agent
|
||||
/// under `byAgent`.
|
||||
///
|
||||
/// Default view: rows sorted by cost descending.
|
||||
/// byTask / byAgent view: rows grouped by (provider, model) so the renderer can
|
||||
|
|
@ -155,74 +156,122 @@ export async function aggregateModels(projects: ProjectSummary[], opts: Aggregat
|
|||
const entry = {
|
||||
displayName: p?.displayName ?? name,
|
||||
formatModel: p
|
||||
? (m: string) => sanitizeModelForDisplay(p.modelDisplayName(m))
|
||||
: sanitizeModelForDisplay,
|
||||
? (m: string) => sanitizeModelForDisplay(fallbackRawModelDisplayName(p.modelDisplayName(m), m))
|
||||
: (m: string) => sanitizeModelForDisplay(getShortModelName(m)),
|
||||
}
|
||||
providerCache.set(name, entry)
|
||||
return entry
|
||||
}
|
||||
|
||||
const rows: ModelReportRow[] = []
|
||||
const rowsByKey = new Map<string, ModelReportRow>()
|
||||
const foldedCategoryCost = new Map<string, Map<CategoryKey, number>>()
|
||||
const foldedTotalCost = new Map<string, number>()
|
||||
const foldedRawSeen = new Set<string>()
|
||||
|
||||
for (const bucket of buckets.values()) {
|
||||
const meta = await resolveProvider(bucket.provider)
|
||||
const modelDisplayName = meta.formatModel(bucket.model)
|
||||
const resolvedKey = bucketKey(bucket.provider, modelDisplayName, bucket.category, bucket.agentType)
|
||||
const displayModelKey = `${bucket.provider} ${modelDisplayName}`
|
||||
const total = bucket.inputTokens + bucket.outputTokens + bucket.cacheWriteTokens + bucket.cacheReadTokens
|
||||
const row: ModelReportRow = {
|
||||
provider: bucket.provider,
|
||||
providerDisplayName: meta.displayName,
|
||||
model: bucket.model,
|
||||
modelDisplayName: meta.formatModel(bucket.model),
|
||||
category: bucket.category,
|
||||
agentType: bucket.agentType,
|
||||
inputTokens: bucket.inputTokens,
|
||||
outputTokens: bucket.outputTokens,
|
||||
cacheWriteTokens: bucket.cacheWriteTokens,
|
||||
cacheReadTokens: bucket.cacheReadTokens,
|
||||
totalTokens: total,
|
||||
costUSD: bucket.costUSD,
|
||||
savingsUSD: bucket.savingsUSD,
|
||||
savingsBaselineModel: bucket.savingsBaselineModel,
|
||||
calls: bucket.calls,
|
||||
// outputTokens already includes reasoning (folded in above), and for Codex
|
||||
// inputTokens is non-cached with cacheReadTokens holding cached input, which
|
||||
// is exactly what the credit rates expect.
|
||||
credits: bucket.provider === 'codex'
|
||||
? codexCredits(bucket.model, {
|
||||
inputTokens: bucket.inputTokens,
|
||||
cachedReadTokens: bucket.cacheReadTokens,
|
||||
outputTokens: bucket.outputTokens,
|
||||
})
|
||||
: null,
|
||||
// Credits are per raw id (aliases can have different rates). Sum only when
|
||||
// every contributing bucket has a known rate; otherwise the merged row is null.
|
||||
const bucketCredits = bucket.provider === 'codex'
|
||||
? codexCredits(bucket.model, {
|
||||
inputTokens: bucket.inputTokens,
|
||||
cachedReadTokens: bucket.cacheReadTokens,
|
||||
outputTokens: bucket.outputTokens,
|
||||
})
|
||||
: null
|
||||
|
||||
const existing = rowsByKey.get(resolvedKey)
|
||||
if (existing) {
|
||||
existing.inputTokens += bucket.inputTokens
|
||||
existing.outputTokens += bucket.outputTokens
|
||||
existing.cacheWriteTokens += bucket.cacheWriteTokens
|
||||
existing.cacheReadTokens += bucket.cacheReadTokens
|
||||
existing.totalTokens += total
|
||||
existing.costUSD += bucket.costUSD
|
||||
existing.savingsUSD += bucket.savingsUSD
|
||||
existing.calls += bucket.calls
|
||||
if (bucket.model < existing.model) existing.model = bucket.model
|
||||
if (existing.savingsBaselineModel && bucket.savingsBaselineModel
|
||||
&& existing.savingsBaselineModel !== bucket.savingsBaselineModel) {
|
||||
existing.savingsBaselineModel = ''
|
||||
} else if (!existing.savingsBaselineModel && bucket.savingsBaselineModel) {
|
||||
existing.savingsBaselineModel = bucket.savingsBaselineModel
|
||||
}
|
||||
if (existing.credits === null || bucketCredits === null) existing.credits = null
|
||||
else existing.credits += bucketCredits
|
||||
} else {
|
||||
rowsByKey.set(resolvedKey, {
|
||||
provider: bucket.provider,
|
||||
providerDisplayName: meta.displayName,
|
||||
model: bucket.model,
|
||||
modelDisplayName,
|
||||
category: bucket.category,
|
||||
agentType: bucket.agentType,
|
||||
inputTokens: bucket.inputTokens,
|
||||
outputTokens: bucket.outputTokens,
|
||||
cacheWriteTokens: bucket.cacheWriteTokens,
|
||||
cacheReadTokens: bucket.cacheReadTokens,
|
||||
totalTokens: total,
|
||||
costUSD: bucket.costUSD,
|
||||
savingsUSD: bucket.savingsUSD,
|
||||
savingsBaselineModel: bucket.savingsBaselineModel,
|
||||
calls: bucket.calls,
|
||||
credits: bucketCredits,
|
||||
})
|
||||
}
|
||||
|
||||
if (!opts.byTask && !opts.byAgent) {
|
||||
const perCat = perModelCategoryCost.get(`${bucket.provider} ${bucket.model}`)
|
||||
if (perCat && perCat.size > 0) {
|
||||
let topCat: TaskCategory = 'general'
|
||||
let topCost = -1
|
||||
let totalCost = 0
|
||||
for (const [cat, cost] of perCat.entries()) {
|
||||
totalCost += cost
|
||||
if (cost > topCost) {
|
||||
topCost = cost
|
||||
topCat = cat
|
||||
}
|
||||
const rawKey = `${bucket.provider} ${bucket.model}`
|
||||
if (!foldedRawSeen.has(rawKey)) {
|
||||
foldedRawSeen.add(rawKey)
|
||||
const rawCat = perModelCategoryCost.get(rawKey)
|
||||
if (rawCat) {
|
||||
let folded = foldedCategoryCost.get(displayModelKey)
|
||||
if (!folded) {
|
||||
folded = new Map()
|
||||
foldedCategoryCost.set(displayModelKey, folded)
|
||||
}
|
||||
row.topCategory = topCat
|
||||
row.topCategoryCost = topCost
|
||||
row.topCategoryShare = totalCost > 0 ? topCost / totalCost : 0
|
||||
for (const [cat, cost] of rawCat) {
|
||||
folded.set(cat, (folded.get(cat) ?? 0) + cost)
|
||||
}
|
||||
}
|
||||
foldedTotalCost.set(
|
||||
displayModelKey,
|
||||
(foldedTotalCost.get(displayModelKey) ?? 0) + (perModelTotalCost.get(rawKey) ?? 0),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const rows = [...rowsByKey.values()]
|
||||
for (const row of rows) {
|
||||
if (opts.byTask || opts.byAgent) continue
|
||||
const perCat = foldedCategoryCost.get(`${row.provider} ${row.modelDisplayName}`)
|
||||
if (!perCat || perCat.size === 0) continue
|
||||
let topCat: TaskCategory = 'general'
|
||||
let topCost = -1
|
||||
let totalCost = 0
|
||||
for (const [cat, cost] of perCat.entries()) {
|
||||
totalCost += cost
|
||||
if (cost > topCost) {
|
||||
topCost = cost
|
||||
topCat = cat
|
||||
}
|
||||
}
|
||||
|
||||
rows.push(row)
|
||||
row.topCategory = topCat
|
||||
row.topCategoryCost = topCost
|
||||
row.topCategoryShare = totalCost > 0 ? topCost / totalCost : 0
|
||||
}
|
||||
|
||||
if (opts.byTask || opts.byAgent) {
|
||||
rows.sort((a, b) => {
|
||||
const aTotal = perModelTotalCost.get(`${a.provider} ${a.model}`) ?? 0
|
||||
const bTotal = perModelTotalCost.get(`${b.provider} ${b.model}`) ?? 0
|
||||
const aTotal = foldedTotalCost.get(`${a.provider} ${a.modelDisplayName}`) ?? 0
|
||||
const bTotal = foldedTotalCost.get(`${b.provider} ${b.modelDisplayName}`) ?? 0
|
||||
if (aTotal !== bTotal) return bTotal - aTotal
|
||||
if (a.provider !== b.provider) return a.provider.localeCompare(b.provider)
|
||||
if (a.model !== b.model) return a.model.localeCompare(b.model)
|
||||
if (a.modelDisplayName !== b.modelDisplayName) return a.modelDisplayName.localeCompare(b.modelDisplayName)
|
||||
return (b.costUSD + b.savingsUSD) - (a.costUSD + a.savingsUSD)
|
||||
})
|
||||
} else {
|
||||
|
|
@ -482,7 +531,7 @@ export function renderTable(
|
|||
const rowEntries: RowCells[] = []
|
||||
let prevProviderModel = ''
|
||||
for (const row of rows) {
|
||||
const groupKey = `${row.provider} ${row.model}`
|
||||
const groupKey = `${row.provider} ${row.modelDisplayName}`
|
||||
const isNewGroup = !grouped || groupKey !== prevProviderModel
|
||||
prevProviderModel = groupKey
|
||||
const allCells = defaultColumns(byTask, byAgent, showSaved).map(col => {
|
||||
|
|
|
|||
|
|
@ -1105,6 +1105,14 @@ export function getShortModelName(model: string): string {
|
|||
return shortModelName(model, new Set())
|
||||
}
|
||||
|
||||
/// Provider-first display name. Local labels win (Cursor estimated suffixes,
|
||||
/// provider tables that intentionally override the global map). If the provider
|
||||
/// echoed the raw id, it missed — fall back to the global resolver instead of
|
||||
/// showing `gpt-5.6-sol` / `accounts/fireworks/models/kimi-k2p6`.
|
||||
export function fallbackRawModelDisplayName(localLabel: string, rawModel: string): string {
|
||||
return localLabel === rawModel ? getShortModelName(rawModel) : localLabel
|
||||
}
|
||||
|
||||
function shortModelName(model: string, seen: Set<string>): string {
|
||||
if (autoModelNames[model]) return autoModelNames[model]
|
||||
if (seen.has(model)) {
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import { readdir, readFile, stat } from 'fs/promises'
|
|||
import { join, basename } from 'path'
|
||||
import { homedir } from 'os'
|
||||
|
||||
import { calculateCost } from '../models.js'
|
||||
import { calculateCost, getShortModelName } from '../models.js'
|
||||
import { openDatabase, type SqliteDatabase } from '../sqlite.js'
|
||||
import { normalizeContentBlocks } from '../content-utils.js'
|
||||
import { estimateTokensFromChars } from '../token-estimate.js'
|
||||
|
|
@ -506,7 +506,7 @@ export function createCursorAgentProvider(baseDirOverride?: string): Provider {
|
|||
|
||||
modelDisplayName(model: string): string {
|
||||
if (model === 'cursor-agent-auto') return 'Cursor (auto)'
|
||||
const label = modelDisplayNames[model] ?? model
|
||||
const label = modelDisplayNames[model] ?? getShortModelName(model)
|
||||
return `${label} (est.)`
|
||||
},
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { getShortModelName } from '../models.js'
|
||||
import type { DateRange } from '../types.js'
|
||||
import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js'
|
||||
import { fetchWithTimeout } from '../fetch-utils.js'
|
||||
|
|
@ -124,7 +125,7 @@ export const vercelGateway: Provider = {
|
|||
|
||||
modelDisplayName(model: string): string {
|
||||
const slash = model.indexOf('/')
|
||||
return slash >= 0 ? model.slice(slash + 1) : model
|
||||
return getShortModelName(slash >= 0 ? model.slice(slash + 1) : model)
|
||||
},
|
||||
|
||||
toolDisplayName(rawTool: string): string {
|
||||
|
|
|
|||
|
|
@ -188,6 +188,56 @@ describe('aggregateModels', () => {
|
|||
expect(rows[0]!.cacheReadTokens).toBe(4000) // not 8000
|
||||
})
|
||||
|
||||
it('falls back from a provider local table miss to the global short name', async () => {
|
||||
const rows = await aggregateModels([makeProject([
|
||||
makeTurn('feature', [makeCall({ provider: 'cursor-agent', model: 'gpt-5.6-sol', costUSD: 2.5 })]),
|
||||
])])
|
||||
expect(rows).toHaveLength(1)
|
||||
expect(rows[0]!.modelDisplayName).toBe('GPT-5.6 Sol (est.)')
|
||||
expect(rows[0]!.model).toBe('gpt-5.6-sol')
|
||||
})
|
||||
|
||||
it('resolves Fireworks path-form ids through the global table', async () => {
|
||||
const rows = await aggregateModels([makeProject([
|
||||
makeTurn('feature', [makeCall({ provider: 'cline', model: 'accounts/fireworks/models/kimi-k2p6', costUSD: 1 })]),
|
||||
])])
|
||||
expect(rows).toHaveLength(1)
|
||||
expect(rows[0]!.modelDisplayName).toBe('Kimi K2.6')
|
||||
expect(rows[0]!.model).toBe('accounts/fireworks/models/kimi-k2p6')
|
||||
})
|
||||
|
||||
it('merges two raw ids that resolve to the same provider + display name', async () => {
|
||||
const rows = await aggregateModels([makeProject([
|
||||
makeTurn('testing', [makeCall({
|
||||
provider: 'cline-cli', model: 'accounts/fireworks/models/glm-5p2',
|
||||
input: 800, output: 67, costUSD: 0.246,
|
||||
})]),
|
||||
makeTurn('conversation', [makeCall({
|
||||
provider: 'cline-cli', model: 'GLM-5.2',
|
||||
input: 15, output: 1, costUSD: 0.019,
|
||||
})]),
|
||||
])])
|
||||
expect(rows).toHaveLength(1)
|
||||
expect(rows[0]!.provider).toBe('cline-cli')
|
||||
expect(rows[0]!.modelDisplayName).toBe('GLM-5.2')
|
||||
expect(rows[0]!.inputTokens).toBe(815)
|
||||
expect(rows[0]!.outputTokens).toBe(68)
|
||||
expect(rows[0]!.costUSD).toBeCloseTo(0.265, 6)
|
||||
expect(rows[0]!.calls).toBe(2)
|
||||
expect(rows[0]!.topCategory).toBe('testing')
|
||||
expect(rows[0]!.topCategoryShare).toBeCloseTo(0.246 / 0.265, 3)
|
||||
})
|
||||
|
||||
it('does not merge the same display name across providers', async () => {
|
||||
const rows = await aggregateModels([makeProject([
|
||||
makeTurn('feature', [makeCall({ provider: 'cline-cli', model: 'glm-5p2', costUSD: 1 })]),
|
||||
makeTurn('feature', [makeCall({ provider: 'hermes', model: 'glm-5p2', costUSD: 2 })]),
|
||||
])])
|
||||
expect(rows).toHaveLength(2)
|
||||
expect(new Set(rows.map(r => r.provider))).toEqual(new Set(['cline-cli', 'hermes']))
|
||||
expect(rows.every(r => r.modelDisplayName === 'GLM-5.2')).toBe(true)
|
||||
})
|
||||
|
||||
it('reports the dominant task type with its cost share in default mode', async () => {
|
||||
const project = makeProject([
|
||||
makeTurn('feature', [makeCall({ provider: 'claude', model: 'claude-sonnet-4-6', costUSD: 6.0, input: 100, output: 20 })]),
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ describe('cursor-agent provider', () => {
|
|||
|
||||
expect(provider.modelDisplayName('claude-5-future-model')).toBe('claude-5-future-model (est.)')
|
||||
expect(provider.modelDisplayName('gpt-9')).toBe('gpt-9 (est.)')
|
||||
expect(provider.modelDisplayName('gpt-5.6-sol')).toBe('GPT-5.6 Sol (est.)')
|
||||
})
|
||||
|
||||
it('returns identity for tool display name', () => {
|
||||
|
|
|
|||
|
|
@ -66,6 +66,11 @@ describe('vercel-gateway provider', () => {
|
|||
expect(calls[0]?.costUSD).toBe(1.25)
|
||||
expect(calls[0]?.model).toBe('anthropic/claude-sonnet-4.6')
|
||||
})
|
||||
|
||||
it('resolves vendor/slug ids through the global short-name table', () => {
|
||||
expect(vercelGateway.modelDisplayName('openai/gpt-5.6-terra')).toBe('GPT-5.6 Terra')
|
||||
expect(vercelGateway.modelDisplayName('accounts/fireworks/models/kimi-k2p6')).toBe('Kimi K2.6')
|
||||
})
|
||||
})
|
||||
|
||||
describe('vercel-gateway end-to-end (parseAllSessions network path)', () => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue