diff --git a/src/models-report.ts b/src/models-report.ts index b784e15a..34d8cafc 100644 --- a/src/models-report.ts +++ b/src/models-report.ts @@ -3,7 +3,7 @@ import stripAnsi from 'strip-ansi' import { codexCredits } from './codex-credits.js' import { formatCost, formatTokens } from './format.js' -import { fallbackRawModelDisplayName, getShortModelName, sanitizeModelForDisplay } from './models.js' +import { fallbackRawModelDisplayName, getShortModelName, resolveCanonicalModelId, sanitizeModelForDisplay } from './models.js' import { getProvider } from './providers/index.js' import { CATEGORY_LABELS, type ProjectSummary, type TaskCategory } from './types.js' @@ -28,8 +28,12 @@ export type ModelReportRow = { savingsBaselineModel: string calls: number /// Codex credit consumption (issues #408/#495). null for non-Codex models or - /// Codex models without a known credit rate. + /// Codex models without a known credit rate. A merged row that mixed rated + /// and unrated buckets stores the partial sum of the rated ones. credits: number | null + /// True when `credits` is a partial sum because some contributing buckets + /// had no known credit rate. + creditsIncomplete?: boolean topCategory?: TaskCategory topCategoryCost?: number topCategoryShare?: number @@ -74,9 +78,9 @@ function bucketKey(provider: string, model: string, category: TaskCategory | nul /// Walks every parsed turn, attributes each assistant call to a /// (provider, raw-model, category, agent) bucket, then merges buckets that -/// resolve to the same provider + display name. Returned rows are keyed by -/// (provider, display name) by default, plus category under `byTask` or agent -/// under `byAgent`. +/// resolve to the same provider + alias-resolved canonical id. Display names +/// stay cosmetic. Returned rows are keyed by (provider, canonical id) by +/// default, plus category under `byTask` or agent under `byAgent`. /// /// Default view: rows sorted by cost descending. /// byTask / byAgent view: rows grouped by (provider, model) so the renderer can @@ -175,11 +179,13 @@ export async function aggregateModels(projects: ProjectSummary[], opts: Aggregat for (const bucket of buckets.values()) { const meta = await resolveProvider(bucket.provider) const modelDisplayName = meta.formatModel(bucket.model) - const resolvedKey = bucketKey(bucket.provider, modelDisplayName, bucket.category, bucket.agentType) - const displayModelKey = `${bucket.provider} ${modelDisplayName}` + const canonicalId = resolveCanonicalModelId(bucket.model) + const resolvedKey = bucketKey(bucket.provider, canonicalId, bucket.category, bucket.agentType) + const foldKey = `${bucket.provider} ${canonicalId}` const total = bucket.inputTokens + bucket.outputTokens + bucket.cacheWriteTokens + bucket.cacheReadTokens - // Credits are per raw id (aliases can have different rates). Sum only when - // every contributing bucket has a known rate; otherwise the merged row is null. + // Credits are per raw id (aliases can have different rates). Sum the + // rated buckets and flag the row incomplete when any contributor is + // unrated — nulling the whole merge would zero a real menubar total. const bucketCredits = bucket.provider === 'codex' ? codexCredits(bucket.model, { inputTokens: bucket.inputTokens, @@ -203,10 +209,11 @@ export async function aggregateModels(projects: ProjectSummary[], opts: Aggregat existing.costUSD += bucket.costUSD existing.savingsUSD += bucket.savingsUSD existing.calls += bucket.calls - if (bucket.model < existing.model) existing.model = bucket.model existing.savingsBaselineModel = resolvedBaseline - if (existing.credits === null || bucketCredits === null) existing.credits = null - else existing.credits += bucketCredits + const existingRated = existing.credits !== null + const incomingRated = bucketCredits !== null + if (incomingRated) existing.credits = (existing.credits ?? 0) + bucketCredits + if (existingRated !== incomingRated) existing.creditsIncomplete = true } else { rowsByKey.set(resolvedKey, { provider: bucket.provider, @@ -233,18 +240,18 @@ export async function aggregateModels(projects: ProjectSummary[], opts: Aggregat foldedRawSeen.add(rawKey) const rawCat = perModelCategoryCost.get(rawKey) if (rawCat) { - let folded = foldedCategoryCost.get(displayModelKey) + let folded = foldedCategoryCost.get(foldKey) if (!folded) { folded = new Map() - foldedCategoryCost.set(displayModelKey, folded) + foldedCategoryCost.set(foldKey, folded) } for (const [cat, cost] of rawCat) { folded.set(cat, (folded.get(cat) ?? 0) + cost) } } foldedTotalCost.set( - displayModelKey, - (foldedTotalCost.get(displayModelKey) ?? 0) + (perModelTotalCost.get(rawKey) ?? 0), + foldKey, + (foldedTotalCost.get(foldKey) ?? 0) + (perModelTotalCost.get(rawKey) ?? 0), ) } } @@ -252,7 +259,7 @@ export async function aggregateModels(projects: ProjectSummary[], opts: Aggregat const rows = [...rowsByKey.values()] for (const row of rows) { if (opts.byTask || opts.byAgent) continue - const perCat = foldedCategoryCost.get(`${row.provider} ${row.modelDisplayName}`) + const perCat = foldedCategoryCost.get(`${row.provider} ${resolveCanonicalModelId(row.model)}`) if (!perCat || perCat.size === 0) continue let topCat: TaskCategory = 'general' let topCost = -1 @@ -271,8 +278,8 @@ export async function aggregateModels(projects: ProjectSummary[], opts: Aggregat if (opts.byTask || opts.byAgent) { rows.sort((a, b) => { - const aTotal = foldedTotalCost.get(`${a.provider} ${a.modelDisplayName}`) ?? 0 - const bTotal = foldedTotalCost.get(`${b.provider} ${b.modelDisplayName}`) ?? 0 + const aTotal = foldedTotalCost.get(`${a.provider} ${resolveCanonicalModelId(a.model)}`) ?? 0 + const bTotal = foldedTotalCost.get(`${b.provider} ${resolveCanonicalModelId(b.model)}`) ?? 0 if (aTotal !== bTotal) return bTotal - aTotal if (a.provider !== b.provider) return a.provider.localeCompare(b.provider) if (a.modelDisplayName !== b.modelDisplayName) return a.modelDisplayName.localeCompare(b.modelDisplayName) @@ -657,6 +664,7 @@ export function renderJson(rows: ModelReportRow[]): string { savingsUSD: r.savingsUSD, savingsBaselineModel: r.savingsBaselineModel, credits: r.credits, + creditsIncomplete: r.creditsIncomplete === true, })), null, 2, diff --git a/src/models.ts b/src/models.ts index 1abbd348..3225686a 100644 --- a/src/models.ts +++ b/src/models.ts @@ -616,6 +616,19 @@ function getCanonicalName(model: string): string { ) } +/// Alias-resolved identity for report merge. Display names stay cosmetic — +/// prefix matches in SHORT_NAMES must not fold distinct SKUs into one row. +/// Path-form ids (`accounts/fireworks/models/`, `cline-pass/`) +/// peel to the leaf so they share a bucket with the bare slug. +export function resolveCanonicalModelId(model: string): string { + const viaUser = Object.hasOwn(userAliases, model) ? userAliases[model]! : model + const aliased = resolveAlias(getCanonicalName(viaUser)) + if (!aliased.includes('/')) return aliased + const leaf = aliased.slice(aliased.lastIndexOf('/') + 1) + if (!leaf) return aliased + return resolveAlias(getCanonicalName(leaf)) +} + // Namespaces the pricing catalog itself uses, plus the ones below. An unknown // `provider/model` must stay unpriced — do not treat `/` as authority. Derived // rather than hand-listed so a vendor LiteLLM already knows (`x-ai/`, `qwen/`, diff --git a/tests/models-report.test.ts b/tests/models-report.test.ts index 5c933eac..31152156 100644 --- a/tests/models-report.test.ts +++ b/tests/models-report.test.ts @@ -8,6 +8,7 @@ import chalk from 'chalk' import stripAnsi from 'strip-ansi' import { aggregateModels, renderTable, renderMarkdown, renderJson, renderCsv, type ModelReportRow } from '../src/models-report.js' +import { setModelAliases } from '../src/models.js' import type { ProjectSummary, SessionSummary, @@ -169,6 +170,32 @@ describe('aggregateModels', () => { expect(byKey['codex:gpt-5.5']!.credits).toBeCloseTo(887.5, 6) expect(byKey['codex:gpt-5']!.credits).toBeNull() expect(byKey['claude:claude-sonnet-4-6']!.credits).toBeNull() + expect(byKey['codex:gpt-5.5']!.creditsIncomplete).toBeFalsy() + }) + + it('partial-sums Codex credits when a merge mixes rated and unrated ids', async () => { + setModelAliases({ 'codex-house-sku': 'gpt-5.5' }) + try { + const rows = await aggregateModels([makeProject([ + makeTurn('feature', [ + makeCall({ provider: 'codex', model: 'gpt-5.5', input: 0, output: 1_000_000, costUSD: 9 }), + ]), + makeTurn('feature', [ + makeCall({ provider: 'codex', model: 'codex-house-sku', input: 0, output: 1_000_000, costUSD: 9 }), + ]), + ])]) + expect(rows).toHaveLength(1) + expect(rows[0]!.model).toBe('gpt-5.5') + // gpt-5.5 output is 750 credits/M; the aliased house SKU has no rate. + expect(rows[0]!.credits).toBeCloseTo(750, 6) + expect(rows[0]!.creditsIncomplete).toBe(true) + expect(rows[0]!.calls).toBe(2) + const parsed = JSON.parse(renderJson(rows)) + expect(parsed[0].credits).toBeCloseTo(750, 6) + expect(parsed[0].creditsIncomplete).toBe(true) + } finally { + setModelAliases({}) + } }) it('includes credits in the JSON output', async () => { @@ -210,20 +237,22 @@ describe('aggregateModels', () => { expect(rows[0]!.model).toBe('accounts/fireworks/models/kimi-k2p6') }) - it('merges two raw ids that resolve to the same provider + display name', async () => { + it('merges two raw ids that resolve to the same alias-resolved canonical id', async () => { const rows = await aggregateModels([makeProject([ makeTurn('testing', [makeCall({ provider: 'cline-cli', model: 'accounts/fireworks/models/glm-5p2', input: 800, output: 67, costUSD: 0.246, })]), makeTurn('conversation', [makeCall({ - provider: 'cline-cli', model: 'GLM-5.2', + provider: 'cline-cli', model: 'glm-5p2', input: 15, output: 1, costUSD: 0.019, })]), ])]) expect(rows).toHaveLength(1) expect(rows[0]!.provider).toBe('cline-cli') expect(rows[0]!.modelDisplayName).toBe('GLM-5.2') + // First-seen raw id, not the lexicographically-smallest of the merge. + expect(rows[0]!.model).toBe('accounts/fireworks/models/glm-5p2') expect(rows[0]!.inputTokens).toBe(815) expect(rows[0]!.outputTokens).toBe(68) expect(rows[0]!.costUSD).toBeCloseTo(0.265, 6) @@ -232,24 +261,53 @@ describe('aggregateModels', () => { expect(rows[0]!.topCategoryShare).toBeCloseTo(0.246 / 0.265, 3) }) + it('does not pick the lexicographically-smallest raw id after a merge', async () => { + const rows = await aggregateModels([makeProject([ + makeTurn('feature', [makeCall({ provider: 'codex', model: 'kimi-k3', costUSD: 2 })]), + makeTurn('feature', [makeCall({ provider: 'codex', model: 'k3', costUSD: 1 })]), + ])]) + expect(rows).toHaveLength(1) + expect(rows[0]!.model).toBe('kimi-k3') + expect(rows[0]!.modelDisplayName).toBe('Kimi K3') + expect(rows[0]!.calls).toBe(2) + }) + + it('does not merge same-provider ids that only share a display name', async () => { + const rows = await aggregateModels([makeProject([ + makeTurn('feature', [makeCall({ provider: 'codex', model: 'gpt-5', costUSD: 3 })]), + makeTurn('feature', [makeCall({ provider: 'codex', model: 'gpt-5-untracked-xyz', costUSD: 1 })]), + makeTurn('feature', [makeCall({ provider: 'cline-cli', model: 'glm-5p2', costUSD: 1 })]), + makeTurn('feature', [makeCall({ provider: 'cline-cli', model: 'GLM-5.2', costUSD: 2 })]), + ])]) + const codex = rows.filter(r => r.provider === 'codex') + expect(codex).toHaveLength(2) + expect(codex.every(r => r.modelDisplayName === 'GPT-5')).toBe(true) + expect(new Set(codex.map(r => r.model))).toEqual(new Set(['gpt-5', 'gpt-5-untracked-xyz'])) + + const cline = rows.filter(r => r.provider === 'cline-cli') + expect(cline).toHaveLength(2) + expect(cline.every(r => r.modelDisplayName === 'GLM-5.2')).toBe(true) + expect(new Set(cline.map(r => r.model))).toEqual(new Set(['glm-5p2', 'GLM-5.2'])) + }) + it('clears a merged savings baseline when three raw ids disagree', async () => { const rows = await aggregateModels([makeProject([ makeTurn('feature', [makeCall({ - provider: 'cline-cli', model: 'glm-5p2', + provider: 'codex', model: 'k3', costUSD: 1, savingsUSD: 2, savingsBaselineModel: 'gpt-4o', })]), makeTurn('feature', [makeCall({ - provider: 'cline-cli', model: 'GLM-5.2', + provider: 'codex', model: 'kimi-k3', costUSD: 1, savingsUSD: 2, savingsBaselineModel: 'claude-sonnet-4-6', })]), makeTurn('feature', [makeCall({ - provider: 'cline-cli', model: 'accounts/fireworks/models/glm-5p2', + provider: 'codex', model: 'k3-agent', costUSD: 1, savingsUSD: 2, savingsBaselineModel: 'gpt-5', })]), ])]) expect(rows).toHaveLength(1) - expect(rows[0]!.provider).toBe('cline-cli') - expect(rows[0]!.modelDisplayName).toBe('GLM-5.2') + expect(rows[0]!.provider).toBe('codex') + expect(rows[0]!.modelDisplayName).toBe('Kimi K3') expect(rows[0]!.savingsUSD).toBe(6) expect(rows[0]!.savingsBaselineModel).toBe('') expect(rows[0]!.calls).toBe(3) diff --git a/tests/models.test.ts b/tests/models.test.ts index b644b784..c9dcba48 100644 --- a/tests/models.test.ts +++ b/tests/models.test.ts @@ -7,6 +7,7 @@ import { findUnpricedModels, getModelCosts, getShortModelName, + resolveCanonicalModelId, calculateCost, loadPricing, setModelAliases, @@ -123,6 +124,22 @@ describe('getModelCosts', () => { }) }) +describe('resolveCanonicalModelId', () => { + it('aliases, peels path-form ids, and leaves display-only collisions distinct', () => { + expect(resolveCanonicalModelId('k3')).toBe('kimi-k3') + expect(resolveCanonicalModelId('k3-agent')).toBe('kimi-k3') + expect(resolveCanonicalModelId('kimi-k3')).toBe('kimi-k3') + expect(resolveCanonicalModelId('accounts/fireworks/models/glm-5p2')).toBe('glm-5p2') + expect(resolveCanonicalModelId('glm-5p2')).toBe('glm-5p2') + expect(resolveCanonicalModelId('GLM-5.2')).toBe('glm-5p1') + expect(resolveCanonicalModelId('gpt-5-fast')).toBe('gpt-5') + expect(resolveCanonicalModelId('gpt-5-untracked-xyz')).toBe('gpt-5-untracked-xyz') + expect(resolveCanonicalModelId('claude-opus-4.6')).toBe('claude-opus-4-6') + expect(resolveCanonicalModelId('kimi-code')).toBe('kimi-k2-thinking') + expect(resolveCanonicalModelId('cline-pass/kimi-k3')).toBe('kimi-k3') + }) +}) + describe('getShortModelName', () => { it('maps gpt-4o-mini correctly (not gpt-4o)', () => { expect(getShortModelName('gpt-4o-mini-2024-07-18')).toBe('GPT-4o Mini')