mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-07-10 01:29:41 +00:00
feat(pricing): automatic gap-fill from models.dev and OpenRouter (#457)
Keep model pricing automatic instead of hand-coding new models. The bundler now layers three sources in priority order: LiteLLM (broad list prices), hand-curated MANUAL_ENTRIES overrides, then a separate last-resort fallback file gap-filled from models.dev first-party makers (official direct prices) and OpenRouter (resale backstop). New models such as MiniMax-M3 ($0.6/$2.4) now price correctly with no per-model code. The fallback is written to its own pricing-fallback.json and consulted only case-insensitively as the final step in getModelCosts, so a reseller variant name can never shadow a canonical or aliased match. Fixes surfaced while building and verifying this: - Alias precedence: LiteLLM ships snowflake/claude-4-opus ($5), which the bundler strips to a bare claude-4-opus key that shadowed the curated alias to claude-opus-4 ($15 official). An explicit alias for a bare name now wins over a coincidental stripped reseller key; the prefixed gateway price is still returned for the fully-qualified id. - Zero-stub guard: LiteLLM [0,0] price stubs (e.g. GigaChat-2-Max) are excluded from the case-insensitive index so a case-mismatched query stays null and keeps firing the unknown-model warning instead of silently reporting $0. - Negative-sentinel guard: OpenRouter returns -1 for variable/BYOK-priced models. The bundler now rejects any non-positive rate pair (and strips the sentinel from cache fields) so a negative per-token cost can never ship and subtract from spend totals. Bundler hardening: bareKey strips @pin and date suffixes to match the runtime canonical form, seen-set dedupes on both full and bare key shapes, and it logs MANUAL_ENTRIES now covered upstream plus models.dev allowlist drift. Extracted buildCosts so the cache-cost heuristics live in one place. Added a data-hygiene test that fails CI if a rebundle reintroduces negative, free, or unreachable fallback entries.
This commit is contained in:
parent
c36f3afa76
commit
a385f65dee
7 changed files with 312 additions and 28 deletions
|
|
@ -2,9 +2,39 @@ import { writeFileSync, mkdirSync } from 'fs'
|
|||
import { dirname, join } from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
|
||||
// Pricing sources, in priority order:
|
||||
// 1. LiteLLM - broad, maintained, tracks provider list prices.
|
||||
// 2. MANUAL_ENTRIES - hand-curated overrides for the primary snapshot.
|
||||
// 3. models.dev - only FIRST-PARTY maker providers (not the 100+
|
||||
// gateways/resellers): official direct price for models
|
||||
// LiteLLM hasn't added yet (e.g. MiniMax-M3).
|
||||
// 4. OpenRouter - resale rates, one clean price per canonical model;
|
||||
// a coverage backstop for makers not in models.dev.
|
||||
//
|
||||
// Output is TWO files:
|
||||
// litellm-snapshot.json - primary (LiteLLM + MANUAL_ENTRIES). Used for the
|
||||
// exact / canonical / prefix lookups.
|
||||
// pricing-fallback.json - gap-fill (models.dev + OpenRouter). Consulted ONLY
|
||||
// as a last resort, so a reseller variant name can
|
||||
// never shadow an existing canonical/alias match.
|
||||
const LITELLM_URL = 'https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json'
|
||||
const MODELS_DEV_URL = 'https://models.dev/api.json'
|
||||
const OPENROUTER_URL = 'https://openrouter.ai/api/v1/models'
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url))
|
||||
const outPath = join(__dirname, '..', 'src', 'data', 'litellm-snapshot.json')
|
||||
const dataDir = join(__dirname, '..', 'src', 'data')
|
||||
const snapshotPath = join(dataDir, 'litellm-snapshot.json')
|
||||
const fallbackPath = join(dataDir, 'pricing-fallback.json')
|
||||
|
||||
// models.dev provider ids that are the actual model MAKERS (publish official
|
||||
// list prices), as opposed to gateways/resellers (openrouter, nano-gpt, vercel,
|
||||
// poe, novita, etc.) that mark up or discount. An id missing here just means
|
||||
// that maker's models fall through to OpenRouter; an unknown id is ignored.
|
||||
const MODELS_DEV_FIRST_PARTY = new Set([
|
||||
'openai', 'anthropic', 'google', 'google-vertex', 'mistral', 'deepseek',
|
||||
'xai', 'minimax', 'minimax-cn', 'moonshotai', 'zhipuai', 'alibaba',
|
||||
'alibaba-cn', 'cohere', 'perplexity', 'inception', 'morph',
|
||||
])
|
||||
|
||||
const MANUAL_ENTRIES = {
|
||||
'MiniMax-M2.7': [0.3e-6, 1.2e-6, 0.375e-6, 0.06e-6],
|
||||
|
|
@ -14,11 +44,12 @@ const MANUAL_ENTRIES = {
|
|||
'deepseek-v4-pro': [4.35e-7, 8.7e-7, 0, 3.625e-9],
|
||||
}
|
||||
|
||||
const snapshot = {}
|
||||
|
||||
// --- Pass 1+2: LiteLLM (primary) ---
|
||||
const res = await fetch(LITELLM_URL)
|
||||
if (!res.ok) throw new Error(`HTTP ${res.status}`)
|
||||
const data = await res.json()
|
||||
|
||||
const snapshot = {}
|
||||
const entries = Object.entries(data).filter(([k]) => k !== 'sample_spec')
|
||||
|
||||
function toVal(entry) {
|
||||
|
|
@ -34,7 +65,6 @@ for (const [name, entry] of entries) {
|
|||
const val = toVal(entry)
|
||||
if (val) snapshot[name] = val
|
||||
}
|
||||
|
||||
// Pass 2: prefixed entries - store full key + stripped (first-write-wins)
|
||||
for (const [name, entry] of entries) {
|
||||
if (!name.includes('/')) continue
|
||||
|
|
@ -45,8 +75,92 @@ for (const [name, entry] of entries) {
|
|||
if (stripped !== name && !snapshot[stripped]) snapshot[stripped] = val
|
||||
}
|
||||
|
||||
// A MANUAL_ENTRY that LiteLLM now ships is a candidate to delete (the override
|
||||
// would otherwise shadow upstream forever with a possibly-stale hand value).
|
||||
for (const k of Object.keys(MANUAL_ENTRIES)) {
|
||||
if (snapshot[k]) console.log(`note: MANUAL_ENTRIES['${k}'] is now in LiteLLM - candidate to remove`)
|
||||
}
|
||||
Object.assign(snapshot, MANUAL_ENTRIES)
|
||||
|
||||
mkdirSync(dirname(outPath), { recursive: true })
|
||||
writeFileSync(outPath, JSON.stringify(snapshot))
|
||||
console.log(`Bundled ${Object.keys(snapshot).length} models → src/data/litellm-snapshot.json`)
|
||||
// --- Gap fill into a SEPARATE fallback map (last-resort only) ---
|
||||
const fallback = {}
|
||||
// Strip the vendor prefix to the last path segment, then the @pin and trailing
|
||||
// -YYYYMMDD date that the runtime's getCanonicalName also strips, so a fallback
|
||||
// key lines up with the canonical form actually queried (otherwise e.g.
|
||||
// `vendor/claude-3-5-sonnet@20241022` becomes a key the lookup can never reach).
|
||||
const bareKey = (name) => name.replace(/^.*\//, '').replace(/@.*$/, '').replace(/-\d{8}$/, '')
|
||||
// `seen` holds every primary key AND its bareKey form (both lowercased) so we
|
||||
// never re-add a model LiteLLM/MANUAL already covers under either shape; fallback
|
||||
// keys are added too so the first source wins (models.dev before OpenRouter).
|
||||
const seen = new Set()
|
||||
for (const k of Object.keys(snapshot)) {
|
||||
seen.add(k.toLowerCase())
|
||||
seen.add(bareKey(k).toLowerCase())
|
||||
}
|
||||
const finite = (v) => { const n = Number(v); return Number.isFinite(n) ? n : null }
|
||||
// A rate pair is usable only if both sides are non-negative and not both zero.
|
||||
// OpenRouter uses -1 as a "variable / BYOK price" sentinel; without this guard a
|
||||
// negative per-token cost would ship and subtract from a user's spend totals.
|
||||
const validRates = (inp, out) => inp != null && out != null && inp >= 0 && out >= 0 && !(inp === 0 && out === 0)
|
||||
// Drop the same negative sentinel on optional cache fields.
|
||||
const nonNeg = (v) => (v != null && v >= 0 ? v : null)
|
||||
function addGap(key, val) {
|
||||
if (!key || !val) return false
|
||||
const lk = key.toLowerCase()
|
||||
if (seen.has(lk)) return false
|
||||
fallback[key] = val
|
||||
seen.add(lk)
|
||||
return true
|
||||
}
|
||||
|
||||
// --- Pass 3: models.dev first-party makers (official list prices) ---
|
||||
try {
|
||||
const md = await (await fetch(MODELS_DEV_URL)).json()
|
||||
// Surface drift in our hand-maintained maker allowlist: if an id we classify
|
||||
// as first-party is gone from the API, it was renamed/removed and the set is
|
||||
// stale (its models would silently fall through to OpenRouter resale rates).
|
||||
for (const id of MODELS_DEV_FIRST_PARTY) {
|
||||
if (!md[id]) console.warn(`note: models.dev no longer lists first-party id '${id}' - allowlist may be stale`)
|
||||
}
|
||||
let added = 0
|
||||
for (const pid of Object.keys(md).sort()) {
|
||||
if (!MODELS_DEV_FIRST_PARTY.has(pid)) continue
|
||||
const models = md[pid].models ?? {}
|
||||
for (const mid of Object.keys(models).sort()) {
|
||||
const c = models[mid].cost
|
||||
if (!c) continue
|
||||
const inp = finite(c.input), out = finite(c.output)
|
||||
if (!validRates(inp, out)) continue
|
||||
// models.dev cost is per MILLION tokens; snapshot is per token.
|
||||
const cw = nonNeg(c.cache_write != null ? finite(c.cache_write) : null)
|
||||
const cr = nonNeg(c.cache_read != null ? finite(c.cache_read) : null)
|
||||
if (addGap(bareKey(mid), [inp / 1e6, out / 1e6, cw != null ? cw / 1e6 : null, cr != null ? cr / 1e6 : null, null])) added++
|
||||
}
|
||||
}
|
||||
console.log(`models.dev (first-party): +${added} models`)
|
||||
} catch (e) {
|
||||
console.warn(`models.dev skipped: ${e.message}`)
|
||||
}
|
||||
|
||||
// --- Pass 4: OpenRouter (resale backstop) ---
|
||||
try {
|
||||
const or = (await (await fetch(OPENROUTER_URL)).json()).data ?? []
|
||||
let added = 0
|
||||
for (const m of or) {
|
||||
const p = m.pricing ?? {}
|
||||
const inp = finite(p.prompt), out = finite(p.completion)
|
||||
if (!validRates(inp, out)) continue
|
||||
// OpenRouter pricing fields are already per-token.
|
||||
const cw = nonNeg(p.input_cache_write != null ? finite(p.input_cache_write) : null)
|
||||
const cr = nonNeg(p.input_cache_read != null ? finite(p.input_cache_read) : null)
|
||||
if (addGap(bareKey(m.id ?? ''), [inp, out, cw, cr, null])) added++
|
||||
}
|
||||
console.log(`openrouter (backstop): +${added} models`)
|
||||
} catch (e) {
|
||||
console.warn(`openrouter skipped: ${e.message}`)
|
||||
}
|
||||
|
||||
mkdirSync(dataDir, { recursive: true })
|
||||
writeFileSync(snapshotPath, JSON.stringify(snapshot))
|
||||
writeFileSync(fallbackPath, JSON.stringify(fallback))
|
||||
console.log(`Bundled ${Object.keys(snapshot).length} primary + ${Object.keys(fallback).length} fallback models`)
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
1
src/data/pricing-fallback.json
Normal file
1
src/data/pricing-fallback.json
Normal file
File diff suppressed because one or more lines are too long
119
src/models.ts
119
src/models.ts
|
|
@ -2,6 +2,7 @@ import { readFile, writeFile, mkdir } from 'fs/promises'
|
|||
import { join } from 'path'
|
||||
import { homedir } from 'os'
|
||||
import snapshotData from './data/litellm-snapshot.json'
|
||||
import fallbackData from './data/pricing-fallback.json'
|
||||
import { fetchWithTimeout } from './fetch-utils.js'
|
||||
|
||||
export type ModelCosts = {
|
||||
|
|
@ -31,18 +32,36 @@ const CACHE_TTL_MS = 24 * 60 * 60 * 1000
|
|||
const WEB_SEARCH_COST = 0.01
|
||||
const ONE_HOUR_CACHE_WRITE_MULTIPLIER_FROM_FIVE_MINUTE_RATE = 1.6
|
||||
|
||||
// Assemble a ModelCosts, applying the cache-cost heuristics (write = 1.25x
|
||||
// input, read = 0.1x input) when a source omits them. Shared by the bundled
|
||||
// tuple path (tupleToCosts) and the live LiteLLM path (parseLiteLLMEntry) so the
|
||||
// multipliers live in exactly one place.
|
||||
function buildCosts(
|
||||
input: number,
|
||||
output: number,
|
||||
cacheWrite: number | null | undefined,
|
||||
cacheRead: number | null | undefined,
|
||||
fast: number | null | undefined,
|
||||
): ModelCosts {
|
||||
return {
|
||||
inputCostPerToken: input,
|
||||
outputCostPerToken: output,
|
||||
cacheWriteCostPerToken: cacheWrite ?? input * 1.25,
|
||||
cacheReadCostPerToken: cacheRead ?? input * 0.1,
|
||||
webSearchCostPerRequest: WEB_SEARCH_COST,
|
||||
fastMultiplier: fast ?? 1,
|
||||
}
|
||||
}
|
||||
|
||||
function tupleToCosts(raw: SnapshotEntry): ModelCosts {
|
||||
const [input, output, cacheWrite, cacheRead, fast] = raw
|
||||
return buildCosts(input, output, cacheWrite, cacheRead, fast)
|
||||
}
|
||||
|
||||
function loadSnapshot(): Map<string, ModelCosts> {
|
||||
const map = new Map<string, ModelCosts>()
|
||||
for (const [name, raw] of Object.entries(snapshotData as unknown as Record<string, SnapshotEntry>)) {
|
||||
const [input, output, cacheWrite, cacheRead, fast] = raw
|
||||
map.set(name, {
|
||||
inputCostPerToken: input,
|
||||
outputCostPerToken: output,
|
||||
cacheWriteCostPerToken: cacheWrite ?? input * 1.25,
|
||||
cacheReadCostPerToken: cacheRead ?? input * 0.1,
|
||||
webSearchCostPerRequest: WEB_SEARCH_COST,
|
||||
fastMultiplier: fast ?? 1,
|
||||
})
|
||||
map.set(name, tupleToCosts(raw))
|
||||
}
|
||||
// TEMP (2026-06-09): Fable 5 / Mythos 5 launch pricing, $10/M input and $50/M output,
|
||||
// until LiteLLM indexes them. Added as snapshot fallbacks so a real LiteLLM entry, once
|
||||
|
|
@ -61,8 +80,21 @@ function loadSnapshot(): Map<string, ModelCosts> {
|
|||
return map
|
||||
}
|
||||
|
||||
// Gap-fill pricing from models.dev / OpenRouter, keyed lowercase. Consulted ONLY
|
||||
// as the last-resort fallback in getModelCosts (never for exact/canonical/prefix
|
||||
// matches), so a reseller variant name can't shadow a real canonical entry.
|
||||
const fallbackCosts: Map<string, ModelCosts> = (() => {
|
||||
const map = new Map<string, ModelCosts>()
|
||||
for (const [name, raw] of Object.entries(fallbackData as unknown as Record<string, SnapshotEntry>)) {
|
||||
const lk = name.toLowerCase()
|
||||
if (!map.has(lk)) map.set(lk, tupleToCosts(raw))
|
||||
}
|
||||
return map
|
||||
})()
|
||||
|
||||
let pricingCache: Map<string, ModelCosts> = loadSnapshot()
|
||||
let sortedPricingKeys: string[] | null = null
|
||||
let lowercasePricingIndex: Map<string, ModelCosts> | null = null
|
||||
|
||||
function getSortedPricingKeys(): string[] {
|
||||
if (sortedPricingKeys === null) {
|
||||
|
|
@ -71,6 +103,33 @@ function getSortedPricingKeys(): string[] {
|
|||
return sortedPricingKeys
|
||||
}
|
||||
|
||||
// Case-insensitive index, built lazily. Lets a session model like `MiniMax-M3`
|
||||
// resolve to a gap-filled OpenRouter key like `minimax-m3` (lowercase slug).
|
||||
// First key wins on a lowercase collision so it stays deterministic.
|
||||
//
|
||||
// Zero-priced entries are excluded: LiteLLM ships `[0,0]` stubs (e.g.
|
||||
// `GigaChat-2-Max`) for models it lists but has no price for. Indexing those
|
||||
// would let a case-mismatched query (`gigachat-2-max`) resolve to a silent $0
|
||||
// instead of returning null, which suppresses the unknown-model warning and
|
||||
// hides real spend. A case-EXACT query still finds the stub via the normal
|
||||
// pipeline; only the fuzzy case-insensitive path skips them.
|
||||
function getLowercasePricingIndex(): Map<string, ModelCosts> {
|
||||
if (lowercasePricingIndex === null) {
|
||||
lowercasePricingIndex = new Map()
|
||||
const priced = (c: ModelCosts) => c.inputCostPerToken > 0 || c.outputCostPerToken > 0
|
||||
// The live pricing data wins on any lowercase collision; the gap-fill only
|
||||
// fills names that resolve to nothing through the normal pipeline.
|
||||
for (const [key, costs] of pricingCache) {
|
||||
const lk = key.toLowerCase()
|
||||
if (priced(costs) && !lowercasePricingIndex.has(lk)) lowercasePricingIndex.set(lk, costs)
|
||||
}
|
||||
for (const [lk, costs] of fallbackCosts) {
|
||||
if (priced(costs) && !lowercasePricingIndex.has(lk)) lowercasePricingIndex.set(lk, costs)
|
||||
}
|
||||
}
|
||||
return lowercasePricingIndex
|
||||
}
|
||||
|
||||
function getCacheDir(): string {
|
||||
if (process.env['CODEBURN_CACHE_DIR']) return process.env['CODEBURN_CACHE_DIR']
|
||||
return join(homedir(), '.cache', 'codeburn')
|
||||
|
|
@ -96,16 +155,13 @@ function parseLiteLLMEntry(entry: LiteLLMEntry): ModelCosts | null {
|
|||
const inputCost = safePerTokenRate(entry.input_cost_per_token)
|
||||
const outputCost = safePerTokenRate(entry.output_cost_per_token)
|
||||
if (inputCost === null || outputCost === null) return null
|
||||
const cacheWrite = safePerTokenRate(entry.cache_creation_input_token_cost) ?? inputCost * 1.25
|
||||
const cacheRead = safePerTokenRate(entry.cache_read_input_token_cost) ?? inputCost * 0.1
|
||||
return {
|
||||
inputCostPerToken: inputCost,
|
||||
outputCostPerToken: outputCost,
|
||||
cacheWriteCostPerToken: cacheWrite,
|
||||
cacheReadCostPerToken: cacheRead,
|
||||
webSearchCostPerRequest: WEB_SEARCH_COST,
|
||||
fastMultiplier: entry.provider_specific_entry?.fast ?? 1,
|
||||
}
|
||||
return buildCosts(
|
||||
inputCost,
|
||||
outputCost,
|
||||
safePerTokenRate(entry.cache_creation_input_token_cost),
|
||||
safePerTokenRate(entry.cache_read_input_token_cost),
|
||||
entry.provider_specific_entry?.fast,
|
||||
)
|
||||
}
|
||||
|
||||
async function fetchAndCachePricing(): Promise<Map<string, ModelCosts>> {
|
||||
|
|
@ -161,12 +217,14 @@ export async function loadPricing(): Promise<void> {
|
|||
if (cached) {
|
||||
pricingCache = mergeSnapshotFallbacks(cached)
|
||||
sortedPricingKeys = null
|
||||
lowercasePricingIndex = null
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
pricingCache = mergeSnapshotFallbacks(await fetchAndCachePricing())
|
||||
sortedPricingKeys = null
|
||||
lowercasePricingIndex = null
|
||||
} catch {
|
||||
// snapshot already loaded at init; nothing more to do
|
||||
}
|
||||
|
|
@ -365,9 +423,20 @@ function getCanonicalName(model: string): string {
|
|||
export function getModelCosts(model: string): ModelCosts | null {
|
||||
// Try with provider prefix preserved (azure/gpt-5.4, openrouter/anthropic/claude-opus-4.6)
|
||||
const withPrefix = model.replace(/@.*$/, '').replace(/-\d{8}$/, '')
|
||||
const canonicalName = getCanonicalName(model)
|
||||
const canonical = resolveAlias(canonicalName)
|
||||
|
||||
// An explicit alias for a bare (un-prefixed) model name is authoritative: it
|
||||
// must win over a coincidental stripped reseller key of the same name. LiteLLM
|
||||
// ships `snowflake/claude-4-opus` ($5), which the bundler strips to a bare
|
||||
// `claude-4-opus` key; without this, that would shadow the curated alias
|
||||
// `claude-4-opus -> claude-opus-4` ($15 official Anthropic price).
|
||||
if (canonical !== canonicalName && withPrefix === canonicalName && pricingCache.has(canonical)) {
|
||||
return pricingCache.get(canonical)!
|
||||
}
|
||||
|
||||
if (pricingCache.has(withPrefix)) return pricingCache.get(withPrefix)!
|
||||
|
||||
const canonical = resolveAlias(getCanonicalName(model))
|
||||
if (pricingCache.has(canonical)) return pricingCache.get(canonical)!
|
||||
|
||||
// Iterate keys longest-first so a model id like `gpt-5-mini` matches the
|
||||
|
|
@ -379,6 +448,16 @@ export function getModelCosts(model: string): ModelCosts | null {
|
|||
}
|
||||
}
|
||||
|
||||
// Case-insensitive fallback: gap-filled keys from OpenRouter are lowercase
|
||||
// slugs (e.g. `minimax-m3`), but sessions report `MiniMax-M3`. Only consulted
|
||||
// after the exact/canonical/prefix attempts, so it never changes a match that
|
||||
// already resolved above.
|
||||
const lowerIndex = getLowercasePricingIndex()
|
||||
const byCanonical = lowerIndex.get(canonical.toLowerCase())
|
||||
if (byCanonical) return byCanonical
|
||||
const byPrefix = lowerIndex.get(withPrefix.toLowerCase())
|
||||
if (byPrefix) return byPrefix
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -27,6 +27,16 @@ describe('MiniMax model pricing', () => {
|
|||
expect(costs!.fastMultiplier).toBe(1)
|
||||
})
|
||||
|
||||
it('returns official pricing for MiniMax-M3', () => {
|
||||
// M3 is gap-filled automatically (LiteLLM, with models.dev first-party as
|
||||
// the cross-check): $0.6/$2.4 per M, the official MiniMax direct price - not
|
||||
// a discounted reseller rate (e.g. OpenRouter's $0.3/$1.2).
|
||||
const costs = getModelCosts('MiniMax-M3')
|
||||
expect(costs).not.toBeNull()
|
||||
expect(costs!.inputCostPerToken).toBe(0.6e-6)
|
||||
expect(costs!.outputCostPerToken).toBe(2.4e-6)
|
||||
})
|
||||
|
||||
it('highspeed pricing is distinct from base model pricing', () => {
|
||||
const base = getModelCosts('MiniMax-M2.7')
|
||||
const fast = getModelCosts('MiniMax-M2.7-highspeed')
|
||||
|
|
|
|||
|
|
@ -349,6 +349,51 @@ describe('Cursor model variants resolve to pricing', () => {
|
|||
}
|
||||
})
|
||||
|
||||
// Regression: LiteLLM ships `snowflake/claude-4-opus` ($5/M, a gateway rate),
|
||||
// which the bundler strips to a bare `claude-4-opus` snapshot key. Without the
|
||||
// alias-precedence guard in getModelCosts, that bare reseller key shadows the
|
||||
// curated alias `claude-4-opus -> claude-opus-4` and mis-prices Opus 4 at a
|
||||
// third of its official list price. Pin the official number so a re-shadowing
|
||||
// fails loudly rather than silently under-reporting spend.
|
||||
describe('alias precedence over stripped reseller keys', () => {
|
||||
it('claude-4-opus resolves to the official Opus 4 list price, not a gateway discount', () => {
|
||||
const aliased = getModelCosts('claude-4-opus')
|
||||
const canonical = getModelCosts('claude-opus-4')
|
||||
expect(aliased).not.toBeNull()
|
||||
expect(canonical).not.toBeNull()
|
||||
expect(aliased!.inputCostPerToken).toBe(canonical!.inputCostPerToken)
|
||||
expect(aliased!.outputCostPerToken).toBe(canonical!.outputCostPerToken)
|
||||
expect(aliased!.inputCostPerToken).toBe(15e-6)
|
||||
expect(aliased!.outputCostPerToken).toBe(75e-6)
|
||||
})
|
||||
|
||||
it('the explicit provider prefix is still honored for the gateway rate', () => {
|
||||
// The guard fires only for the bare name; a fully-qualified gateway id must
|
||||
// still return that gateway's own price when LiteLLM publishes one.
|
||||
const gateway = getModelCosts('snowflake/claude-4-opus')
|
||||
const bare = getModelCosts('claude-4-opus')
|
||||
expect(gateway).not.toBeNull()
|
||||
expect(gateway!.inputCostPerToken).toBeLessThan(bare!.inputCostPerToken)
|
||||
})
|
||||
})
|
||||
|
||||
// The case-insensitive index that lets `MiniMax-M3` reach a lowercase
|
||||
// `minimax-m3` slug must NOT let a case-mismatched query resolve to one of
|
||||
// LiteLLM's [0,0] price stubs (e.g. `GigaChat-2-Max`). Doing so would flip an
|
||||
// honest null (which fires the "no pricing data, will show $0" warning) into a
|
||||
// silent $0 and hide real spend. A case-EXACT query still finds the stub.
|
||||
describe('zero-priced stubs do not satisfy case-insensitive lookup', () => {
|
||||
it('a case-mismatched query to a [0,0] stub stays null', () => {
|
||||
expect(getModelCosts('gigachat-2-max')).toBeNull()
|
||||
})
|
||||
|
||||
it('the case-exact stub still resolves (just at zero cost)', () => {
|
||||
const exact = getModelCosts('GigaChat-2-Max')
|
||||
expect(exact).not.toBeNull()
|
||||
expect(exact!.inputCostPerToken).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('DeepSeek v4 models resolve to pricing', () => {
|
||||
it('deepseek-v4-pro has current official discounted pricing', () => {
|
||||
const costs = getModelCosts('deepseek-v4-pro')
|
||||
|
|
|
|||
35
tests/pricing-fallback-data.test.ts
Normal file
35
tests/pricing-fallback-data.test.ts
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
import { describe, it, expect } from 'vitest'
|
||||
|
||||
import fallback from '../src/data/pricing-fallback.json' assert { type: 'json' }
|
||||
|
||||
// The gap-fill fallback is generated from models.dev / OpenRouter. These assert
|
||||
// the bundler's hygiene guarantees on the committed artifact, so a future
|
||||
// rebundle that regresses them fails CI rather than shipping bad pricing.
|
||||
describe('pricing-fallback.json data hygiene', () => {
|
||||
const entries = Object.entries(fallback as Record<string, (number | null)[]>)
|
||||
|
||||
it('is non-empty', () => {
|
||||
expect(entries.length).toBeGreaterThan(50)
|
||||
})
|
||||
|
||||
it('has no negative rates (OpenRouter -1 "variable price" sentinels)', () => {
|
||||
const bad = entries.filter(([, v]) => (v[0] ?? 0) < 0 || (v[1] ?? 0) < 0 || (v[2] ?? 0) < 0 || (v[3] ?? 0) < 0)
|
||||
expect(bad.map(([k]) => k)).toEqual([])
|
||||
})
|
||||
|
||||
it('has no entry that is free on both input and output', () => {
|
||||
const bad = entries.filter(([, v]) => v[0] === 0 && v[1] === 0)
|
||||
expect(bad.map(([k]) => k)).toEqual([])
|
||||
})
|
||||
|
||||
it('has no unreachable @pin or date-suffixed keys', () => {
|
||||
const bad = entries.filter(([k]) => /@/.test(k) || /\d{8}$/.test(k))
|
||||
expect(bad.map(([k]) => k)).toEqual([])
|
||||
})
|
||||
|
||||
it('stores per-token rates (no per-million values leaked through)', () => {
|
||||
// A per-million value would be >= 1; real per-token rates are tiny.
|
||||
const bad = entries.filter(([, v]) => (v[0] ?? 0) >= 1 || (v[1] ?? 0) >= 1)
|
||||
expect(bad.map(([k]) => k)).toEqual([])
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue