mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-07-10 01:29:41 +00:00
fix: DeepSeek v4 Claude pricing through stale runtime cache (#367)
Some checks are pending
CI / semgrep (push) Waiting to run
Some checks are pending
CI / semgrep (push) Waiting to run
This commit is contained in:
parent
f2e023cb83
commit
17eada2aa1
6 changed files with 229 additions and 3 deletions
|
|
@ -9,6 +9,9 @@ const outPath = join(__dirname, '..', 'src', 'data', 'litellm-snapshot.json')
|
|||
const MANUAL_ENTRIES = {
|
||||
'MiniMax-M2.7': [0.3e-6, 1.2e-6, 0.375e-6, 0.06e-6],
|
||||
'MiniMax-M2.7-highspeed': [0.6e-6, 2.4e-6, 0.375e-6, 0.06e-6],
|
||||
// LiteLLM PR #27056 is not merged yet. Source: https://api-docs.deepseek.com/quick_start/pricing
|
||||
'deepseek-v4-flash': [1.4e-7, 2.8e-7, 0, 2.8e-9],
|
||||
'deepseek-v4-pro': [4.35e-7, 8.7e-7, 0, 3.625e-9],
|
||||
}
|
||||
|
||||
const res = await fetch(LITELLM_URL)
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -59,6 +59,7 @@ function getSortedPricingKeys(): string[] {
|
|||
}
|
||||
|
||||
function getCacheDir(): string {
|
||||
if (process.env['CODEBURN_CACHE_DIR']) return process.env['CODEBURN_CACHE_DIR']
|
||||
return join(homedir(), '.cache', 'codeburn')
|
||||
}
|
||||
|
||||
|
|
@ -131,16 +132,23 @@ async function loadCachedPricing(): Promise<Map<string, ModelCosts> | null> {
|
|||
}
|
||||
}
|
||||
|
||||
function mergeSnapshotFallbacks(pricing: Map<string, ModelCosts>): Map<string, ModelCosts> {
|
||||
for (const [name, costs] of loadSnapshot()) {
|
||||
if (!pricing.has(name)) pricing.set(name, costs)
|
||||
}
|
||||
return pricing
|
||||
}
|
||||
|
||||
export async function loadPricing(): Promise<void> {
|
||||
const cached = await loadCachedPricing()
|
||||
if (cached) {
|
||||
pricingCache = cached
|
||||
pricingCache = mergeSnapshotFallbacks(cached)
|
||||
sortedPricingKeys = null
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
pricingCache = await fetchAndCachePricing()
|
||||
pricingCache = mergeSnapshotFallbacks(await fetchAndCachePricing())
|
||||
sortedPricingKeys = null
|
||||
} catch {
|
||||
// snapshot already loaded at init; nothing more to do
|
||||
|
|
@ -431,6 +439,8 @@ const SHORT_NAMES: Record<string, string> = {
|
|||
'kimi-k2': 'Kimi K2',
|
||||
'kimi-latest': 'Kimi Latest',
|
||||
'moonshot-v1': 'Moonshot v1',
|
||||
'deepseek-v4-pro': 'DeepSeek v4 Pro',
|
||||
'deepseek-v4-flash': 'DeepSeek v4 Flash',
|
||||
'deepseek-coder-max': 'DeepSeek Coder Max',
|
||||
'deepseek-coder': 'DeepSeek Coder',
|
||||
'deepseek-r1': 'DeepSeek R1',
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@ const shortNames: Record<string, string> = {
|
|||
'claude-3-5-sonnet': 'Sonnet 3.5',
|
||||
'claude-haiku-4-5': 'Haiku 4.5',
|
||||
'claude-3-5-haiku': 'Haiku 3.5',
|
||||
'deepseek-v4-pro': 'DeepSeek v4 Pro',
|
||||
'deepseek-v4-flash': 'DeepSeek v4 Flash',
|
||||
}
|
||||
|
||||
function expandHome(p: string): string {
|
||||
|
|
|
|||
133
tests/cli-deepseek-v4-pricing.test.ts
Normal file
133
tests/cli-deepseek-v4-pricing.test.ts
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
function runCli(args: string[], home: string) {
|
||||
return spawnSync(process.execPath, ['--import', 'tsx', 'src/cli.ts', ...args], {
|
||||
cwd: process.cwd(),
|
||||
env: {
|
||||
...process.env,
|
||||
CLAUDE_CONFIG_DIR: join(home, '.claude'),
|
||||
HOME: home,
|
||||
TZ: 'UTC',
|
||||
},
|
||||
encoding: 'utf-8',
|
||||
timeout: 30_000,
|
||||
})
|
||||
}
|
||||
|
||||
function userLine(content: string, timestamp: string): string {
|
||||
return JSON.stringify({
|
||||
type: 'user',
|
||||
sessionId: 'deepseek-v4-session',
|
||||
timestamp,
|
||||
cwd: '/tmp/deepseek-v4-validation',
|
||||
message: { role: 'user', content },
|
||||
})
|
||||
}
|
||||
|
||||
function assistantLine(model: string, timestamp: string, messageId: string, usage: Record<string, number>): string {
|
||||
return JSON.stringify({
|
||||
type: 'assistant',
|
||||
sessionId: 'deepseek-v4-session',
|
||||
timestamp,
|
||||
cwd: '/tmp/deepseek-v4-validation',
|
||||
message: {
|
||||
id: messageId,
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
model,
|
||||
content: [
|
||||
{ type: 'text', text: 'updated pricing code' },
|
||||
{ type: 'tool_use', id: `tu-${messageId}`, name: 'Edit', input: { file_path: '/tmp/deepseek-v4-validation/pricing.ts', old_string: 'old', new_string: 'new' } },
|
||||
],
|
||||
usage,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
describe('CLI DeepSeek v4 Claude pricing regression', () => {
|
||||
it('prices DeepSeek v4 Claude sessions even when the runtime LiteLLM cache lacks those models', async () => {
|
||||
const home = await mkdtemp(join(tmpdir(), 'codeburn-deepseek-v4-cli-'))
|
||||
|
||||
try {
|
||||
const projectDir = join(home, '.claude', 'projects', 'deepseek-v4-validation')
|
||||
const cacheDir = join(home, '.cache', 'codeburn')
|
||||
await mkdir(projectDir, { recursive: true })
|
||||
await mkdir(cacheDir, { recursive: true })
|
||||
|
||||
await writeFile(join(cacheDir, 'litellm-pricing.json'), JSON.stringify({
|
||||
timestamp: Date.now(),
|
||||
data: {
|
||||
'gpt-4o-mini': {
|
||||
inputCostPerToken: 1.5e-7,
|
||||
outputCostPerToken: 6e-7,
|
||||
cacheWriteCostPerToken: 0,
|
||||
cacheReadCostPerToken: 7.5e-8,
|
||||
webSearchCostPerRequest: 0.01,
|
||||
fastMultiplier: 1,
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
await writeFile(
|
||||
join(projectDir, 'session.jsonl'),
|
||||
[
|
||||
userLine('Use DeepSeek v4 through the Claude-compatible endpoint.', '2026-05-20T10:00:00.000Z'),
|
||||
assistantLine('deepseek-v4-pro', '2026-05-20T10:01:00.000Z', 'deepseek-v4-pro', {
|
||||
input_tokens: 2_477_914,
|
||||
output_tokens: 762_994,
|
||||
cache_read_input_tokens: 258_556_928,
|
||||
cache_creation_input_tokens: 0,
|
||||
}),
|
||||
userLine('Validate the flash model path too.', '2026-05-20T10:02:00.000Z'),
|
||||
assistantLine('deepseek-v4-flash', '2026-05-20T10:03:00.000Z', 'deepseek-v4-flash', {
|
||||
input_tokens: 1_552_573,
|
||||
output_tokens: 353_914,
|
||||
cache_read_input_tokens: 48_388_608,
|
||||
cache_creation_input_tokens: 0,
|
||||
}),
|
||||
].join('\n') + '\n',
|
||||
)
|
||||
|
||||
const result = runCli([
|
||||
'--format', 'json',
|
||||
'--from', '2026-05-20',
|
||||
'--to', '2026-05-20',
|
||||
'--provider', 'claude',
|
||||
], home)
|
||||
|
||||
expect(result.status, `stderr: ${result.stderr}`).toBe(0)
|
||||
|
||||
const report = JSON.parse(result.stdout) as {
|
||||
overview: { cost: number; calls: number; tokens: { cacheRead: number } }
|
||||
models: Array<{ name: string; cost: number; calls: number; inputTokens: number; outputTokens: number; cacheReadTokens: number }>
|
||||
}
|
||||
const pro = report.models.find(m => m.name === 'DeepSeek v4 Pro')
|
||||
const flash = report.models.find(m => m.name === 'DeepSeek v4 Flash')
|
||||
|
||||
expect(report.overview.calls).toBe(2)
|
||||
expect(report.overview.tokens.cacheRead).toBe(306_945_536)
|
||||
expect(report.overview.cost).toBeCloseTo(3.13091, 5)
|
||||
|
||||
expect(pro).toBeDefined()
|
||||
expect(pro!.calls).toBe(1)
|
||||
expect(pro!.inputTokens).toBe(2_477_914)
|
||||
expect(pro!.outputTokens).toBe(762_994)
|
||||
expect(pro!.cacheReadTokens).toBe(258_556_928)
|
||||
expect(pro!.cost).toBeCloseTo(2.678966, 6)
|
||||
|
||||
expect(flash).toBeDefined()
|
||||
expect(flash!.calls).toBe(1)
|
||||
expect(flash!.inputTokens).toBe(1_552_573)
|
||||
expect(flash!.outputTokens).toBe(353_914)
|
||||
expect(flash!.cacheReadTokens).toBe(48_388_608)
|
||||
expect(flash!.cost).toBeCloseTo(0.451944, 6)
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
|
@ -1,3 +1,6 @@
|
|||
import { mkdtemp, mkdir, rm, writeFile } from 'fs/promises'
|
||||
import { tmpdir } from 'os'
|
||||
import { join } from 'path'
|
||||
import { describe, it, expect, beforeAll, afterEach } from 'vitest'
|
||||
|
||||
import { getModelCosts, getShortModelName, calculateCost, loadPricing, setModelAliases } from '../src/models.js'
|
||||
|
|
@ -260,3 +263,78 @@ describe('Cursor model variants resolve to pricing', () => {
|
|||
})
|
||||
}
|
||||
})
|
||||
|
||||
describe('DeepSeek v4 models resolve to pricing', () => {
|
||||
it('deepseek-v4-pro has current official discounted pricing', () => {
|
||||
const costs = getModelCosts('deepseek-v4-pro')
|
||||
expect(costs).not.toBeNull()
|
||||
expect(costs!.inputCostPerToken).toBe(4.35e-7)
|
||||
expect(costs!.outputCostPerToken).toBe(8.7e-7)
|
||||
expect(costs!.cacheReadCostPerToken).toBe(3.625e-9)
|
||||
expect(costs!.cacheWriteCostPerToken).toBe(0)
|
||||
})
|
||||
|
||||
it('deepseek-v4-flash has current official pricing', () => {
|
||||
const costs = getModelCosts('deepseek-v4-flash')
|
||||
expect(costs).not.toBeNull()
|
||||
expect(costs!.inputCostPerToken).toBe(1.4e-7)
|
||||
expect(costs!.outputCostPerToken).toBe(2.8e-7)
|
||||
expect(costs!.cacheReadCostPerToken).toBe(2.8e-9)
|
||||
expect(costs!.cacheWriteCostPerToken).toBe(0)
|
||||
})
|
||||
|
||||
it('provider-prefixed DeepSeek v4 names resolve to the same pricing', () => {
|
||||
expect(getModelCosts('deepseek/deepseek-v4-pro')).toEqual(getModelCosts('deepseek-v4-pro'))
|
||||
expect(getModelCosts('deepseek/deepseek-v4-flash')).toEqual(getModelCosts('deepseek-v4-flash'))
|
||||
})
|
||||
|
||||
it('calculates non-zero costs for observed DeepSeek v4 Claude usage', () => {
|
||||
const pro = calculateCost('deepseek-v4-pro', 2_477_914, 762_994, 0, 258_556_928, 0)
|
||||
const flash = calculateCost('deepseek-v4-flash', 1_552_573, 353_914, 0, 48_388_608, 0)
|
||||
|
||||
expect(pro).toBeCloseTo(2.68, 2)
|
||||
expect(flash).toBeCloseTo(0.45, 2)
|
||||
})
|
||||
|
||||
it('uses DeepSeek v4 display names', () => {
|
||||
expect(getShortModelName('deepseek-v4-pro')).toBe('DeepSeek v4 Pro')
|
||||
expect(getShortModelName('deepseek-v4-flash')).toBe('DeepSeek v4 Flash')
|
||||
})
|
||||
|
||||
it('keeps bundled DeepSeek v4 fallback entries when runtime pricing cache is stale', async () => {
|
||||
const previousCacheDir = process.env['CODEBURN_CACHE_DIR']
|
||||
const cacheRoot = await mkdtemp(join(tmpdir(), 'codeburn-pricing-cache-'))
|
||||
|
||||
try {
|
||||
process.env['CODEBURN_CACHE_DIR'] = cacheRoot
|
||||
await mkdir(cacheRoot, { recursive: true })
|
||||
await writeFile(join(cacheRoot, 'litellm-pricing.json'), JSON.stringify({
|
||||
timestamp: Date.now(),
|
||||
data: {
|
||||
'gpt-4o-mini': {
|
||||
inputCostPerToken: 9e-7,
|
||||
outputCostPerToken: 1.8e-6,
|
||||
cacheWriteCostPerToken: 0,
|
||||
cacheReadCostPerToken: 9e-8,
|
||||
webSearchCostPerRequest: 0.01,
|
||||
fastMultiplier: 1,
|
||||
},
|
||||
},
|
||||
}), 'utf-8')
|
||||
|
||||
await loadPricing()
|
||||
|
||||
expect(getModelCosts('gpt-4o-mini')!.inputCostPerToken).toBe(9e-7)
|
||||
expect(getModelCosts('deepseek-v4-pro')!.inputCostPerToken).toBe(4.35e-7)
|
||||
expect(getModelCosts('deepseek-v4-flash')!.inputCostPerToken).toBe(1.4e-7)
|
||||
} finally {
|
||||
if (previousCacheDir === undefined) {
|
||||
delete process.env['CODEBURN_CACHE_DIR']
|
||||
} else {
|
||||
process.env['CODEBURN_CACHE_DIR'] = previousCacheDir
|
||||
}
|
||||
await rm(cacheRoot, { recursive: true, force: true })
|
||||
await loadPricing()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue