feat(providers): add open-design provider for per-model usage tracking (#559)

This commit is contained in:
ozymandiashh 2026-06-28 20:17:23 +03:00 committed by GitHub
parent 2a0edd0d68
commit fd44d995c7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 429 additions and 2 deletions

View file

@ -243,6 +243,7 @@ const BUILTIN_ALIASES: Record<string, string> = {
'copilot-auto': 'claude-sonnet-4-5',
'copilot-openai-auto': 'gpt-5.3-codex',
'copilot-anthropic-auto': 'claude-sonnet-4-5',
'openai-codex:gpt-5.5': 'gpt-5.5',
'ibm-bob-auto': 'claude-sonnet-4-5',
'kiro-auto': 'claude-sonnet-4-5',
'cline-auto': 'claude-sonnet-4-5',

View file

@ -14,6 +14,7 @@ import { kimi } from './kimi.js'
import { mistralVibe } from './mistral-vibe.js'
import { mux } from './mux.js'
import { openclaw } from './openclaw.js'
import { openDesign } from './open-design.js'
import { pi, omp } from './pi.js'
import { qwen } from './qwen.js'
import { rooCode } from './roo-code.js'
@ -170,7 +171,7 @@ async function loadZcode(): Promise<Provider | null> {
}
}
const coreProviders: Provider[] = [claude, cline, codebuff, codex, copilot, devin, droid, gemini, hermes, ibmBob, kiloCode, kiro, kimi, mistralVibe, mux, openclaw, pi, omp, qwen, rooCode, zerostack, grok]
const coreProviders: Provider[] = [claude, cline, codebuff, codex, copilot, devin, droid, gemini, hermes, ibmBob, kiloCode, kiro, kimi, mistralVibe, mux, openclaw, openDesign, pi, omp, qwen, rooCode, zerostack, grok]
// Lazily loaded providers, listed by name so --provider validation works even
// when an optional module fails to load. Must stay in sync with getAllProviders.

View file

@ -0,0 +1,259 @@
import { readdir, stat } from 'fs/promises'
import { basename, dirname, join } from 'path'
import { homedir, platform } from 'os'
import { readSessionLines } from '../fs-utils.js'
import { calculateCost } from '../models.js'
import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js'
const PROVIDER_NAME = 'open-design'
const ENV_DIR = 'CODEBURN_OPEN_DESIGN_DIR'
const modelDisplayNames = new Map<string, string>([
['openai-codex:gpt-5.5', 'GPT-5.5'],
['glm-5.2', 'GLM-5.2'],
['GLM-5.2', 'GLM-5.2'],
])
type OpenDesignEntry = {
id?: unknown
event?: unknown
data?: unknown
timestamp?: unknown
}
type TokenUsage = {
inputTokens: number
outputTokens: number
cacheReadTokens: number
reasoningTokens: number
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value)
}
function stringValue(value: unknown): string | undefined {
return typeof value === 'string' && value.length > 0 ? value : undefined
}
function tokenValue(value: unknown): number {
return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0
}
function timestampValue(value: unknown): string {
const text = stringValue(value)
if (text) return text
if (typeof value !== 'number' || !Number.isFinite(value)) return ''
const date = new Date(value)
return Number.isNaN(date.getTime()) ? '' : date.toISOString()
}
function parseEvent(line: string | Buffer): OpenDesignEntry | null {
const text = (typeof line === 'string' ? line : line.toString('utf-8')).trim()
if (!text) return null
try {
const parsed = JSON.parse(text) as unknown
return isRecord(parsed) ? parsed : null
} catch {
return null
}
}
function parseUsage(data: unknown): TokenUsage | null {
if (!isRecord(data) || data['type'] !== 'usage') return null
const usage = data['usage']
if (!isRecord(usage)) return null
return {
inputTokens: tokenValue(usage['input_tokens']),
outputTokens: tokenValue(usage['output_tokens']),
cacheReadTokens: tokenValue(usage['cached_read_tokens']),
reasoningTokens: tokenValue(usage['thought_tokens']),
}
}
function getOpenDesignDir(): string {
const override = process.env[ENV_DIR]
if (override) return override
const home = homedir()
const os = platform()
if (os === 'darwin') {
return join(home, 'Library', 'Application Support', 'Open Design')
}
if (os === 'win32') {
return join(process.env['APPDATA'] ?? join(home, 'AppData', 'Roaming'), 'Open Design')
}
return join(home, '.config', 'Open Design')
}
function namespaceFromDataDir(dataDir: string): string {
const ns = basename(dirname(dataDir))
return ns && ns !== 'namespaces' ? ns : PROVIDER_NAME
}
function namespaceFromRunsDir(runsDir: string): string {
return namespaceFromDataDir(dirname(runsDir))
}
async function discoverRunsDir(runsDir: string, project: string): Promise<SessionSource[]> {
const sources: SessionSource[] = []
let runDirs: string[]
try {
runDirs = await readdir(runsDir)
} catch {
return sources
}
for (const runDir of runDirs) {
const eventsPath = join(runsDir, runDir, 'events.jsonl')
const s = await stat(eventsPath).catch(() => null)
if (!s?.isFile()) continue
sources.push({ path: eventsPath, project, provider: PROVIDER_NAME })
}
return sources
}
async function discoverNamespacesDir(namespacesDir: string): Promise<SessionSource[]> {
const sources: SessionSource[] = []
let namespaces: string[]
try {
namespaces = await readdir(namespacesDir)
} catch {
return sources
}
for (const ns of namespaces) {
const runsDir = join(namespacesDir, ns, 'data', 'runs')
sources.push(...await discoverRunsDir(runsDir, ns))
}
return sources
}
function dedupeSources(sources: SessionSource[]): SessionSource[] {
const seen = new Set<string>()
const out: SessionSource[] = []
for (const source of sources) {
if (seen.has(source.path)) continue
seen.add(source.path)
out.push(source)
}
return out
}
async function discoverOpenDesignSessions(baseDir: string): Promise<SessionSource[]> {
const baseName = basename(baseDir)
if (baseName === 'runs') {
return discoverRunsDir(baseDir, namespaceFromRunsDir(baseDir))
}
if (baseName === 'data') {
return discoverRunsDir(join(baseDir, 'runs'), namespaceFromDataDir(baseDir))
}
const sources: SessionSource[] = []
sources.push(...await discoverRunsDir(join(baseDir, 'data', 'runs'), basename(baseDir) || PROVIDER_NAME))
sources.push(...await discoverRunsDir(join(baseDir, 'runs'), basename(baseDir) || PROVIDER_NAME))
sources.push(...await discoverNamespacesDir(baseName === 'namespaces' ? baseDir : join(baseDir, 'namespaces')))
return dedupeSources(sources)
}
function createParser(source: SessionSource, seenKeys: Set<string>): SessionParser {
return {
async *parse(): AsyncGenerator<ParsedProviderCall> {
const sessionId = basename(dirname(source.path))
let currentModel = ''
let fallbackEventCounter = 0
for await (const line of readSessionLines(source.path)) {
const entry = parseEvent(line)
if (!entry) continue
const eventName = stringValue(entry.event)
const data = entry.data
if (eventName === 'start' && isRecord(data)) {
const model = stringValue(data['model'])
if (model) currentModel = model
continue
}
if (eventName !== 'agent' || !isRecord(data)) continue
if (data['type'] === 'status') {
const model = stringValue(data['model'])
if (model) currentModel = model
continue
}
const usage = parseUsage(data)
if (!usage || !currentModel) continue
const eventId = stringValue(entry.id) ?? `line-${fallbackEventCounter++}`
const dedupKey = `${PROVIDER_NAME}:${sessionId}:${eventId}`
if (seenKeys.has(dedupKey)) continue
seenKeys.add(dedupKey)
const uncachedInputTokens = Math.max(0, usage.inputTokens - usage.cacheReadTokens)
const costUSD = calculateCost(
currentModel,
uncachedInputTokens,
usage.outputTokens + usage.reasoningTokens,
0,
usage.cacheReadTokens,
0,
)
yield {
provider: PROVIDER_NAME,
sessionId,
project: source.project,
model: currentModel,
inputTokens: uncachedInputTokens,
outputTokens: usage.outputTokens,
cacheCreationInputTokens: 0,
cacheReadInputTokens: usage.cacheReadTokens,
cachedInputTokens: usage.cacheReadTokens,
reasoningTokens: usage.reasoningTokens,
webSearchRequests: 0,
costUSD,
tools: [],
bashCommands: [],
timestamp: timestampValue(entry.timestamp),
speed: 'standard',
deduplicationKey: dedupKey,
userMessage: '',
}
}
},
}
}
export function createOpenDesignProvider(overrideDir?: string): Provider {
return {
name: PROVIDER_NAME,
displayName: 'Open Design',
modelDisplayName(model: string): string {
return modelDisplayNames.get(model) ?? model
},
toolDisplayName(rawTool: string): string {
return rawTool
},
async discoverSessions(): Promise<SessionSource[]> {
return discoverOpenDesignSessions(overrideDir ?? getOpenDesignDir())
},
createSessionParser(source: SessionSource, seenKeys: Set<string>): SessionParser {
return createParser(source, seenKeys)
},
}
}
export const openDesign = createOpenDesignProvider()

View file

@ -0,0 +1,5 @@
{"id":"evt-start","event":"start","timestamp":"2026-06-22T10:00:00.000Z","data":{"model":"openai-codex:gpt-5.5"}}
{"id":"evt-codex-usage","event":"agent","timestamp":1782122405000,"data":{"type":"usage","usage":{"input_tokens":1000,"output_tokens":200,"cached_read_tokens":50,"thought_tokens":25,"total_tokens":1200},"costUsd":null}}
{"id":"evt-status-glm","event":"agent","timestamp":"2026-06-22T10:00:10.000Z","data":{"type":"status","model":"glm-5.2"}}
{"id":"evt-glm-usage","event":"agent","timestamp":"2026-06-22T10:00:15.000Z","data":{"type":"usage","usage":{"input_tokens":3000,"output_tokens":400,"cached_read_tokens":100,"thought_tokens":60,"total_tokens":3560},"costUsd":null}}
{"id":"evt-glm-usage","event":"agent","timestamp":"2026-06-22T10:00:16.000Z","data":{"type":"usage","usage":{"input_tokens":9999,"output_tokens":9999,"cached_read_tokens":9999,"thought_tokens":9999,"total_tokens":39996},"costUsd":null}}

View file

@ -0,0 +1,3 @@
{"id":"evt-start","event":"start","timestamp":"2026-06-22T11:00:00.000Z","data":{"model":"glm-5.2"}}
{"id":"evt-status","event":"agent","timestamp":"2026-06-22T11:00:05.000Z","data":{"type":"status","model":"openai-codex:gpt-5.5"}}
{"id":"evt-message","event":"agent","timestamp":"2026-06-22T11:00:10.000Z","data":{"type":"message","text":"no usage was emitted"}}

View file

@ -0,0 +1,3 @@
{"id":"evt-start","event":"start","timestamp":"2026-06-22T12:00:00.000Z","data":{"model":"glm-5.2"}}
{"id":"evt-before-transition","event":"agent","timestamp":"2026-06-22T12:00:05.000Z","data":{"type":"usage","usage":{"input_tokens":777,"output_tokens":33,"cached_read_tokens":7,"thought_tokens":3,"total_tokens":820},"costUsd":null}}
{"id":"evt-status","event":"agent","timestamp":"2026-06-22T12:00:10.000Z","data":{"type":"status","model":"openai-codex:gpt-5.5"}}

View file

@ -3,7 +3,7 @@ import { providers, getAllProviders, getProvider } from '../src/providers/index.
describe('provider registry', () => {
it('has core providers registered synchronously', () => {
expect(providers.map(p => p.name)).toEqual(['claude', 'cline', 'codebuff', 'codex', 'copilot', 'devin', 'droid', 'gemini', 'hermes', 'ibm-bob', 'kilo-code', 'kiro', 'kimi', 'mistral-vibe', 'mux', 'openclaw', 'pi', 'omp', 'qwen', 'roo-code', 'zerostack', 'grok'])
expect(providers.map(p => p.name)).toEqual(['claude', 'cline', 'codebuff', 'codex', 'copilot', 'devin', 'droid', 'gemini', 'hermes', 'ibm-bob', 'kilo-code', 'kiro', 'kimi', 'mistral-vibe', 'mux', 'openclaw', 'open-design', 'pi', 'omp', 'qwen', 'roo-code', 'zerostack', 'grok'])
})
it('codebuff tool display names normalize codebuff-native names to canonical set', () => {

View file

@ -0,0 +1,155 @@
import { mkdtemp, rm } from 'fs/promises'
import { tmpdir } from 'os'
import { join } from 'path'
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
import { calculateCost } from '../../src/models.js'
import { clearSessionCache, filterProjectsByDateRange, parseAllSessions } from '../../src/parser.js'
import { allProviderNames } from '../../src/providers/index.js'
import { createOpenDesignProvider } from '../../src/providers/open-design.js'
import type { ParsedProviderCall, SessionSource } from '../../src/providers/types.js'
const fixtureRoot = join(import.meta.dirname, '../fixtures/open-design')
const dataDir = join(fixtureRoot, 'namespaces', 'release-stable', 'data')
let previousOverride: string | undefined
let previousCacheDir: string | undefined
let cacheDir: string | undefined
async function collect(source: SessionSource, seenKeys = new Set<string>()): Promise<ParsedProviderCall[]> {
const provider = createOpenDesignProvider()
const calls: ParsedProviderCall[] = []
for await (const call of provider.createSessionParser(source, seenKeys).parse()) calls.push(call)
return calls
}
async function fixtureSource(runId: string): Promise<SessionSource> {
const provider = createOpenDesignProvider()
const sources = await provider.discoverSessions()
const source = sources.find(s => s.path.includes(`${runId}/events.jsonl`))
expect(source).toBeDefined()
return source!
}
describe('open-design provider', () => {
beforeEach(async () => {
previousOverride = process.env['CODEBURN_OPEN_DESIGN_DIR']
previousCacheDir = process.env['CODEBURN_CACHE_DIR']
cacheDir = await mkdtemp(join(tmpdir(), 'codeburn-open-design-cache-'))
process.env['CODEBURN_OPEN_DESIGN_DIR'] = dataDir
process.env['CODEBURN_CACHE_DIR'] = cacheDir
clearSessionCache()
})
afterEach(async () => {
clearSessionCache()
if (previousOverride === undefined) {
delete process.env['CODEBURN_OPEN_DESIGN_DIR']
} else {
process.env['CODEBURN_OPEN_DESIGN_DIR'] = previousOverride
}
if (previousCacheDir === undefined) {
delete process.env['CODEBURN_CACHE_DIR']
} else {
process.env['CODEBURN_CACHE_DIR'] = previousCacheDir
}
if (cacheDir) await rm(cacheDir, { recursive: true, force: true })
cacheDir = undefined
})
it('discovers per-run events.jsonl files from the env override data dir', async () => {
const provider = createOpenDesignProvider()
const sources = await provider.discoverSessions()
expect(sources.map(s => s.provider).every(p => p === 'open-design')).toBe(true)
expect(sources.map(s => s.project)).toEqual(['release-stable', 'release-stable', 'release-stable'])
expect(sources.map(s => s.path).sort()).toEqual([
join(dataDir, 'runs', 'run-mixed', 'events.jsonl'),
join(dataDir, 'runs', 'run-no-usage', 'events.jsonl'),
join(dataDir, 'runs', 'run-start-seeded', 'events.jsonl'),
].sort())
})
it('parses a mixed-model run into separate per-model usage calls', async () => {
const calls = await collect(await fixtureSource('run-mixed'))
expect(calls).toHaveLength(2)
expect(calls.map(c => c.model)).toEqual(['openai-codex:gpt-5.5', 'glm-5.2'])
const codex = calls[0]!
expect(codex.provider).toBe('open-design')
expect(codex.sessionId).toBe('run-mixed')
expect(codex.inputTokens).toBe(950)
expect(codex.outputTokens).toBe(200)
expect(codex.cacheCreationInputTokens).toBe(0)
expect(codex.cacheReadInputTokens).toBe(50)
expect(codex.cachedInputTokens).toBe(50)
expect(codex.reasoningTokens).toBe(25)
expect(codex.timestamp).toBe('2026-06-22T10:00:05.000Z')
expect(new Date(codex.timestamp).toISOString()).toBe(codex.timestamp)
expect(codex.costUSD).toBeCloseTo(
calculateCost(codex.model, 950, 225, 0, 50, 0),
12,
)
const glm = calls[1]!
expect(glm.inputTokens).toBe(2900)
expect(glm.outputTokens).toBe(400)
expect(glm.cacheCreationInputTokens).toBe(0)
expect(glm.cacheReadInputTokens).toBe(100)
expect(glm.cachedInputTokens).toBe(100)
expect(glm.reasoningTokens).toBe(60)
expect(glm.timestamp).toBe('2026-06-22T10:00:15.000Z')
expect(glm.costUSD).toBeGreaterThan(0)
})
it('does not emit calls for a run with no usage events', async () => {
const calls = await collect(await fixtureSource('run-no-usage'))
expect(calls).toHaveLength(0)
})
it('uses the start-seeded model before any status transition', async () => {
const calls = await collect(await fixtureSource('run-start-seeded'))
expect(calls).toHaveLength(1)
expect(calls[0]!.model).toBe('glm-5.2')
expect(calls[0]!.inputTokens).toBe(770)
expect(calls[0]!.outputTokens).toBe(33)
expect(calls[0]!.cacheReadInputTokens).toBe(7)
expect(calls[0]!.reasoningTokens).toBe(3)
expect(calls[0]!.costUSD).toBeGreaterThan(0)
})
it('keeps numeric epoch timestamp usage in date-scoped aggregation', async () => {
const projects = await parseAllSessions(undefined, 'open-design')
const filtered = filterProjectsByDateRange(projects, {
start: new Date('2026-06-22T00:00:00.000Z'),
end: new Date('2026-06-22T23:59:59.999Z'),
})
const calls = filtered.flatMap(project =>
project.sessions.flatMap(session =>
session.turns.flatMap(turn => turn.assistantCalls),
),
)
const numericTimestampCall = calls.find(call =>
call.deduplicationKey === 'open-design:run-mixed:evt-codex-usage',
)
expect(numericTimestampCall?.timestamp).toBe('2026-06-22T10:00:05.000Z')
})
it('deduplicates usage events per run and event id across parser runs', async () => {
const source = await fixtureSource('run-mixed')
const seenKeys = new Set<string>()
const first = await collect(source, seenKeys)
const second = await collect(source, seenKeys)
expect(first).toHaveLength(2)
expect(second).toHaveLength(0)
})
it('registers open-design as a core provider', () => {
expect(allProviderNames()).toContain('open-design')
})
})