mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-07-18 13:34:24 +00:00
2fc4d32e66
Some checks are pending
CI / semgrep (push) Waiting to run
* fix(devin): report friendly GPT model names with effort tier (#487) Devin transcripts label each step with a generation_model in dash form with an effort suffix (gpt-5-3-codex-xhigh). getShortModelName is keyed in dot form (gpt-5.3-codex), and its version-boundary check matches the dash id against the base gpt-5 entry, collapsing every Devin GPT variant to GPT-5. Normalize the dash form to canonical dot form, map through the short-name table, and surface the effort tier, e.g. GPT-5.3 Codex (xhigh). Fall back to the friendly model_name when generation_model is an opaque MODEL_* id, and read extra.generation_model so ATIF v1.7 transcripts behave like v1.4. Devin rows in the JSON report and model-efficiency now key on the friendly name. * fix(devin): stop dated GPT snapshots being mislabeled as versions The friendly-name path treated any two-number dash id (gpt-4-1106-preview) as a dotted minor version, producing corrupt labels like 'GPT-4.1106 Preview'. - Restrict the dash-to-dot rewrite to a single-digit minor at a token boundary, matching Devin's real ids (gpt-5-3-codex) while leaving dated snapshots alone. - In getFriendlyGptName, defer to getShortModelName when any suffix token is purely numeric, so unknown snapshot ids pass through raw instead of being fabricated into a fake friendly name. Adds a regression case (gpt-4-1106-preview) to the Devin variant matrix. * chore: remove accidental node_modules symlink An absolute-path node_modules symlink was committed by mistake. It leaked a local username, was a dangling symlink for everyone else, and broke git/npm on checkout (git won't replace a real node_modules/ with the symlink). It slipped past .gitignore because the ignore rule is 'node_modules/' with a trailing slash, which matches a directory but not a symlink.
98 lines
2.9 KiB
TypeScript
98 lines
2.9 KiB
TypeScript
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
|
|
import { tmpdir } from 'node:os'
|
|
import { join } from 'node:path'
|
|
import { spawnSync } from 'node:child_process'
|
|
|
|
import { describe, expect, it } from 'vitest'
|
|
|
|
function runCli(args: string[], home: string) {
|
|
return spawnSync(process.execPath, ['--import', 'tsx', 'src/cli.ts', ...args], {
|
|
cwd: process.cwd(),
|
|
env: {
|
|
...process.env,
|
|
HOME: home,
|
|
USERPROFILE: home,
|
|
HOMEPATH: home,
|
|
HOMEDRIVE: '',
|
|
CLAUDE_CONFIG_DIR: join(home, '.claude'),
|
|
CODEBURN_CACHE_DIR: join(home, '.cache', 'codeburn'),
|
|
TZ: 'UTC',
|
|
},
|
|
encoding: 'utf-8',
|
|
timeout: 30_000,
|
|
})
|
|
}
|
|
|
|
describe('codeburn report Devin model variants', () => {
|
|
it('keeps friendly Devin effort-tier names in JSON model rows and efficiency rows', async () => {
|
|
const home = await mkdtemp(join(tmpdir(), 'codeburn-cli-devin-models-'))
|
|
try {
|
|
await mkdir(join(home, '.config', 'codeburn'), { recursive: true })
|
|
await writeFile(join(home, '.config', 'codeburn', 'config.json'), JSON.stringify({
|
|
devin: { acuUsdRate: 1 },
|
|
}))
|
|
|
|
const transcriptsDir = join(home, '.local', 'share', 'devin', 'cli', 'transcripts')
|
|
await mkdir(transcriptsDir, { recursive: true })
|
|
await writeFile(join(transcriptsDir, 'session-487.json'), JSON.stringify({
|
|
schema_version: '1.4',
|
|
session_id: 'session-487',
|
|
agent: { model_name: 'GPT-5.4' },
|
|
steps: [
|
|
{
|
|
step_id: 1,
|
|
message: 'fix the model row',
|
|
metadata: { is_user_input: true, created_at: '2026-04-10T09:00:00.000Z' },
|
|
},
|
|
{
|
|
step_id: 2,
|
|
message: 'editing',
|
|
tool_calls: [{ function_name: 'Edit' }],
|
|
metadata: {
|
|
created_at: '2026-04-10T09:01:00.000Z',
|
|
committed_acu_cost: 0.25,
|
|
generation_model: 'gpt-5-3-codex-xhigh',
|
|
metrics: { input_tokens: 100, output_tokens: 25 },
|
|
},
|
|
},
|
|
],
|
|
}))
|
|
|
|
const result = runCli([
|
|
'report',
|
|
'--format',
|
|
'json',
|
|
'--from',
|
|
'2026-04-10',
|
|
'--to',
|
|
'2026-04-10',
|
|
'--provider',
|
|
'devin',
|
|
], home)
|
|
|
|
expect(result.status, result.stderr).toBe(0)
|
|
const report = JSON.parse(result.stdout) as {
|
|
models: Array<{
|
|
name: string
|
|
calls: number
|
|
cost: number
|
|
editTurns: number
|
|
oneShotTurns: number
|
|
costPerEdit: number | null
|
|
}>
|
|
}
|
|
|
|
expect(report.models).toHaveLength(1)
|
|
expect(report.models[0]).toMatchObject({
|
|
name: 'GPT-5.3 Codex (xhigh)',
|
|
calls: 1,
|
|
cost: 0.25,
|
|
editTurns: 1,
|
|
oneShotTurns: 1,
|
|
costPerEdit: 0.25,
|
|
})
|
|
} finally {
|
|
await rm(home, { recursive: true, force: true })
|
|
}
|
|
})
|
|
})
|