Merge c105cf691e into 1aa05bdbbe

docs/design/devin-full-model-name-reporting.md

@@ -0,0 +1,118 @@
+# Devin Full Model Name Reporting
+
+- **Date:** 2026-06-12
+- **Status:** Proposed
+- **Scope:** `codeburn report --format json` model names for Devin sessions
+
+## Problem
+
+Devin transcripts include a specific generation model id (for example
+`gpt-5-3-codex-xhigh`), but the JSON report currently shows a collapsed short
+name (for example `GPT-5`).
+
+Example mismatch:
+
+- Transcript field: `steps[].metadata.generation_model = "gpt-5-3-codex-xhigh"`
+- Report field: `models[].name = "GPT-5"`
+
+## Root Cause
+
+1. The Devin provider already extracts the full model id in this order:
+   `step.metadata.generation_model` -> `step.model_name` ->
+   `transcript.agent.model_name` -> `sessions.model` -> `"devin"`.
+2. Later, parser session summarization buckets all providers with
+   `getShortModelName(call.model)`, which collapses many model variants into
+   broad display buckets.
+3. `buildJsonReport()` reads `session.modelBreakdown` keys directly, so once the
+   model key is collapsed in the parser, the JSON report cannot recover the
+   original full id.
+
+### Code references
+
+- Devin model extraction: `src/providers/devin.ts:getModelName()`
+- Collapsing point: `src/parser.ts:buildSessionSummary()` using
+  `getShortModelName(call.model)`
+- JSON report output source: `src/main.ts:buildJsonReport()` iterating
+  `session.modelBreakdown`
+
+## Proposed Change
+
+Apply provider-aware model-key bucketing during parser aggregation:
+
+- Keep existing short-name bucketing for most providers.
+- Preserve raw `call.model` for provider `devin`.
+
+### Targeted code changes
+
+1. **`src/parser.ts`**
+   - In `buildSessionSummary()`, replace direct use of
+     `getShortModelName(call.model)` with a small helper:
+     - `if (call.provider === 'devin') return call.model`
+     - otherwise return `getShortModelName(call.model)`.
+   - Use this helper for `modelBreakdown` keying.
+
+2. **`src/model-efficiency.ts`**
+   - Apply the same provider-aware keying logic when computing model efficiency.
+   - This keeps `editTurns/oneShotRate/costPerEdit` aligned with
+     `modelBreakdown` names surfaced in JSON.
+
+3. **`src/providers/devin.ts`** (for `codeburn models` command consistency)
+   - Change `modelDisplayName(model)` to preserve the raw model id for Devin
+     instead of re-shortening through `getShortModelName()`.
+   - This avoids a separate path re-collapsing Devin variants in model-table
+     output.
+
+## Claude Comparison
+
+Yes, Claude output is currently short-name grouped by design.
+
+- Claude provider display uses `getShortModelName(model)`.
+- Claude parser stores raw `message.model` on each call, but session-level model
+  breakdown is still keyed through parser-wide short-name bucketing.
+
+That is why Claude report JSON typically shows names like:
+
+- `Opus 4.6`
+- `Opus 4.8`
+- `Haiku 4.5`
+
+instead of raw ids such as `claude-opus-4-6`.
+
+This proposal intentionally changes only Devin behavior.
+
+## Expected Result
+
+After implementation, Devin rows in `report --format json` should preserve full
+model ids when available, for example:
+
+- `gpt-5-3-codex-xhigh`
+
+instead of a collapsed bucket such as:
+
+- `GPT-5`
+
+## Tests
+
+1. **Provider parsing test**
+   - Extend `tests/providers/devin.test.ts` with a call whose
+     `generation_model` is a variant id (`gpt-5-3-codex-xhigh`) and assert that
+     parsed call model remains exact.
+
+2. **CLI JSON regression test**
+   - Add a CLI integration test (similar style to `tests/cli-*.test.ts`) that:
+     - writes a Devin transcript fixture
+     - runs `codeburn --format json ...`
+     - asserts `models[].name` contains `gpt-5-3-codex-xhigh`.
+
+3. **Model-efficiency alignment test**
+   - Add or extend a test to verify efficiency fields are computed under the
+     same model key used by JSON model rows for Devin sessions.
+
+## Risks / Trade-offs
+
+- **Higher model-cardinality for Devin:** more distinct model rows (desired for
+  forensic accuracy).
+- **Mixed naming styles across providers:** Devin may show raw ids while Claude
+  remains short-name grouped (intentional in this proposal).
+- **No cache migration required:** cached turns already store raw `call.model`;
+  the changed grouping is applied at summary-build time.

src/model-efficiency.ts

@@ -22,6 +22,11 @@ function rate(num: number, den: number): number | null {
 export function aggregateModelEfficiency(projects: ProjectSummary[]): Map<string, ModelEfficiency> {
   const byModel = new Map<string, MutableModelEfficiency>()
 
+  function modelKey(provider: string, model: string): string {
+    if (provider === 'devin') return model
+    return getShortModelName(model)
+  }
+
   function ensure(model: string): MutableModelEfficiency {
     let stats = byModel.get(model)
     if (!stats) {
@@ -36,16 +41,16 @@ export function aggregateModelEfficiency(projects: ProjectSummary[]): Map<string
       for (const turn of session.turns) {
         if (!turn.hasEdits || turn.assistantCalls.length === 0) continue
 
-        const primaryCall = turn.assistantCalls.find(c => getShortModelName(c.model) !== '<synthetic>')
+        const primaryCall = turn.assistantCalls.find(c => modelKey(c.provider, c.model) !== '<synthetic>')
         if (!primaryCall) continue
-        const primaryModel = getShortModelName(primaryCall.model)
+        const primaryModel = modelKey(primaryCall.provider, primaryCall.model)
 
         const stats = ensure(primaryModel)
         stats.editTurns++
         if (turn.retries === 0) stats.oneShotTurns++
         stats.retries += turn.retries
         stats.editCostUSD += turn.assistantCalls.reduce((sum, call) => {
-          return getShortModelName(call.model) === '<synthetic>' ? sum : sum + call.costUSD
+          return modelKey(call.provider, call.model) === '<synthetic>' ? sum : sum + call.costUSD
         }, 0)
       }
     }

src/parser.ts

@@ -1294,6 +1294,11 @@ function buildSessionSummary(
   let firstTs = ''
   let lastTs = ''
 
+  function modelBreakdownKey(call: ParsedApiCall): string {
+    if (call.provider === 'devin') return call.model
+    return getShortModelName(call.model)
+  }
+
   for (const turn of turns) {
     const turnCost = turn.assistantCalls.reduce((s, c) => s + c.costUSD, 0)
     const turnSavings = turn.assistantCalls.reduce((s, c) => s + (c.savingsUSD ?? 0), 0)
@@ -1335,7 +1340,7 @@ function buildSessionSummary(
       totalCacheWrite += call.usage.cacheCreationInputTokens
       apiCalls++
 
-      const modelKey = getShortModelName(call.model)
+      const modelKey = modelBreakdownKey(call)
       if (!modelBreakdown[modelKey]) {
         modelBreakdown[modelKey] = {
           calls: 0,

src/providers/devin.ts

@@ -2,7 +2,6 @@ import { readdir, stat } from "fs/promises";
 import { basename, join } from "path";
 import { homedir } from "os";
 
-import { getShortModelName } from "../models.js";
 import { openDatabase } from "../sqlite.js";
 import { readConfig } from "../config.js";
 import type {
@@ -336,7 +335,7 @@ export function createDevinProvider(cliDir: string): Provider {
     displayName: DEVIN_PROVIDER_DISPLAY_NAME,
 
     modelDisplayName(model: string): string {
-      return getShortModelName(model);
+      return model;
     },
 
     toolDisplayName(rawTool: string): string {

tests/cli-devin-model-variants.test.ts

@@ -0,0 +1,106 @@
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { spawnSync } from 'node:child_process'
+
+import { describe, expect, it } from 'vitest'
+
+function runCli(args: string[], home: string) {
+  return spawnSync(process.execPath, ['--import', 'tsx', 'src/cli.ts', ...args], {
+    cwd: process.cwd(),
+    env: {
+      ...process.env,
+      HOME: home,
+      CLAUDE_CONFIG_DIR: join(home, '.claude'),
+      TZ: 'UTC',
+    },
+    encoding: 'utf-8',
+    timeout: 30_000,
+  })
+}
+
+describe('CLI Devin model variants', () => {
+  it('keeps Devin GPT-5 variants separate in report JSON', async () => {
+    const home = await mkdtemp(join(tmpdir(), 'codeburn-devin-models-'))
+
+    try {
+      const cliDir = join(home, '.local', 'share', 'devin', 'cli')
+      const transcriptsDir = join(cliDir, 'transcripts')
+      await mkdir(transcriptsDir, { recursive: true })
+      await mkdir(join(home, '.config', 'codeburn'), { recursive: true })
+      await writeFile(join(home, '.config', 'codeburn', 'config.json'), JSON.stringify({
+        devin: { acuUsdRate: 2.25 },
+      }))
+
+      const transcript = {
+        schema_version: '1',
+        session_id: 'devin-model-variants',
+        agent: { model_name: 'GPT-5.3-Codex' },
+        steps: [
+          {
+            step_id: 1,
+            source: 'user',
+            message: 'compare model variants',
+            metadata: {
+              is_user_input: true,
+              created_at: '2026-06-12T10:00:00.000Z',
+            },
+          },
+          {
+            step_id: 2,
+            source: 'agent',
+            message: '',
+            metadata: {
+              created_at: '2026-06-12T10:00:01.000Z',
+              committed_acu_cost: 0.1,
+              generation_model: 'gpt-5-3-codex-xhigh',
+              metrics: {
+                input_tokens: 100,
+                output_tokens: 10,
+              },
+            },
+          },
+          {
+            step_id: 3,
+            source: 'agent',
+            message: '',
+            metadata: {
+              created_at: '2026-06-12T10:00:02.000Z',
+              committed_acu_cost: 0.2,
+              generation_model: 'gpt-5-4-low',
+              metrics: {
+                input_tokens: 200,
+                output_tokens: 20,
+              },
+            },
+          },
+        ],
+      }
+      await writeFile(
+        join(transcriptsDir, 'devin-model-variants.json'),
+        JSON.stringify(transcript),
+      )
+
+      const result = runCli([
+        '--format', 'json',
+        '--from', '2026-06-12',
+        '--to', '2026-06-12',
+        '--provider', 'devin',
+      ], home)
+
+      expect(result.status, `stderr: ${result.stderr}`).toBe(0)
+
+      const report = JSON.parse(result.stdout) as {
+        models: Array<{ name: string; calls: number }>
+      }
+      const names = report.models.map(m => m.name)
+      expect(names).toEqual(expect.arrayContaining([
+        'gpt-5-3-codex-xhigh',
+        'gpt-5-4-low',
+      ]))
+      expect(names).not.toContain('GPT-5')
+    } finally {
+      await rm(home, { recursive: true, force: true })
+    }
+  })
+})

tests/model-efficiency.test.ts

@@ -3,9 +3,9 @@ import { describe, expect, it } from 'vitest'
 import { aggregateModelEfficiency } from '../src/model-efficiency.js'
 import type { ClassifiedTurn, ParsedApiCall, ProjectSummary, SessionSummary } from '../src/types.js'
 
-function call(model: string, costUSD = 1): ParsedApiCall {
+function call(model: string, costUSD = 1, provider: 'claude' | 'devin' = 'claude'): ParsedApiCall {
   return {
-    provider: 'claude',
+    provider,
     model,
     usage: {
       inputTokens: 100,
@@ -25,7 +25,7 @@ function call(model: string, costUSD = 1): ParsedApiCall {
     speed: 'standard',
     timestamp: '2026-05-05T00:00:00Z',
     bashCommands: [],
-    deduplicationKey: `${model}-${costUSD}`,
+    deduplicationKey: `${provider}-${model}-${costUSD}`,
   }
 }
 
@@ -138,4 +138,18 @@ describe('aggregateModelEfficiency', () => {
 
     expect(stats.size).toBe(0)
   })
+
+  it('keeps Devin model variants as raw model ids', () => {
+    const stats = aggregateModelEfficiency([project([
+      turn('gpt-5-3-codex-xhigh', { hasEdits: true, retries: 0, costUSD: 2 }),
+      turn('gpt-5-4-low', { hasEdits: true, retries: 1, costUSD: 3 }),
+    ].map(t => ({
+      ...t,
+      assistantCalls: t.assistantCalls.map(c => ({ ...c, provider: 'devin' as const })),
+    })))])
+
+    expect(stats.has('gpt-5-3-codex-xhigh')).toBe(true)
+    expect(stats.has('gpt-5-4-low')).toBe(true)
+    expect(stats.has('GPT-5')).toBe(false)
+  })
 })

tests/providers/devin.test.ts

@@ -70,6 +70,11 @@ function createSessionsDb(): void {
 }
 
 describe('devin provider', () => {
+  it('keeps raw model ids in display names', () => {
+    const provider = createDevinProvider(tmpDir)
+    expect(provider.modelDisplayName('gpt-5-3-codex-xhigh')).toBe('gpt-5-3-codex-xhigh')
+  })
+
   it('discovers Devin CLI transcript json files', async () => {
     await configureDevinRate()
     const filePath = await writeTranscript('glimmer-platinum.json', { steps: [] })