fix(copilot): classify CLI sessions by source provenance, not producer (#945)
Some checks failed
CI / semgrep (push) Has been cancelled
Tests / test (push) Has been cancelled

Fixes #944.
This commit is contained in:
Matthew Kelch 2026-08-08 21:51:44 -04:00 committed by GitHub
parent 74e69ba2fd
commit 3536a1d3ac
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 630 additions and 60 deletions

View file

@ -12,6 +12,8 @@
- `codeburn sync push --attribution` (opt-in): sends git attribution spans — the session→commit correlation from `codeburn yield` (`codeburn.session.attribution` and `codeburn.commit` span types with normalized repo remote, commit SHAs, merged/reverted state, and PR links). Nothing new is sent without the flag; local-only repos and Windows filesystem paths are never emitted as repo identities, and sessions whose project path no longer resolves never inherit the push-time working directory's repo. See docs/sync/README.md "Git attribution".
### Fixed (CLI)
- **Copilot CLI sessions report their input and cache tokens.** The Copilot CLI writes the same `producer: 'copilot-agent'` in its `session.start` events that VS Code transcripts carry, so content-based detection classified every CLI session as a transcript and skipped its `session.shutdown` rollup — the only place the CLI records input, cache-read and cache-write tokens — leaving cache hit rate at 0.0% and dramatically underreporting cost. Whether a file is a transcript is now decided by where discovery found it, never by its contents. Resumed sessions, whose legs each append a cumulative rollup, are billed as per-leg deltas so a growing session never double-counts or goes stale; the GitHub Copilot desktop app writes the same session store, so its usage is covered by the same fix. The copilot session cache takes a parse-version bump and the daily cache bumps from v16 to v17 for the one-time re-parse that heals already-recorded days whose logs still exist. (#944)
- **Copilot CLI subagent runs are attributed to their agent.** Newer CLIs announce delegation with `subagent.started`/`subagent.completed` rather than `subagent.selected`, so delegated turns lost their agent label; the label now also clears when the subagent completes instead of bleeding onto the parent's later turns. Rides the #944 re-parse, so already-cached sessions gain the attribution. (#944)
- **`--project` / `--exclude` now apply to the headline totals, not just the detail panels.** The durable headline unions the carry-forward daily cache with today's live parse, and the cached days were sliced to the requested provider but never to the requested project — so the Overview panel counted excluded projects while By Project / By Activity / By Model (built from the name-filtered parse) left them out, and the two could not be reconciled. Cost, calls, sessions and savings are now sliced out of the per-project day stats the cache has carried since v15. Tokens, models and categories have no per-project split in the cache, so under a project filter they come from the (project-filtered) live parse instead; cached days — or provider slices — carried from before v15 have no project split at all, so they cannot be attributed to a filtered project, and the terminal overview now states how much was set aside rather than folding it into the total. (#864)
- **Codex parser corrections**: fork-replay no longer double-counts `patch_apply_end` and `mcp_tool_call_end`; `exec` is normalized to Bash; `custom_tool_call` events are handled; token_count lines larger than 32 KiB now parse exact token counts instead of estimating. Codex session cache bumps from v7 to v8 for a one-time re-parse. Only tool attribution changes for ordinary sessions, leaving their cost identical; sessions that logged an oversized token_count line are repriced from exact counts instead of an estimate. (#805)

View file

@ -5,7 +5,13 @@ import { homedir } from 'os'
import { join } from 'path'
import type { DateRange, ProjectSummary } from './types.js'
// Bumped to 16: Codex discovery is structural instead of originator-gated
// Bumped to 17: copilot CLI sessions were misclassified as VS Code transcripts
// (#944), so days finalized at v16 or earlier carry output-only copilot costs —
// the session.shutdown rollup's input/cache tokens were dropped. Raising
// MIN_SUPPORTED_VERSION forces the one-time re-derivation under the
// provenance-based classification; sourceless days carry forward as-is.
//
// v16: Codex discovery is structural instead of originator-gated
// (#873/#626), so rollouts written by third-party frontends driving
// `codex app-server` ("t3code_desktop", "JetBrains.IntelliJ IDEA", ...) now
// contribute usage that v15 rollups never contained. Those files were rejected
@ -67,8 +73,8 @@ import type { DateRange, ProjectSummary } from './types.js'
// that older binaries skipped. v8 added local-model savings to the daily
// rollup; the `savingsConfigHash` field is invalidated separately when the
// user changes their `localModelSavings` mapping.
export const DAILY_CACHE_VERSION = 16
const MIN_SUPPORTED_VERSION = 16
export const DAILY_CACHE_VERSION = 17
const MIN_SUPPORTED_VERSION = 17
// Version-suffixed so different binaries each own a distinct file and never
// clobber an incompatible schema. Bumping the version mints a fresh filename;
// adoptOlderDailyCaches then unions days out of every previous file (including

View file

@ -194,6 +194,9 @@ type SubagentSelectedData = {
agentName: string
agentDisplayName?: string
tools?: string[]
// Present on subagent.started/completed (CLI ≥ ~1.0.7x): the delegation
// tool call that launched the run, used to pair completed with started.
toolCallId?: string
}
// Per-model usage rollup the CLI writes into session.shutdown. inputTokens is
@ -217,6 +220,8 @@ type CopilotEvent =
| { type: 'user.message'; data: UserMessageData; timestamp?: string }
| { type: 'assistant.message'; data: AssistantMessageData; timestamp?: string }
| { type: 'subagent.selected'; data: SubagentSelectedData; timestamp?: string }
| { type: 'subagent.started'; data: SubagentSelectedData; timestamp?: string }
| { type: 'subagent.completed'; data: SubagentSelectedData; timestamp?: string }
| { type: 'session.shutdown'; data: SessionShutdownData; timestamp?: string }
type ChatJournalPathSegment = string | number
@ -693,50 +698,56 @@ function inferTranscriptModel(lines: string[]): string {
}
// ---------------------------------------------------------------------------
// JSONL parser (handles both regular session-state events and VS Code
// transcript format via session.start { producer: 'copilot-agent' })
// JSONL parser (handles both regular CLI session-state events and the VS Code
// transcript format — the same event vocabulary, but transcripts carry no
// token counts and no session.shutdown rollup)
// ---------------------------------------------------------------------------
/**
* `isTranscript` comes from discovery (where the file lives), never from
* content: the Copilot CLI writes the same session.start producer
* ('copilot-agent') that VS Code transcripts carry, so producer sniffing
* misread every CLI session as a transcript and dropped its session.shutdown
* input/cache rollup (#944).
*/
function createJsonlParser(
source: SessionSource,
seenKeys: Set<string>
seenKeys: Set<string>,
isTranscript: boolean
): SessionParser {
return {
async *parse(): AsyncGenerator<ParsedProviderCall> {
const content = await readSessionFile(source.path)
if (!content) return
const sessionId = basename(dirname(source.path))
// CLI session-state files live at <sessionId>/events.jsonl; transcripts
// at transcripts/<sessionId>.jsonl — keying the latter on the parent dir
// would collapse every transcript into one "transcripts" session (and
// one shared dedup namespace).
const sessionId = isTranscript
? basename(source.path, '.jsonl')
: basename(dirname(source.path))
const lines = content.split('\n').filter((l) => l.trim())
// Detect VS Code transcript format: the first session.start event has
// { producer: 'copilot-agent' } and no outputTokens in messages.
let isTranscript = false
let currentModel = ''
let pendingUserMessage = ''
// Track the active subagent for this session (from subagent.selected events).
// Resets when a new subagent is selected.
let currentSubagentType: string | undefined
// First pass: detect format and infer transcript model if needed.
for (const line of lines) {
try {
const ev = JSON.parse(line) as CopilotEvent
if (ev.type === 'session.start') {
const data = ev.data as SessionStartData & { producer?: string }
if (data.producer === 'copilot-agent') {
isTranscript = true
}
break
}
if (ev.type === 'session.model_change') break // regular format
} catch {
continue
}
}
// Subagent attribution. Older CLIs write subagent.selected — sticky
// until replaced, never cleared. CLI ≥ ~1.0.7x brackets each run with
// started/completed instead; runs can nest or overlap, so completed
// removes ONLY its own toolCallId's entry and the label falls back to
// the still-active run (or the sticky selected value) rather than
// wiping attribution for everything in flight.
let selectedSubagentType: string | undefined
const activeSubagents: Array<{ toolCallId: string; name: string }> = []
const currentSubagentType = (): string | undefined =>
activeSubagents[activeSubagents.length - 1]?.name ?? selectedSubagentType
if (isTranscript) {
// Tool-call-id prefix inference seeds the model; it must not gate the
// whole file, or a transcript carrying explicit model info
// (session.model_change / per-message model) but no tool calls would
// yield nothing. Messages that still end up modelless are skipped
// individually below.
currentModel = inferTranscriptModel(lines)
if (!currentModel) return // no toolCallIds to infer model from
}
// Shutdown rollups may lack their own timestamp; remember the last
@ -744,6 +755,15 @@ function createJsonlParser(
// timestamp, which the date-range filters silently drop.
let lastEventTimestamp = ''
// A resumed session appends one session.shutdown PER LEG, each carrying
// CUMULATIVE per-model totals. Emitting each rollup whole would need the
// cache to update a prior call in place — the durable merge is
// append-only by dedup key — so we emit per-leg DELTAS keyed by
// occurrence instead: re-parses of a growing file append only the new
// leg, and each leg lands on its own timestamp.
const prevShutdownUsage = new Map<string, ShutdownModelUsage>()
const shutdownCountByModel = new Map<string, number>()
for (const line of lines) {
let event: CopilotEvent
try {
@ -766,7 +786,34 @@ function createJsonlParser(
}
if (event.type === 'subagent.selected') {
currentSubagentType = (event.data as SubagentSelectedData).agentName
selectedSubagentType = (event.data as SubagentSelectedData).agentName
continue
}
if (event.type === 'subagent.started') {
const data = event.data as SubagentSelectedData
activeSubagents.push({ toolCallId: data.toolCallId ?? '', name: data.agentName })
continue
}
if (event.type === 'subagent.completed') {
const id = (event.data as SubagentSelectedData).toolCallId ?? ''
if (!id) {
// ID-less completion (transitional CLIs that key nothing, like
// subagent.selected): end the most recently started run; explicit
// no-op on an empty stack.
activeSubagents.pop()
continue
}
for (let i = activeSubagents.length - 1; i >= 0; i--) {
if (activeSubagents[i]!.toolCallId === id) {
activeSubagents.splice(i, 1)
break
}
}
// A non-empty id that matches nothing refers to a run we never saw
// start — leave the active runs alone rather than evicting an
// unrelated one.
continue
}
@ -783,11 +830,12 @@ function createJsonlParser(
// is gated to the CLI (non-transcript) format, leaving VS Code,
// JetBrains and OTel sources untouched.
//
// We emit one supplementary call per model carrying ONLY the
// input/cache tokens the per-turn events lack; output is excluded so
// the assistant.message output (and its cost) is not double-counted.
// Combined with the per-turn output cost, this yields the full,
// CLI-measured session cost.
// We emit one supplementary call per model PER SHUTDOWN LEG (resumed
// sessions write one cumulative rollup per leg; see the delta
// tracking above) carrying ONLY the input/cache tokens the per-turn
// events lack; output is excluded so the assistant.message output
// (and its cost) is not double-counted. Combined with the per-turn
// output cost, this yields the full, CLI-measured session cost.
if (isTranscript) continue
const shutdownData = event.data as SessionShutdownData
const modelMetrics = shutdownData.modelMetrics
@ -801,23 +849,49 @@ function createJsonlParser(
const usage = metrics['usage']
if (!isRecord(usage)) continue
const cacheReadTokens = numberOrZero(usage['cacheReadTokens'])
const cacheWriteTokens = numberOrZero(usage['cacheWriteTokens'])
const reasoningTokens = numberOrZero(usage['reasoningTokens'])
const cumulative: Required<ShutdownModelUsage> = {
inputTokens: numberOrZero(usage['inputTokens']),
outputTokens: numberOrZero(usage['outputTokens']),
cacheReadTokens: numberOrZero(usage['cacheReadTokens']),
cacheWriteTokens: numberOrZero(usage['cacheWriteTokens']),
reasoningTokens: numberOrZero(usage['reasoningTokens']),
}
const prevRaw = prevShutdownUsage.get(model)
prevShutdownUsage.set(model, cumulative)
const n = (shutdownCountByModel.get(model) ?? 0) + 1
shutdownCountByModel.set(model, n)
// A cumulative total BELOW the previous rollup means the CLI reset
// its counters (a fresh accounting epoch): delta from zero, else
// this leg's post-reset usage would be clamped away entirely.
// inputTokens is the monotonic sentinel — it is cache-inclusive,
// so any usage at all grows it.
const prev =
prevRaw && cumulative.inputTokens < numberOrZero(prevRaw.inputTokens)
? undefined
: prevRaw
// This leg's contribution: cumulative minus the previous rollup.
// The clamp guards any remaining non-monotonic field.
const delta = (k: keyof ShutdownModelUsage): number =>
Math.max(0, cumulative[k] - numberOrZero(prev?.[k]))
const cacheReadTokens = delta('cacheReadTokens')
const cacheWriteTokens = delta('cacheWriteTokens')
const reasoningTokens = delta('reasoningTokens')
// usage.inputTokens is cache-INCLUSIVE (input + cache_read +
// cache_write). calculateCost expects the uncached input alone with
// cache tokens billed separately, so subtract the cache components.
// Clamp at 0 in case a future schema reports input non-inclusively.
const inputTokens = Math.max(
0,
numberOrZero(usage['inputTokens']) - cacheReadTokens - cacheWriteTokens
delta('inputTokens') - cacheReadTokens - cacheWriteTokens
)
// Nothing this call would add over the per-turn events, so skip it
// to avoid an empty $0 row (output is intentionally excluded).
if (inputTokens === 0 && cacheReadTokens === 0 && cacheWriteTokens === 0) continue
if (inputTokens === 0 && cacheReadTokens === 0 && cacheWriteTokens === 0 && reasoningTokens === 0) continue
const dedupKey = `copilot:${sessionId}:shutdown:${model}`
const dedupKey = `copilot:${sessionId}:shutdown:${model}:${n}`
if (seenKeys.has(dedupKey)) continue
seenKeys.add(dedupKey)
@ -898,6 +972,7 @@ function createJsonlParser(
// Cost will be lower than actual API cost. This is the original
// behaviour — OTel data (below) replaces it when available.
const costUSD = calculateCost(currentModel, 0, outputTokens, 0, 0, 0)
const subagentType = currentSubagentType()
yield {
provider: 'copilot',
@ -914,7 +989,7 @@ function createJsonlParser(
tools,
bashCommands,
skills: skills.length > 0 ? skills : undefined,
subagentTypes: currentSubagentType ? [currentSubagentType] : undefined,
subagentTypes: subagentType ? [subagentType] : undefined,
timestamp: event.timestamp ?? '',
speed: 'standard' as const,
deduplicationKey: dedupKey,
@ -1837,6 +1912,12 @@ interface JsonlSessionSource extends SessionSource {
sourceType: 'jsonl'
}
// A VS Code workspaceStorage transcript. Distinct from 'jsonl' (CLI
// session-state) so classification rides provenance, not file contents (#944).
interface TranscriptSessionSource extends SessionSource {
sourceType: 'transcript'
}
interface ChatSessionSource extends SessionSource {
sourceType: 'chatsession'
}
@ -1874,6 +1955,10 @@ function isJetBrainsSource(source: SessionSource): source is JetBrainsSessionSou
return (source as JetBrainsSessionSource).sourceType === 'jetbrains'
}
function isTranscriptSource(source: SessionSource): source is TranscriptSessionSource {
return (source as TranscriptSessionSource).sourceType === 'transcript'
}
// ---------------------------------------------------------------------------
// Session discovery: JSONL (original)
// ---------------------------------------------------------------------------
@ -2242,8 +2327,8 @@ async function discoverEmptyWindowChatSessions(
*/
async function discoverTranscriptSessions(
workspaceStorageDirs: string[]
): Promise<JsonlSessionSource[]> {
const sources: JsonlSessionSource[] = []
): Promise<TranscriptSessionSource[]> {
const sources: TranscriptSessionSource[] = []
for (const wsDir of workspaceStorageDirs) {
let hashDirs: string[]
@ -2275,7 +2360,7 @@ async function discoverTranscriptSessions(
path: join(transcriptsDir, file),
project,
provider: 'copilot',
sourceType: 'jsonl',
sourceType: 'transcript',
})
}
}
@ -2418,7 +2503,7 @@ export function createCopilotProvider(
if (isJetBrainsSource(source)) {
return createJetBrainsParser(source, seenKeys)
}
return createJsonlParser(source, seenKeys)
return createJsonlParser(source, seenKeys, isTranscriptSource(source))
},
}
}

View file

@ -224,7 +224,10 @@ export const PROVIDER_PARSE_VERSIONS: Record<string, string> = {
codex: 'mcp-attribution-v5-est-cost-active-timing-mcp-wait-rich-capture-v1-cross-provider-pr-v1',
cursor: 'composer-anchored-crediting-v1-est-cost',
'cursor-agent': 'workspaceless-transcript-v1',
copilot: 'cli-shutdown-cost-v1-skills',
// source-provenance-v1 (#944): CLI sessions were misread as VS Code
// transcripts (both carry producer 'copilot-agent'), skipping the shutdown
// input/cache rollup; this bump re-parses them so the missing tokens land.
copilot: 'cli-shutdown-cost-v1-skills-source-provenance-v1',
grok: 'estimated-cost-v1',
hermes: 'reasoning-output-accounting-v1-est-cost',
'lingtai-tui': 'token-ledger-registry-activity-v3',

View file

@ -605,3 +605,77 @@ describe('(h) provider filter excludes claude from the orphan pass', () => {
expect(totalCost(after)).toBeCloseTo(costBefore, 10)
})
})
// ═══════════════════════════════════════════════════════════════════════════
// (f) Growing resumed CLI session: durable merge appends only the new leg
// ═══════════════════════════════════════════════════════════════════════════
// Resumed Copilot CLI sessions append one CUMULATIVE session.shutdown per leg
// (#944). The parser emits per-leg deltas keyed by occurrence; this exercises
// the PRODUCTION merge path — the durable union-by-dedup-key merge against the
// on-disk cache when the file grows between parses — which the unit tests
// (which pre-seed seenKeys) cannot reach.
describe('(f) growing resumed CLI session durable merge', () => {
it('totals equal the final cumulative rollup after the file grows a leg', async () => {
const sessionStateDir = join(tmpHome, 'session-state')
await mkdir(sessionStateDir, { recursive: true })
vi.stubEnv('CODEBURN_COPILOT_SESSION_STATE_DIR', sessionStateDir)
vi.stubEnv('CODEBURN_COPILOT_DISABLE_OTEL', '1')
vi.stubEnv('CODEBURN_COPILOT_WS_STORAGE_DIR', join(tmpHome, 'no-ws'))
vi.stubEnv('CODEBURN_COPILOT_GLOBAL_STORAGE_DIR', join(tmpHome, 'no-global'))
vi.stubEnv('CODEBURN_COPILOT_JETBRAINS_DIR', join(tmpHome, 'no-jb'))
const base = Date.now() - 5 * 24 * 60 * 60 * 1000
const at = (offsetSec: number): string => new Date(base + offsetSec * 1000).toISOString()
const dir = join(sessionStateDir, 'sess-grow')
await mkdir(dir, { recursive: true })
await writeFile(join(dir, 'workspace.yaml'), 'id: sess-grow\ncwd: /home/user/testproj\n')
const eventsPath = join(dir, 'events.jsonl')
// Cumulative rollups from a real resumed CLI 1.0.78 session.
const shutdown = (ts: string, inputTokens: number, cacheReadTokens: number, cacheWriteTokens: number, outputTokens: number) =>
JSON.stringify({
type: 'session.shutdown',
timestamp: ts,
data: {
shutdownType: 'routine',
modelMetrics: {
'claude-sonnet-4-5': {
requests: { count: 1, cost: 1 },
usage: { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens: 0 },
},
},
},
})
const leg1 = [
JSON.stringify({ type: 'session.model_change', timestamp: at(0), data: { newModel: 'claude-sonnet-4-5' } }),
JSON.stringify({ type: 'assistant.message', timestamp: at(10), data: { messageId: 'msg-1', outputTokens: 17, toolRequests: [] } }),
shutdown(at(20), 24672, 0, 24670, 17),
]
await writeFile(eventsPath, leg1.join('\n') + '\n')
const sumUsage = (projects: Awaited<ReturnType<typeof parseAllSessions>>) => {
const calls = projects.flatMap(p => p.sessions).flatMap(s => s.turns).flatMap(t => t.assistantCalls)
return {
input: calls.reduce((s, c) => s + c.usage.inputTokens, 0),
cacheRead: calls.reduce((s, c) => s + c.usage.cacheReadInputTokens, 0),
cacheWrite: calls.reduce((s, c) => s + c.usage.cacheCreationInputTokens, 0),
}
}
const first = sumUsage(await parseAllSessions(undefined, 'copilot'))
expect(first).toEqual({ input: 2, cacheRead: 0, cacheWrite: 24670 })
// The session resumes: leg 2 appends per-turn events plus a CUMULATIVE
// rollup. The cached leg-1 delta must be kept once and only the leg-2
// delta appended — totals equal the final cumulative rollup exactly.
clearSessionCache()
await writeFile(eventsPath, [
...leg1,
JSON.stringify({ type: 'assistant.message', timestamp: at(100), data: { messageId: 'msg-2', outputTokens: 132, toolRequests: [] } }),
shutdown(at(120), 74463, 49489, 24968, 149),
].join('\n') + '\n')
const second = sumUsage(await parseAllSessions(undefined, 'copilot'))
expect(second).toEqual({ input: 74463 - 49489 - 24968, cacheRead: 49489, cacheWrite: 24968 })
})
})

View file

@ -124,6 +124,18 @@ async function collectCalls(source: { path: string; project: string; provider: s
return calls
}
// Write a transcript inside the test's tmpDir sandbox, but at the production
// directory shape — {ws}/{hash}/GitHub.copilot-chat/transcripts/<id>.jsonl —
// because sessionId derivation reads the path structure (file basename for
// transcripts). Never touches the real VS Code storage.
async function createTranscriptFile(sessionId: string, lines: string[]) {
const transcriptsDir = join(tmpDir, 'ws', 'hash1', 'GitHub.copilot-chat', 'transcripts')
await mkdir(transcriptsDir, { recursive: true })
const path = join(transcriptsDir, `${sessionId}.jsonl`)
await writeFile(path, lines.join('\n') + '\n')
return path
}
describe('copilot provider - JSONL parsing', () => {
beforeEach(async () => {
tmpDir = await mkdtemp(join(tmpdir(), 'copilot-test-'))
@ -335,8 +347,122 @@ describe('copilot provider - JSONL parsing', () => {
expect(calls[0]!.model).toBe('gpt-4.1')
})
it('attributes turns between subagent.started and subagent.completed to the subagent', async () => {
// CLI ≥ ~1.0.7x writes subagent.started/completed (not subagent.selected);
// event shapes from a real delegating 1.0.78 session. The label must cover
// the subagent's turns and clear afterwards, not bleed onto the parent's.
const eventsPath = await createSessionDir('sess-subagent-cli', [
modelChange('claude-sonnet-5'),
userMessage('delegate a search'),
JSON.stringify({
type: 'subagent.started',
timestamp: '2026-08-07T10:00:11Z',
data: { toolCallId: 'toolu_01SZnHjC', agentName: 'explore', agentDisplayName: 'Explore Agent' },
}),
JSON.stringify({
type: 'assistant.message',
timestamp: '2026-08-07T10:00:14Z',
data: { messageId: 'msg-sub', model: 'claude-haiku-4.5', outputTokens: 197, toolRequests: [] },
}),
JSON.stringify({
type: 'subagent.completed',
timestamp: '2026-08-07T10:00:19Z',
data: { toolCallId: 'toolu_01SZnHjC', agentName: 'explore', model: 'claude-haiku-4.5', totalTokens: 26435 },
}),
JSON.stringify({
type: 'assistant.message',
timestamp: '2026-08-07T10:00:22Z',
data: { messageId: 'msg-parent', model: 'claude-sonnet-5', outputTokens: 51, toolRequests: [] },
}),
])
const calls = await collectCalls({ path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'jsonl' })
const sub = calls.find(c => c.deduplicationKey.endsWith(':msg-sub'))!
expect(sub.subagentTypes).toEqual(['explore'])
expect(sub.model).toBe('claude-haiku-4.5')
const parent = calls.find(c => c.deduplicationKey.endsWith(':msg-parent'))!
expect(parent.subagentTypes).toBeUndefined()
})
it('completing a nested subagent restores the outer label, matched by toolCallId', async () => {
const started = (id: string, name: string) =>
JSON.stringify({ type: 'subagent.started', timestamp: '2026-08-07T10:00:11Z', data: { toolCallId: id, agentName: name } })
const completed = (id: string) =>
JSON.stringify({ type: 'subagent.completed', timestamp: '2026-08-07T10:00:19Z', data: { toolCallId: id, agentName: 'x' } })
const msg = (messageId: string, outputTokens = 10) =>
JSON.stringify({ type: 'assistant.message', timestamp: '2026-08-07T10:00:14Z', data: { messageId, model: 'claude-sonnet-5', outputTokens, toolRequests: [] } })
const eventsPath = await createSessionDir('sess-subagent-nested', [
modelChange('claude-sonnet-5'),
started('call-A', 'explore'),
started('call-B', 'plan'),
msg('msg-inner'), // while B runs → 'plan'
completed('call-B'),
msg('msg-outer'), // B done, A still active → 'explore', NOT unlabeled
completed('call-A'),
msg('msg-after'), // all done → no label
])
const calls = await collectCalls({ path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'jsonl' })
const byId = (id: string) => calls.find(c => c.deduplicationKey.endsWith(`:${id}`))!
expect(byId('msg-inner').subagentTypes).toEqual(['plan'])
expect(byId('msg-outer').subagentTypes).toEqual(['explore'])
expect(byId('msg-after').subagentTypes).toBeUndefined()
})
it('ignores a completed event whose non-empty toolCallId matches no active run', async () => {
// A completion for a run we never saw start must not evict an unrelated
// active run; only a genuinely ID-less completion may pop the stack.
const eventsPath = await createSessionDir('sess-subagent-unmatched', [
modelChange('claude-sonnet-5'),
JSON.stringify({
type: 'subagent.started',
timestamp: '2026-08-07T10:00:11Z',
data: { toolCallId: 'call-A', agentName: 'explore' },
}),
JSON.stringify({
type: 'subagent.completed',
timestamp: '2026-08-07T10:00:12Z',
data: { toolCallId: 'call-unknown', agentName: 'phantom' },
}),
JSON.stringify({
type: 'assistant.message',
timestamp: '2026-08-07T10:00:14Z',
data: { messageId: 'msg-1', model: 'claude-sonnet-5', outputTokens: 10, toolRequests: [] },
}),
JSON.stringify({
type: 'subagent.completed',
timestamp: '2026-08-07T10:00:15Z',
data: { agentName: 'legacy-no-id' },
}),
JSON.stringify({
type: 'assistant.message',
timestamp: '2026-08-07T10:00:16Z',
data: { messageId: 'msg-2', model: 'claude-sonnet-5', outputTokens: 12, toolRequests: [] },
}),
])
const calls = await collectCalls({ path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'jsonl' })
// The unmatched completion left 'explore' active…
expect(calls.find(c => c.deduplicationKey.endsWith(':msg-1'))!.subagentTypes).toEqual(['explore'])
// …and the ID-less completion (legacy shape) ended it.
expect(calls.find(c => c.deduplicationKey.endsWith(':msg-2'))!.subagentTypes).toBeUndefined()
})
it('keeps subagent.selected sticky when no completed event ever arrives', async () => {
// Older CLIs only write subagent.selected; nothing clears it.
const eventsPath = await createSessionDir('sess-subagent-selected', [
modelChange('claude-sonnet-5'),
JSON.stringify({ type: 'subagent.selected', data: { agentName: 'refactor' } }),
assistantMessage({ messageId: 'msg-1', outputTokens: 25 }),
assistantMessage({ messageId: 'msg-2', outputTokens: 30, timestamp: '2026-04-15T10:01:00Z' }),
])
const calls = await collectCalls({ path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'jsonl' })
expect(calls.map(c => c.subagentTypes)).toEqual([['refactor'], ['refactor']])
})
it('infers OpenAI auto bucket for transcript toolCallId prefix call_', async () => {
const eventsPath = await createSessionDir('sess-tr-call', [
const eventsPath = await createTranscriptFile('sess-tr-call', [
transcriptSessionStart('sess-tr-call'),
transcriptUserMessage('check model inference'),
transcriptAssistantMessage({
@ -346,16 +472,20 @@ describe('copilot provider - JSONL parsing', () => {
}),
])
const source = { path: eventsPath, project: 'test', provider: 'copilot' }
const source = { path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'transcript' }
const calls: ParsedProviderCall[] = []
for await (const call of copilot.createSessionParser(source, new Set()).parse()) calls.push(call)
expect(calls).toHaveLength(1)
expect(calls[0]!.model).toBe('copilot-openai-auto')
// Each transcript is its own session, keyed by file basename — NOT the
// shared parent dir name 'transcripts', which would collapse every
// transcript into one session and one dedup namespace.
expect(calls[0]!.sessionId).toBe('sess-tr-call')
})
it('infers Anthropic auto bucket for transcript toolCallId prefixes tooluse_/toolu_vrtx_', async () => {
const eventsPath = await createSessionDir('sess-tr-claude', [
const eventsPath = await createTranscriptFile('sess-tr-claude', [
transcriptSessionStart('sess-tr-claude'),
transcriptUserMessage('check model inference'),
transcriptAssistantMessage({
@ -365,7 +495,7 @@ describe('copilot provider - JSONL parsing', () => {
}),
])
const source = { path: eventsPath, project: 'test', provider: 'copilot' }
const source = { path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'transcript' }
const calls: ParsedProviderCall[] = []
for await (const call of copilot.createSessionParser(source, new Set()).parse()) calls.push(call)
@ -374,7 +504,7 @@ describe('copilot provider - JSONL parsing', () => {
})
it('chooses the dominant inferred transcript model when prefixes are mixed', async () => {
const eventsPath = await createSessionDir('sess-tr-mixed', [
const eventsPath = await createTranscriptFile('sess-tr-mixed', [
transcriptSessionStart('sess-tr-mixed'),
transcriptUserMessage('mixed'),
transcriptAssistantMessage({
@ -394,7 +524,7 @@ describe('copilot provider - JSONL parsing', () => {
}),
])
const source = { path: eventsPath, project: 'test', provider: 'copilot' }
const source = { path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'transcript' }
const calls: ParsedProviderCall[] = []
for await (const call of copilot.createSessionParser(source, new Set()).parse()) calls.push(call)
@ -402,8 +532,35 @@ describe('copilot provider - JSONL parsing', () => {
expect(calls.every(c => c.model === 'copilot-openai-auto')).toBe(true)
})
it('parses a producerless transcript with explicit model info and no tool calls', async () => {
// Prefix inference has nothing to work with here; the explicit
// session.model_change must still establish the model, and the shutdown
// rollup must stay ignored — provenance, not the producer field, gates it.
const eventsPath = await createTranscriptFile('sess-tr-explicit', [
JSON.stringify({ type: 'session.start', data: { sessionId: 'sess-tr-explicit' } }),
modelChange('gpt-4.1'),
transcriptUserMessage('hi'),
JSON.stringify({
type: 'assistant.message',
timestamp: '2026-04-15T10:00:15Z',
data: { messageId: 'msg-1', outputTokens: 80, toolRequests: [] },
}),
shutdownEvent({
modelMetrics: {
'gpt-4.1': { inputTokens: 1000, outputTokens: 80, cacheReadTokens: 500, cacheWriteTokens: 200 },
},
}),
])
const calls = await collectCalls({ path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'transcript' })
expect(calls).toHaveLength(1)
expect(calls[0]!.model).toBe('gpt-4.1')
expect(calls[0]!.outputTokens).toBe(80)
expect(calls.every(c => !c.deduplicationKey.includes(':shutdown:'))).toBe(true)
})
it('normalizes Copilot MCP tool names from VS Code transcripts', async () => {
const eventsPath = await createSessionDir('sess-tr-mcp-tools', [
const eventsPath = await createTranscriptFile('sess-tr-mcp-tools', [
transcriptSessionStart('sess-tr-mcp-tools'),
transcriptUserMessage('use GitHub MCP'),
transcriptAssistantMessage({
@ -414,7 +571,7 @@ describe('copilot provider - JSONL parsing', () => {
}),
])
const source = { path: eventsPath, project: 'test', provider: 'copilot' }
const source = { path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'transcript' }
const calls: ParsedProviderCall[] = []
for await (const call of copilot.createSessionParser(source, new Set()).parse()) calls.push(call)
@ -458,7 +615,7 @@ describe('copilot provider - session.shutdown token/cost rollup', () => {
// One per-turn assistant.message call + one supplementary shutdown call.
expect(calls).toHaveLength(2)
const shutdown = calls.find(c => c.deduplicationKey === 'copilot:sess-shutdown:shutdown:claude-sonnet-4-5')
const shutdown = calls.find(c => c.deduplicationKey === 'copilot:sess-shutdown:shutdown:claude-sonnet-4-5:1')
expect(shutdown).toBeDefined()
expect(shutdown!.model).toBe('claude-sonnet-4-5')
expect(shutdown!.inputTokens).toBe(4) // 71282 - 35495 - 35783
@ -546,6 +703,86 @@ describe('copilot provider - session.shutdown token/cost rollup', () => {
expect(gpt.costUSD).toBeCloseTo(calculateCost('gpt-5', 50, 0, 0, 5000, 0), 12)
})
it('emits per-leg deltas for a resumed session with cumulative shutdown rollups', async () => {
// Numbers from a real resumed CLI 1.0.78 session (3 legs via --resume):
// each leg appends a session.shutdown whose modelMetrics are CUMULATIVE.
// Emitting deltas keyed by occurrence keeps a growing file append-only
// under the durable union-by-key cache merge — re-parsing after each
// resume adds only the new leg, never double-counting earlier ones.
const legs = [
{ inputTokens: 24672, outputTokens: 17, cacheReadTokens: 0, cacheWriteTokens: 24670 },
{ inputTokens: 74463, outputTokens: 149, cacheReadTokens: 49489, cacheWriteTokens: 24968 },
{ inputTokens: 124783, outputTokens: 243, cacheReadTokens: 99569, cacheWriteTokens: 25204 },
]
const lines = [modelChange('claude-sonnet-5'), assistantMessage({ messageId: 'msg-1', outputTokens: 17 })]
for (const [i, leg] of legs.entries()) {
lines.push(shutdownEvent({ modelMetrics: { 'claude-sonnet-5': leg }, timestamp: `2026-08-0${i + 1}T10:00:00Z` }))
}
const eventsPath = await createSessionDir('sess-resumed', lines)
const calls = await collectCalls({ path: eventsPath, project: 'myproject', provider: 'copilot', sourceType: 'jsonl' })
const shutdowns = calls.filter(c => c.deduplicationKey.includes(':shutdown:'))
expect(shutdowns.map(c => c.deduplicationKey)).toEqual([
'copilot:sess-resumed:shutdown:claude-sonnet-5:1',
'copilot:sess-resumed:shutdown:claude-sonnet-5:2',
'copilot:sess-resumed:shutdown:claude-sonnet-5:3',
])
// Each leg lands on its own shutdown timestamp (a resumed session can
// span days; whole-rollup emission would collapse them onto one).
expect(shutdowns.map(c => c.timestamp)).toEqual([
'2026-08-01T10:00:00Z', '2026-08-02T10:00:00Z', '2026-08-03T10:00:00Z',
])
// Per-leg deltas sum exactly to the final cumulative rollup.
const sum = (k: 'inputTokens' | 'cacheReadInputTokens' | 'cacheCreationInputTokens') =>
shutdowns.reduce((a, c) => a + c[k], 0)
expect(sum('cacheReadInputTokens')).toBe(99569)
expect(sum('cacheCreationInputTokens')).toBe(25204)
expect(sum('inputTokens')).toBe(124783 - 99569 - 25204)
// A later re-parse of the grown file (prior legs already cached) emits
// only what the seen-key set lacks.
const seen = new Set(calls.map(c => c.deduplicationKey))
const again = await collectCalls({ path: eventsPath, project: 'myproject', provider: 'copilot', sourceType: 'jsonl' }, seen)
expect(again).toHaveLength(0)
})
it('starts a fresh delta baseline when a cumulative rollup goes backwards (counter reset)', async () => {
// Hypothetical but cheap to guard: if the CLI ever resets its counters
// mid-session, the post-reset epoch must be billed from zero — a stale
// high-water baseline would clamp it away (and the reset leg's real usage
// with it).
const eventsPath = await createSessionDir('sess-reset', [
modelChange('claude-sonnet-5'),
assistantMessage({ messageId: 'msg-1', outputTokens: 10 }),
shutdownEvent({
modelMetrics: { 'claude-sonnet-5': { inputTokens: 10000, outputTokens: 10, cacheReadTokens: 0, cacheWriteTokens: 5000 } },
timestamp: '2026-08-01T10:00:00Z',
}),
// Reset: cumulative drops below the previous rollup → new epoch.
shutdownEvent({
modelMetrics: { 'claude-sonnet-5': { inputTokens: 2000, outputTokens: 5, cacheReadTokens: 0, cacheWriteTokens: 1000 } },
timestamp: '2026-08-02T10:00:00Z',
}),
shutdownEvent({
modelMetrics: { 'claude-sonnet-5': { inputTokens: 5000, outputTokens: 8, cacheReadTokens: 2000, cacheWriteTokens: 1500 } },
timestamp: '2026-08-03T10:00:00Z',
}),
])
const calls = await collectCalls({ path: eventsPath, project: 'myproject', provider: 'copilot', sourceType: 'jsonl' })
const shutdowns = calls.filter(c => c.deduplicationKey.includes(':shutdown:'))
expect(shutdowns).toHaveLength(3)
// Leg 1: epoch-1 usage in full.
expect(shutdowns[0]!.inputTokens).toBe(5000) // 10000 0 5000
expect(shutdowns[0]!.cacheCreationInputTokens).toBe(5000)
// Leg 2 (reset): billed from zero, not clamped away against the old baseline.
expect(shutdowns[1]!.inputTokens).toBe(1000) // 2000 0 1000
expect(shutdowns[1]!.cacheCreationInputTokens).toBe(1000)
// Leg 3: normal delta within the new epoch.
expect(shutdowns[2]!.inputTokens).toBe(500) // (50002000) 2000 500
expect(shutdowns[2]!.cacheReadInputTokens).toBe(2000)
expect(shutdowns[2]!.cacheCreationInputTokens).toBe(500)
})
it('keeps shutdown dedup keys stable across re-parses', async () => {
const eventsPath = await createSessionDir('sess-reparse', [
modelChange('claude-sonnet-4-5'),
@ -594,7 +831,7 @@ describe('copilot provider - session.shutdown token/cost rollup', () => {
})
it('ignores session.shutdown for VS Code transcript sessions', async () => {
const eventsPath = await createSessionDir('sess-tr-shutdown', [
const eventsPath = await createTranscriptFile('sess-tr-shutdown', [
transcriptSessionStart('sess-tr-shutdown'),
transcriptUserMessage('hi'),
transcriptAssistantMessage({ messageId: 'msg-1', content: 'done', toolCallIds: ['call_abc'] }),
@ -604,7 +841,7 @@ describe('copilot provider - session.shutdown token/cost rollup', () => {
},
}),
])
const source = { path: eventsPath, project: 'test', provider: 'copilot' }
const source = { path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'transcript' }
const calls = await collectCalls(source)
// Only the transcript assistant call; the shutdown rollup is CLI-only.
@ -612,6 +849,165 @@ describe('copilot provider - session.shutdown token/cost rollup', () => {
expect(calls.every(c => !c.deduplicationKey.includes(':shutdown:'))).toBe(true)
expect(calls[0]!.model).toBe('copilot-openai-auto')
})
// Regression test for #944: events are redacted copies of a real Copilot CLI
// 1.0.78 session. The CLI writes the same producer ('copilot-agent') as VS
// Code transcripts, so content sniffing skipped this session's shutdown
// rollup — reporting 100 of its 49,573 tokens and zero input/cache.
it('parses a CLI session whose session.start carries producer copilot-agent (issue #944)', async () => {
const eventsPath = await createSessionDir('sess-cli-producer', [
JSON.stringify({
type: 'session.start',
timestamp: '2026-08-07T17:56:35.573Z',
data: {
sessionId: 'sess-cli-producer',
version: 1,
producer: 'copilot-agent',
copilotVersion: '1.0.78',
startTime: '2026-08-07T17:56:35.554Z',
context: { cwd: '/home/user/myproject' },
},
}),
JSON.stringify({
type: 'session.model_change',
timestamp: '2026-08-07T17:56:36.725Z',
data: { newModel: 'claude-sonnet-5', reasoningEffort: null },
}),
JSON.stringify({
type: 'user.message',
timestamp: '2026-08-07T17:56:36.732Z',
data: { content: 'Run echo and summarize the output.' },
}),
JSON.stringify({
type: 'assistant.message',
timestamp: '2026-08-07T17:56:38.763Z',
data: {
messageId: 'a982a391-9ee3-4fbd-89a9-26d5af78c890',
model: 'claude-sonnet-5',
content: '',
toolRequests: [{
toolCallId: 'toolu_017eL3f5aeGiLoALignYMZEN',
name: 'bash',
arguments: { command: 'echo codeburn-repro-944', description: 'Echo test string' },
type: 'function',
}],
turnId: '0',
outputTokens: 81,
},
}),
JSON.stringify({
type: 'assistant.message',
timestamp: '2026-08-07T17:56:40.417Z',
data: {
messageId: '8758ea51-797f-4285-972c-495911e2839f',
model: 'claude-sonnet-5',
content: 'The command printed the string "codeburn-repro-944".',
toolRequests: [],
turnId: '1',
outputTokens: 19,
},
}),
JSON.stringify({
type: 'session.shutdown',
timestamp: '2026-08-07T17:56:40.591Z',
data: {
shutdownType: 'routine',
sessionStartTime: 1786125395554,
modelMetrics: {
'claude-sonnet-5': {
requests: { count: 2, cost: 1 },
usage: { inputTokens: 49473, outputTokens: 100, cacheReadTokens: 24678, cacheWriteTokens: 24791, reasoningTokens: 0 },
},
},
},
}),
])
// Discovery tags session-state files 'jsonl'; provenance, not the shared
// producer value, must classify this as a CLI session.
const calls = await collectCalls({ path: eventsPath, project: 'myproject', provider: 'copilot', sourceType: 'jsonl' })
// Two per-turn output calls with the REAL model — not the
// 'copilot-anthropic-auto' bucket transcript inference would pick from the
// toolu_ toolCallId prefix.
const perTurn = calls.filter(c => !c.deduplicationKey.includes(':shutdown:'))
expect(perTurn.map(c => c.outputTokens)).toEqual([81, 19])
expect(perTurn.every(c => c.model === 'claude-sonnet-5')).toBe(true)
// The shutdown rollup lands: the tokens the misclassification dropped.
const shutdown = calls.find(c => c.deduplicationKey === 'copilot:sess-cli-producer:shutdown:claude-sonnet-5:1')
expect(shutdown).toBeDefined()
expect(shutdown!.inputTokens).toBe(4) // 49473 24678 24791 (cache-inclusive)
expect(shutdown!.cacheReadInputTokens).toBe(24678)
expect(shutdown!.cacheCreationInputTokens).toBe(24791)
expect(shutdown!.outputTokens).toBe(0) // owned by the per-turn events
expect(shutdown!.costIsEstimated).toBe(false)
expect(shutdown!.costUSD).toBeCloseTo(calculateCost('claude-sonnet-5', 4, 0, 24791, 24678, 0), 12)
expect(shutdown!.costUSD).toBeGreaterThan(0)
})
it('treats a bare (untagged) source as CLI format, not transcript', async () => {
// Producer sniffing must not resurface for sources without a sourceType tag
// (the pre-tagging shape): same events, same result as the tagged parse.
const eventsPath = await createSessionDir('sess-cli-untagged', [
JSON.stringify({
type: 'session.start',
timestamp: '2026-08-07T17:56:35.573Z',
data: { sessionId: 'sess-cli-untagged', producer: 'copilot-agent', copilotVersion: '1.0.78' },
}),
modelChange('claude-sonnet-5'),
userMessage('hello'),
assistantMessage({ messageId: 'msg-1', outputTokens: 42 }),
shutdownEvent({
modelMetrics: {
'claude-sonnet-5': { inputTokens: 1000, outputTokens: 42, cacheReadTokens: 600, cacheWriteTokens: 300 },
},
}),
])
const calls = await collectCalls({ path: eventsPath, project: 'myproject', provider: 'copilot' })
expect(calls.some(c => c.deduplicationKey.includes(':shutdown:'))).toBe(true)
expect(calls.find(c => c.deduplicationKey.includes(':shutdown:'))!.cacheReadInputTokens).toBe(600)
})
it('wires discovery through parsing: a discovered CLI session keeps its shutdown rollup', async () => {
// The full #944 pipeline: discoverSessions must tag the session-state file
// so that the parser it hands off to keeps the shutdown tokens.
await createSessionDir('sess-wire', [
JSON.stringify({
type: 'session.start',
timestamp: '2026-08-07T17:56:35.573Z',
data: { sessionId: 'sess-wire', producer: 'copilot-agent', copilotVersion: '1.0.78' },
}),
modelChange('claude-sonnet-5'),
userMessage('hello'),
assistantMessage({ messageId: 'msg-1', outputTokens: 42 }),
shutdownEvent({
modelMetrics: {
'claude-sonnet-5': { inputTokens: 5000, outputTokens: 42, cacheReadTokens: 3000, cacheWriteTokens: 1500 },
},
}),
])
// Keep discovery hermetic: a real agent-traces.db on the host must not leak in.
vi.stubEnv('CODEBURN_COPILOT_DISABLE_OTEL', '1')
try {
const provider = createCopilotProvider(tmpDir, '/nonexistent/vscode', '/nonexistent/global', '/nonexistent/jetbrains')
const sessions = await provider.discoverSessions()
expect(sessions).toHaveLength(1)
const calls: ParsedProviderCall[] = []
for await (const call of provider.createSessionParser(sessions[0]!, new Set()).parse()) calls.push(call)
const shutdown = calls.find(c => c.deduplicationKey === 'copilot:sess-wire:shutdown:claude-sonnet-5:1')
expect(shutdown).toBeDefined()
expect(shutdown!.inputTokens).toBe(500) // 5000 3000 1500
expect(shutdown!.cacheReadInputTokens).toBe(3000)
expect(shutdown!.cacheCreationInputTokens).toBe(1500)
} finally {
vi.unstubAllEnvs()
}
})
})
describe('copilot provider - chatSessions parsing', () => {
@ -811,6 +1207,9 @@ describe('copilot provider - discoverSessions', () => {
expect(sessions).toHaveLength(2)
expect(sessions.every(s => s.provider === 'copilot')).toBe(true)
expect(sessions.every(s => s.path.endsWith('events.jsonl'))).toBe(true)
// Session-state files are tagged as CLI sources — the tag (not the file's
// producer value) decides transcript vs CLI parsing (#944).
expect(sessions.every(s => (s as { sourceType?: string }).sourceType === 'jsonl')).toBe(true)
})
it('reads project name from workspace.yaml cwd', async () => {
@ -864,6 +1263,7 @@ describe('copilot provider - discoverSessions', () => {
expect(sessions).toHaveLength(1)
expect(sessions[0]!.project).toBe('myapp')
expect(sessions[0]!.path).toContain('session-1.jsonl')
expect((sessions[0] as { sourceType?: string }).sourceType).toBe('transcript')
})
it('includes VSCodium workspaceStorage paths on all supported platforms', () => {