mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-08-20 14:04:25 +00:00
fix(copilot): classify CLI sessions by source provenance, not producer (#945)
Fixes #944.
This commit is contained in:
parent
74e69ba2fd
commit
3536a1d3ac
6 changed files with 630 additions and 60 deletions
|
|
@ -12,6 +12,8 @@
|
|||
- `codeburn sync push --attribution` (opt-in): sends git attribution spans — the session→commit correlation from `codeburn yield` (`codeburn.session.attribution` and `codeburn.commit` span types with normalized repo remote, commit SHAs, merged/reverted state, and PR links). Nothing new is sent without the flag; local-only repos and Windows filesystem paths are never emitted as repo identities, and sessions whose project path no longer resolves never inherit the push-time working directory's repo. See docs/sync/README.md "Git attribution".
|
||||
|
||||
### Fixed (CLI)
|
||||
- **Copilot CLI sessions report their input and cache tokens.** The Copilot CLI writes the same `producer: 'copilot-agent'` in its `session.start` events that VS Code transcripts carry, so content-based detection classified every CLI session as a transcript and skipped its `session.shutdown` rollup — the only place the CLI records input, cache-read and cache-write tokens — leaving cache hit rate at 0.0% and dramatically underreporting cost. Whether a file is a transcript is now decided by where discovery found it, never by its contents. Resumed sessions, whose legs each append a cumulative rollup, are billed as per-leg deltas so a growing session never double-counts or goes stale; the GitHub Copilot desktop app writes the same session store, so its usage is covered by the same fix. The copilot session cache takes a parse-version bump and the daily cache bumps from v16 to v17 for the one-time re-parse that heals already-recorded days whose logs still exist. (#944)
|
||||
- **Copilot CLI subagent runs are attributed to their agent.** Newer CLIs announce delegation with `subagent.started`/`subagent.completed` rather than `subagent.selected`, so delegated turns lost their agent label; the label now also clears when the subagent completes instead of bleeding onto the parent's later turns. Rides the #944 re-parse, so already-cached sessions gain the attribution. (#944)
|
||||
- **`--project` / `--exclude` now apply to the headline totals, not just the detail panels.** The durable headline unions the carry-forward daily cache with today's live parse, and the cached days were sliced to the requested provider but never to the requested project — so the Overview panel counted excluded projects while By Project / By Activity / By Model (built from the name-filtered parse) left them out, and the two could not be reconciled. Cost, calls, sessions and savings are now sliced out of the per-project day stats the cache has carried since v15. Tokens, models and categories have no per-project split in the cache, so under a project filter they come from the (project-filtered) live parse instead; cached days — or provider slices — carried from before v15 have no project split at all, so they cannot be attributed to a filtered project, and the terminal overview now states how much was set aside rather than folding it into the total. (#864)
|
||||
- **Codex parser corrections**: fork-replay no longer double-counts `patch_apply_end` and `mcp_tool_call_end`; `exec` is normalized to Bash; `custom_tool_call` events are handled; token_count lines larger than 32 KiB now parse exact token counts instead of estimating. Codex session cache bumps from v7 to v8 for a one-time re-parse. Only tool attribution changes for ordinary sessions, leaving their cost identical; sessions that logged an oversized token_count line are repriced from exact counts instead of an estimate. (#805)
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,13 @@ import { homedir } from 'os'
|
|||
import { join } from 'path'
|
||||
import type { DateRange, ProjectSummary } from './types.js'
|
||||
|
||||
// Bumped to 16: Codex discovery is structural instead of originator-gated
|
||||
// Bumped to 17: copilot CLI sessions were misclassified as VS Code transcripts
|
||||
// (#944), so days finalized at v16 or earlier carry output-only copilot costs —
|
||||
// the session.shutdown rollup's input/cache tokens were dropped. Raising
|
||||
// MIN_SUPPORTED_VERSION forces the one-time re-derivation under the
|
||||
// provenance-based classification; sourceless days carry forward as-is.
|
||||
//
|
||||
// v16: Codex discovery is structural instead of originator-gated
|
||||
// (#873/#626), so rollouts written by third-party frontends driving
|
||||
// `codex app-server` ("t3code_desktop", "JetBrains.IntelliJ IDEA", ...) now
|
||||
// contribute usage that v15 rollups never contained. Those files were rejected
|
||||
|
|
@ -67,8 +73,8 @@ import type { DateRange, ProjectSummary } from './types.js'
|
|||
// that older binaries skipped. v8 added local-model savings to the daily
|
||||
// rollup; the `savingsConfigHash` field is invalidated separately when the
|
||||
// user changes their `localModelSavings` mapping.
|
||||
export const DAILY_CACHE_VERSION = 16
|
||||
const MIN_SUPPORTED_VERSION = 16
|
||||
export const DAILY_CACHE_VERSION = 17
|
||||
const MIN_SUPPORTED_VERSION = 17
|
||||
// Version-suffixed so different binaries each own a distinct file and never
|
||||
// clobber an incompatible schema. Bumping the version mints a fresh filename;
|
||||
// adoptOlderDailyCaches then unions days out of every previous file (including
|
||||
|
|
|
|||
|
|
@ -194,6 +194,9 @@ type SubagentSelectedData = {
|
|||
agentName: string
|
||||
agentDisplayName?: string
|
||||
tools?: string[]
|
||||
// Present on subagent.started/completed (CLI ≥ ~1.0.7x): the delegation
|
||||
// tool call that launched the run, used to pair completed with started.
|
||||
toolCallId?: string
|
||||
}
|
||||
|
||||
// Per-model usage rollup the CLI writes into session.shutdown. inputTokens is
|
||||
|
|
@ -217,6 +220,8 @@ type CopilotEvent =
|
|||
| { type: 'user.message'; data: UserMessageData; timestamp?: string }
|
||||
| { type: 'assistant.message'; data: AssistantMessageData; timestamp?: string }
|
||||
| { type: 'subagent.selected'; data: SubagentSelectedData; timestamp?: string }
|
||||
| { type: 'subagent.started'; data: SubagentSelectedData; timestamp?: string }
|
||||
| { type: 'subagent.completed'; data: SubagentSelectedData; timestamp?: string }
|
||||
| { type: 'session.shutdown'; data: SessionShutdownData; timestamp?: string }
|
||||
|
||||
type ChatJournalPathSegment = string | number
|
||||
|
|
@ -693,50 +698,56 @@ function inferTranscriptModel(lines: string[]): string {
|
|||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// JSONL parser (handles both regular session-state events and VS Code
|
||||
// transcript format via session.start { producer: 'copilot-agent' })
|
||||
// JSONL parser (handles both regular CLI session-state events and the VS Code
|
||||
// transcript format — the same event vocabulary, but transcripts carry no
|
||||
// token counts and no session.shutdown rollup)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* `isTranscript` comes from discovery (where the file lives), never from
|
||||
* content: the Copilot CLI writes the same session.start producer
|
||||
* ('copilot-agent') that VS Code transcripts carry, so producer sniffing
|
||||
* misread every CLI session as a transcript and dropped its session.shutdown
|
||||
* input/cache rollup (#944).
|
||||
*/
|
||||
function createJsonlParser(
|
||||
source: SessionSource,
|
||||
seenKeys: Set<string>
|
||||
seenKeys: Set<string>,
|
||||
isTranscript: boolean
|
||||
): SessionParser {
|
||||
return {
|
||||
async *parse(): AsyncGenerator<ParsedProviderCall> {
|
||||
const content = await readSessionFile(source.path)
|
||||
if (!content) return
|
||||
const sessionId = basename(dirname(source.path))
|
||||
// CLI session-state files live at <sessionId>/events.jsonl; transcripts
|
||||
// at transcripts/<sessionId>.jsonl — keying the latter on the parent dir
|
||||
// would collapse every transcript into one "transcripts" session (and
|
||||
// one shared dedup namespace).
|
||||
const sessionId = isTranscript
|
||||
? basename(source.path, '.jsonl')
|
||||
: basename(dirname(source.path))
|
||||
const lines = content.split('\n').filter((l) => l.trim())
|
||||
|
||||
// Detect VS Code transcript format: the first session.start event has
|
||||
// { producer: 'copilot-agent' } and no outputTokens in messages.
|
||||
let isTranscript = false
|
||||
let currentModel = ''
|
||||
let pendingUserMessage = ''
|
||||
// Track the active subagent for this session (from subagent.selected events).
|
||||
// Resets when a new subagent is selected.
|
||||
let currentSubagentType: string | undefined
|
||||
|
||||
// First pass: detect format and infer transcript model if needed.
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const ev = JSON.parse(line) as CopilotEvent
|
||||
if (ev.type === 'session.start') {
|
||||
const data = ev.data as SessionStartData & { producer?: string }
|
||||
if (data.producer === 'copilot-agent') {
|
||||
isTranscript = true
|
||||
}
|
||||
break
|
||||
}
|
||||
if (ev.type === 'session.model_change') break // regular format
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
}
|
||||
// Subagent attribution. Older CLIs write subagent.selected — sticky
|
||||
// until replaced, never cleared. CLI ≥ ~1.0.7x brackets each run with
|
||||
// started/completed instead; runs can nest or overlap, so completed
|
||||
// removes ONLY its own toolCallId's entry and the label falls back to
|
||||
// the still-active run (or the sticky selected value) rather than
|
||||
// wiping attribution for everything in flight.
|
||||
let selectedSubagentType: string | undefined
|
||||
const activeSubagents: Array<{ toolCallId: string; name: string }> = []
|
||||
const currentSubagentType = (): string | undefined =>
|
||||
activeSubagents[activeSubagents.length - 1]?.name ?? selectedSubagentType
|
||||
|
||||
if (isTranscript) {
|
||||
// Tool-call-id prefix inference seeds the model; it must not gate the
|
||||
// whole file, or a transcript carrying explicit model info
|
||||
// (session.model_change / per-message model) but no tool calls would
|
||||
// yield nothing. Messages that still end up modelless are skipped
|
||||
// individually below.
|
||||
currentModel = inferTranscriptModel(lines)
|
||||
if (!currentModel) return // no toolCallIds to infer model from
|
||||
}
|
||||
|
||||
// Shutdown rollups may lack their own timestamp; remember the last
|
||||
|
|
@ -744,6 +755,15 @@ function createJsonlParser(
|
|||
// timestamp, which the date-range filters silently drop.
|
||||
let lastEventTimestamp = ''
|
||||
|
||||
// A resumed session appends one session.shutdown PER LEG, each carrying
|
||||
// CUMULATIVE per-model totals. Emitting each rollup whole would need the
|
||||
// cache to update a prior call in place — the durable merge is
|
||||
// append-only by dedup key — so we emit per-leg DELTAS keyed by
|
||||
// occurrence instead: re-parses of a growing file append only the new
|
||||
// leg, and each leg lands on its own timestamp.
|
||||
const prevShutdownUsage = new Map<string, ShutdownModelUsage>()
|
||||
const shutdownCountByModel = new Map<string, number>()
|
||||
|
||||
for (const line of lines) {
|
||||
let event: CopilotEvent
|
||||
try {
|
||||
|
|
@ -766,7 +786,34 @@ function createJsonlParser(
|
|||
}
|
||||
|
||||
if (event.type === 'subagent.selected') {
|
||||
currentSubagentType = (event.data as SubagentSelectedData).agentName
|
||||
selectedSubagentType = (event.data as SubagentSelectedData).agentName
|
||||
continue
|
||||
}
|
||||
|
||||
if (event.type === 'subagent.started') {
|
||||
const data = event.data as SubagentSelectedData
|
||||
activeSubagents.push({ toolCallId: data.toolCallId ?? '', name: data.agentName })
|
||||
continue
|
||||
}
|
||||
|
||||
if (event.type === 'subagent.completed') {
|
||||
const id = (event.data as SubagentSelectedData).toolCallId ?? ''
|
||||
if (!id) {
|
||||
// ID-less completion (transitional CLIs that key nothing, like
|
||||
// subagent.selected): end the most recently started run; explicit
|
||||
// no-op on an empty stack.
|
||||
activeSubagents.pop()
|
||||
continue
|
||||
}
|
||||
for (let i = activeSubagents.length - 1; i >= 0; i--) {
|
||||
if (activeSubagents[i]!.toolCallId === id) {
|
||||
activeSubagents.splice(i, 1)
|
||||
break
|
||||
}
|
||||
}
|
||||
// A non-empty id that matches nothing refers to a run we never saw
|
||||
// start — leave the active runs alone rather than evicting an
|
||||
// unrelated one.
|
||||
continue
|
||||
}
|
||||
|
||||
|
|
@ -783,11 +830,12 @@ function createJsonlParser(
|
|||
// is gated to the CLI (non-transcript) format, leaving VS Code,
|
||||
// JetBrains and OTel sources untouched.
|
||||
//
|
||||
// We emit one supplementary call per model carrying ONLY the
|
||||
// input/cache tokens the per-turn events lack; output is excluded so
|
||||
// the assistant.message output (and its cost) is not double-counted.
|
||||
// Combined with the per-turn output cost, this yields the full,
|
||||
// CLI-measured session cost.
|
||||
// We emit one supplementary call per model PER SHUTDOWN LEG (resumed
|
||||
// sessions write one cumulative rollup per leg; see the delta
|
||||
// tracking above) carrying ONLY the input/cache tokens the per-turn
|
||||
// events lack; output is excluded so the assistant.message output
|
||||
// (and its cost) is not double-counted. Combined with the per-turn
|
||||
// output cost, this yields the full, CLI-measured session cost.
|
||||
if (isTranscript) continue
|
||||
const shutdownData = event.data as SessionShutdownData
|
||||
const modelMetrics = shutdownData.modelMetrics
|
||||
|
|
@ -801,23 +849,49 @@ function createJsonlParser(
|
|||
const usage = metrics['usage']
|
||||
if (!isRecord(usage)) continue
|
||||
|
||||
const cacheReadTokens = numberOrZero(usage['cacheReadTokens'])
|
||||
const cacheWriteTokens = numberOrZero(usage['cacheWriteTokens'])
|
||||
const reasoningTokens = numberOrZero(usage['reasoningTokens'])
|
||||
const cumulative: Required<ShutdownModelUsage> = {
|
||||
inputTokens: numberOrZero(usage['inputTokens']),
|
||||
outputTokens: numberOrZero(usage['outputTokens']),
|
||||
cacheReadTokens: numberOrZero(usage['cacheReadTokens']),
|
||||
cacheWriteTokens: numberOrZero(usage['cacheWriteTokens']),
|
||||
reasoningTokens: numberOrZero(usage['reasoningTokens']),
|
||||
}
|
||||
const prevRaw = prevShutdownUsage.get(model)
|
||||
prevShutdownUsage.set(model, cumulative)
|
||||
const n = (shutdownCountByModel.get(model) ?? 0) + 1
|
||||
shutdownCountByModel.set(model, n)
|
||||
|
||||
// A cumulative total BELOW the previous rollup means the CLI reset
|
||||
// its counters (a fresh accounting epoch): delta from zero, else
|
||||
// this leg's post-reset usage would be clamped away entirely.
|
||||
// inputTokens is the monotonic sentinel — it is cache-inclusive,
|
||||
// so any usage at all grows it.
|
||||
const prev =
|
||||
prevRaw && cumulative.inputTokens < numberOrZero(prevRaw.inputTokens)
|
||||
? undefined
|
||||
: prevRaw
|
||||
|
||||
// This leg's contribution: cumulative minus the previous rollup.
|
||||
// The clamp guards any remaining non-monotonic field.
|
||||
const delta = (k: keyof ShutdownModelUsage): number =>
|
||||
Math.max(0, cumulative[k] - numberOrZero(prev?.[k]))
|
||||
const cacheReadTokens = delta('cacheReadTokens')
|
||||
const cacheWriteTokens = delta('cacheWriteTokens')
|
||||
const reasoningTokens = delta('reasoningTokens')
|
||||
// usage.inputTokens is cache-INCLUSIVE (input + cache_read +
|
||||
// cache_write). calculateCost expects the uncached input alone with
|
||||
// cache tokens billed separately, so subtract the cache components.
|
||||
// Clamp at 0 in case a future schema reports input non-inclusively.
|
||||
const inputTokens = Math.max(
|
||||
0,
|
||||
numberOrZero(usage['inputTokens']) - cacheReadTokens - cacheWriteTokens
|
||||
delta('inputTokens') - cacheReadTokens - cacheWriteTokens
|
||||
)
|
||||
|
||||
// Nothing this call would add over the per-turn events, so skip it
|
||||
// to avoid an empty $0 row (output is intentionally excluded).
|
||||
if (inputTokens === 0 && cacheReadTokens === 0 && cacheWriteTokens === 0) continue
|
||||
if (inputTokens === 0 && cacheReadTokens === 0 && cacheWriteTokens === 0 && reasoningTokens === 0) continue
|
||||
|
||||
const dedupKey = `copilot:${sessionId}:shutdown:${model}`
|
||||
const dedupKey = `copilot:${sessionId}:shutdown:${model}:${n}`
|
||||
if (seenKeys.has(dedupKey)) continue
|
||||
seenKeys.add(dedupKey)
|
||||
|
||||
|
|
@ -898,6 +972,7 @@ function createJsonlParser(
|
|||
// Cost will be lower than actual API cost. This is the original
|
||||
// behaviour — OTel data (below) replaces it when available.
|
||||
const costUSD = calculateCost(currentModel, 0, outputTokens, 0, 0, 0)
|
||||
const subagentType = currentSubagentType()
|
||||
|
||||
yield {
|
||||
provider: 'copilot',
|
||||
|
|
@ -914,7 +989,7 @@ function createJsonlParser(
|
|||
tools,
|
||||
bashCommands,
|
||||
skills: skills.length > 0 ? skills : undefined,
|
||||
subagentTypes: currentSubagentType ? [currentSubagentType] : undefined,
|
||||
subagentTypes: subagentType ? [subagentType] : undefined,
|
||||
timestamp: event.timestamp ?? '',
|
||||
speed: 'standard' as const,
|
||||
deduplicationKey: dedupKey,
|
||||
|
|
@ -1837,6 +1912,12 @@ interface JsonlSessionSource extends SessionSource {
|
|||
sourceType: 'jsonl'
|
||||
}
|
||||
|
||||
// A VS Code workspaceStorage transcript. Distinct from 'jsonl' (CLI
|
||||
// session-state) so classification rides provenance, not file contents (#944).
|
||||
interface TranscriptSessionSource extends SessionSource {
|
||||
sourceType: 'transcript'
|
||||
}
|
||||
|
||||
interface ChatSessionSource extends SessionSource {
|
||||
sourceType: 'chatsession'
|
||||
}
|
||||
|
|
@ -1874,6 +1955,10 @@ function isJetBrainsSource(source: SessionSource): source is JetBrainsSessionSou
|
|||
return (source as JetBrainsSessionSource).sourceType === 'jetbrains'
|
||||
}
|
||||
|
||||
function isTranscriptSource(source: SessionSource): source is TranscriptSessionSource {
|
||||
return (source as TranscriptSessionSource).sourceType === 'transcript'
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Session discovery: JSONL (original)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -2242,8 +2327,8 @@ async function discoverEmptyWindowChatSessions(
|
|||
*/
|
||||
async function discoverTranscriptSessions(
|
||||
workspaceStorageDirs: string[]
|
||||
): Promise<JsonlSessionSource[]> {
|
||||
const sources: JsonlSessionSource[] = []
|
||||
): Promise<TranscriptSessionSource[]> {
|
||||
const sources: TranscriptSessionSource[] = []
|
||||
|
||||
for (const wsDir of workspaceStorageDirs) {
|
||||
let hashDirs: string[]
|
||||
|
|
@ -2275,7 +2360,7 @@ async function discoverTranscriptSessions(
|
|||
path: join(transcriptsDir, file),
|
||||
project,
|
||||
provider: 'copilot',
|
||||
sourceType: 'jsonl',
|
||||
sourceType: 'transcript',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -2418,7 +2503,7 @@ export function createCopilotProvider(
|
|||
if (isJetBrainsSource(source)) {
|
||||
return createJetBrainsParser(source, seenKeys)
|
||||
}
|
||||
return createJsonlParser(source, seenKeys)
|
||||
return createJsonlParser(source, seenKeys, isTranscriptSource(source))
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -224,7 +224,10 @@ export const PROVIDER_PARSE_VERSIONS: Record<string, string> = {
|
|||
codex: 'mcp-attribution-v5-est-cost-active-timing-mcp-wait-rich-capture-v1-cross-provider-pr-v1',
|
||||
cursor: 'composer-anchored-crediting-v1-est-cost',
|
||||
'cursor-agent': 'workspaceless-transcript-v1',
|
||||
copilot: 'cli-shutdown-cost-v1-skills',
|
||||
// source-provenance-v1 (#944): CLI sessions were misread as VS Code
|
||||
// transcripts (both carry producer 'copilot-agent'), skipping the shutdown
|
||||
// input/cache rollup; this bump re-parses them so the missing tokens land.
|
||||
copilot: 'cli-shutdown-cost-v1-skills-source-provenance-v1',
|
||||
grok: 'estimated-cost-v1',
|
||||
hermes: 'reasoning-output-accounting-v1-est-cost',
|
||||
'lingtai-tui': 'token-ledger-registry-activity-v3',
|
||||
|
|
|
|||
|
|
@ -605,3 +605,77 @@ describe('(h) provider filter excludes claude from the orphan pass', () => {
|
|||
expect(totalCost(after)).toBeCloseTo(costBefore, 10)
|
||||
})
|
||||
})
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// (f) Growing resumed CLI session: durable merge appends only the new leg
|
||||
// ═══════════════════════════════════════════════════════════════════════════
|
||||
// Resumed Copilot CLI sessions append one CUMULATIVE session.shutdown per leg
|
||||
// (#944). The parser emits per-leg deltas keyed by occurrence; this exercises
|
||||
// the PRODUCTION merge path — the durable union-by-dedup-key merge against the
|
||||
// on-disk cache when the file grows between parses — which the unit tests
|
||||
// (which pre-seed seenKeys) cannot reach.
|
||||
describe('(f) growing resumed CLI session durable merge', () => {
|
||||
it('totals equal the final cumulative rollup after the file grows a leg', async () => {
|
||||
const sessionStateDir = join(tmpHome, 'session-state')
|
||||
await mkdir(sessionStateDir, { recursive: true })
|
||||
vi.stubEnv('CODEBURN_COPILOT_SESSION_STATE_DIR', sessionStateDir)
|
||||
vi.stubEnv('CODEBURN_COPILOT_DISABLE_OTEL', '1')
|
||||
vi.stubEnv('CODEBURN_COPILOT_WS_STORAGE_DIR', join(tmpHome, 'no-ws'))
|
||||
vi.stubEnv('CODEBURN_COPILOT_GLOBAL_STORAGE_DIR', join(tmpHome, 'no-global'))
|
||||
vi.stubEnv('CODEBURN_COPILOT_JETBRAINS_DIR', join(tmpHome, 'no-jb'))
|
||||
|
||||
const base = Date.now() - 5 * 24 * 60 * 60 * 1000
|
||||
const at = (offsetSec: number): string => new Date(base + offsetSec * 1000).toISOString()
|
||||
const dir = join(sessionStateDir, 'sess-grow')
|
||||
await mkdir(dir, { recursive: true })
|
||||
await writeFile(join(dir, 'workspace.yaml'), 'id: sess-grow\ncwd: /home/user/testproj\n')
|
||||
const eventsPath = join(dir, 'events.jsonl')
|
||||
|
||||
// Cumulative rollups from a real resumed CLI 1.0.78 session.
|
||||
const shutdown = (ts: string, inputTokens: number, cacheReadTokens: number, cacheWriteTokens: number, outputTokens: number) =>
|
||||
JSON.stringify({
|
||||
type: 'session.shutdown',
|
||||
timestamp: ts,
|
||||
data: {
|
||||
shutdownType: 'routine',
|
||||
modelMetrics: {
|
||||
'claude-sonnet-4-5': {
|
||||
requests: { count: 1, cost: 1 },
|
||||
usage: { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, reasoningTokens: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
const leg1 = [
|
||||
JSON.stringify({ type: 'session.model_change', timestamp: at(0), data: { newModel: 'claude-sonnet-4-5' } }),
|
||||
JSON.stringify({ type: 'assistant.message', timestamp: at(10), data: { messageId: 'msg-1', outputTokens: 17, toolRequests: [] } }),
|
||||
shutdown(at(20), 24672, 0, 24670, 17),
|
||||
]
|
||||
await writeFile(eventsPath, leg1.join('\n') + '\n')
|
||||
|
||||
const sumUsage = (projects: Awaited<ReturnType<typeof parseAllSessions>>) => {
|
||||
const calls = projects.flatMap(p => p.sessions).flatMap(s => s.turns).flatMap(t => t.assistantCalls)
|
||||
return {
|
||||
input: calls.reduce((s, c) => s + c.usage.inputTokens, 0),
|
||||
cacheRead: calls.reduce((s, c) => s + c.usage.cacheReadInputTokens, 0),
|
||||
cacheWrite: calls.reduce((s, c) => s + c.usage.cacheCreationInputTokens, 0),
|
||||
}
|
||||
}
|
||||
|
||||
const first = sumUsage(await parseAllSessions(undefined, 'copilot'))
|
||||
expect(first).toEqual({ input: 2, cacheRead: 0, cacheWrite: 24670 })
|
||||
|
||||
// The session resumes: leg 2 appends per-turn events plus a CUMULATIVE
|
||||
// rollup. The cached leg-1 delta must be kept once and only the leg-2
|
||||
// delta appended — totals equal the final cumulative rollup exactly.
|
||||
clearSessionCache()
|
||||
await writeFile(eventsPath, [
|
||||
...leg1,
|
||||
JSON.stringify({ type: 'assistant.message', timestamp: at(100), data: { messageId: 'msg-2', outputTokens: 132, toolRequests: [] } }),
|
||||
shutdown(at(120), 74463, 49489, 24968, 149),
|
||||
].join('\n') + '\n')
|
||||
|
||||
const second = sumUsage(await parseAllSessions(undefined, 'copilot'))
|
||||
expect(second).toEqual({ input: 74463 - 49489 - 24968, cacheRead: 49489, cacheWrite: 24968 })
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -124,6 +124,18 @@ async function collectCalls(source: { path: string; project: string; provider: s
|
|||
return calls
|
||||
}
|
||||
|
||||
// Write a transcript inside the test's tmpDir sandbox, but at the production
|
||||
// directory shape — {ws}/{hash}/GitHub.copilot-chat/transcripts/<id>.jsonl —
|
||||
// because sessionId derivation reads the path structure (file basename for
|
||||
// transcripts). Never touches the real VS Code storage.
|
||||
async function createTranscriptFile(sessionId: string, lines: string[]) {
|
||||
const transcriptsDir = join(tmpDir, 'ws', 'hash1', 'GitHub.copilot-chat', 'transcripts')
|
||||
await mkdir(transcriptsDir, { recursive: true })
|
||||
const path = join(transcriptsDir, `${sessionId}.jsonl`)
|
||||
await writeFile(path, lines.join('\n') + '\n')
|
||||
return path
|
||||
}
|
||||
|
||||
describe('copilot provider - JSONL parsing', () => {
|
||||
beforeEach(async () => {
|
||||
tmpDir = await mkdtemp(join(tmpdir(), 'copilot-test-'))
|
||||
|
|
@ -335,8 +347,122 @@ describe('copilot provider - JSONL parsing', () => {
|
|||
expect(calls[0]!.model).toBe('gpt-4.1')
|
||||
})
|
||||
|
||||
it('attributes turns between subagent.started and subagent.completed to the subagent', async () => {
|
||||
// CLI ≥ ~1.0.7x writes subagent.started/completed (not subagent.selected);
|
||||
// event shapes from a real delegating 1.0.78 session. The label must cover
|
||||
// the subagent's turns and clear afterwards, not bleed onto the parent's.
|
||||
const eventsPath = await createSessionDir('sess-subagent-cli', [
|
||||
modelChange('claude-sonnet-5'),
|
||||
userMessage('delegate a search'),
|
||||
JSON.stringify({
|
||||
type: 'subagent.started',
|
||||
timestamp: '2026-08-07T10:00:11Z',
|
||||
data: { toolCallId: 'toolu_01SZnHjC', agentName: 'explore', agentDisplayName: 'Explore Agent' },
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: 'assistant.message',
|
||||
timestamp: '2026-08-07T10:00:14Z',
|
||||
data: { messageId: 'msg-sub', model: 'claude-haiku-4.5', outputTokens: 197, toolRequests: [] },
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: 'subagent.completed',
|
||||
timestamp: '2026-08-07T10:00:19Z',
|
||||
data: { toolCallId: 'toolu_01SZnHjC', agentName: 'explore', model: 'claude-haiku-4.5', totalTokens: 26435 },
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: 'assistant.message',
|
||||
timestamp: '2026-08-07T10:00:22Z',
|
||||
data: { messageId: 'msg-parent', model: 'claude-sonnet-5', outputTokens: 51, toolRequests: [] },
|
||||
}),
|
||||
])
|
||||
|
||||
const calls = await collectCalls({ path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'jsonl' })
|
||||
const sub = calls.find(c => c.deduplicationKey.endsWith(':msg-sub'))!
|
||||
expect(sub.subagentTypes).toEqual(['explore'])
|
||||
expect(sub.model).toBe('claude-haiku-4.5')
|
||||
const parent = calls.find(c => c.deduplicationKey.endsWith(':msg-parent'))!
|
||||
expect(parent.subagentTypes).toBeUndefined()
|
||||
})
|
||||
|
||||
it('completing a nested subagent restores the outer label, matched by toolCallId', async () => {
|
||||
const started = (id: string, name: string) =>
|
||||
JSON.stringify({ type: 'subagent.started', timestamp: '2026-08-07T10:00:11Z', data: { toolCallId: id, agentName: name } })
|
||||
const completed = (id: string) =>
|
||||
JSON.stringify({ type: 'subagent.completed', timestamp: '2026-08-07T10:00:19Z', data: { toolCallId: id, agentName: 'x' } })
|
||||
const msg = (messageId: string, outputTokens = 10) =>
|
||||
JSON.stringify({ type: 'assistant.message', timestamp: '2026-08-07T10:00:14Z', data: { messageId, model: 'claude-sonnet-5', outputTokens, toolRequests: [] } })
|
||||
|
||||
const eventsPath = await createSessionDir('sess-subagent-nested', [
|
||||
modelChange('claude-sonnet-5'),
|
||||
started('call-A', 'explore'),
|
||||
started('call-B', 'plan'),
|
||||
msg('msg-inner'), // while B runs → 'plan'
|
||||
completed('call-B'),
|
||||
msg('msg-outer'), // B done, A still active → 'explore', NOT unlabeled
|
||||
completed('call-A'),
|
||||
msg('msg-after'), // all done → no label
|
||||
])
|
||||
|
||||
const calls = await collectCalls({ path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'jsonl' })
|
||||
const byId = (id: string) => calls.find(c => c.deduplicationKey.endsWith(`:${id}`))!
|
||||
expect(byId('msg-inner').subagentTypes).toEqual(['plan'])
|
||||
expect(byId('msg-outer').subagentTypes).toEqual(['explore'])
|
||||
expect(byId('msg-after').subagentTypes).toBeUndefined()
|
||||
})
|
||||
|
||||
it('ignores a completed event whose non-empty toolCallId matches no active run', async () => {
|
||||
// A completion for a run we never saw start must not evict an unrelated
|
||||
// active run; only a genuinely ID-less completion may pop the stack.
|
||||
const eventsPath = await createSessionDir('sess-subagent-unmatched', [
|
||||
modelChange('claude-sonnet-5'),
|
||||
JSON.stringify({
|
||||
type: 'subagent.started',
|
||||
timestamp: '2026-08-07T10:00:11Z',
|
||||
data: { toolCallId: 'call-A', agentName: 'explore' },
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: 'subagent.completed',
|
||||
timestamp: '2026-08-07T10:00:12Z',
|
||||
data: { toolCallId: 'call-unknown', agentName: 'phantom' },
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: 'assistant.message',
|
||||
timestamp: '2026-08-07T10:00:14Z',
|
||||
data: { messageId: 'msg-1', model: 'claude-sonnet-5', outputTokens: 10, toolRequests: [] },
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: 'subagent.completed',
|
||||
timestamp: '2026-08-07T10:00:15Z',
|
||||
data: { agentName: 'legacy-no-id' },
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: 'assistant.message',
|
||||
timestamp: '2026-08-07T10:00:16Z',
|
||||
data: { messageId: 'msg-2', model: 'claude-sonnet-5', outputTokens: 12, toolRequests: [] },
|
||||
}),
|
||||
])
|
||||
|
||||
const calls = await collectCalls({ path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'jsonl' })
|
||||
// The unmatched completion left 'explore' active…
|
||||
expect(calls.find(c => c.deduplicationKey.endsWith(':msg-1'))!.subagentTypes).toEqual(['explore'])
|
||||
// …and the ID-less completion (legacy shape) ended it.
|
||||
expect(calls.find(c => c.deduplicationKey.endsWith(':msg-2'))!.subagentTypes).toBeUndefined()
|
||||
})
|
||||
|
||||
it('keeps subagent.selected sticky when no completed event ever arrives', async () => {
|
||||
// Older CLIs only write subagent.selected; nothing clears it.
|
||||
const eventsPath = await createSessionDir('sess-subagent-selected', [
|
||||
modelChange('claude-sonnet-5'),
|
||||
JSON.stringify({ type: 'subagent.selected', data: { agentName: 'refactor' } }),
|
||||
assistantMessage({ messageId: 'msg-1', outputTokens: 25 }),
|
||||
assistantMessage({ messageId: 'msg-2', outputTokens: 30, timestamp: '2026-04-15T10:01:00Z' }),
|
||||
])
|
||||
const calls = await collectCalls({ path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'jsonl' })
|
||||
expect(calls.map(c => c.subagentTypes)).toEqual([['refactor'], ['refactor']])
|
||||
})
|
||||
|
||||
it('infers OpenAI auto bucket for transcript toolCallId prefix call_', async () => {
|
||||
const eventsPath = await createSessionDir('sess-tr-call', [
|
||||
const eventsPath = await createTranscriptFile('sess-tr-call', [
|
||||
transcriptSessionStart('sess-tr-call'),
|
||||
transcriptUserMessage('check model inference'),
|
||||
transcriptAssistantMessage({
|
||||
|
|
@ -346,16 +472,20 @@ describe('copilot provider - JSONL parsing', () => {
|
|||
}),
|
||||
])
|
||||
|
||||
const source = { path: eventsPath, project: 'test', provider: 'copilot' }
|
||||
const source = { path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'transcript' }
|
||||
const calls: ParsedProviderCall[] = []
|
||||
for await (const call of copilot.createSessionParser(source, new Set()).parse()) calls.push(call)
|
||||
|
||||
expect(calls).toHaveLength(1)
|
||||
expect(calls[0]!.model).toBe('copilot-openai-auto')
|
||||
// Each transcript is its own session, keyed by file basename — NOT the
|
||||
// shared parent dir name 'transcripts', which would collapse every
|
||||
// transcript into one session and one dedup namespace.
|
||||
expect(calls[0]!.sessionId).toBe('sess-tr-call')
|
||||
})
|
||||
|
||||
it('infers Anthropic auto bucket for transcript toolCallId prefixes tooluse_/toolu_vrtx_', async () => {
|
||||
const eventsPath = await createSessionDir('sess-tr-claude', [
|
||||
const eventsPath = await createTranscriptFile('sess-tr-claude', [
|
||||
transcriptSessionStart('sess-tr-claude'),
|
||||
transcriptUserMessage('check model inference'),
|
||||
transcriptAssistantMessage({
|
||||
|
|
@ -365,7 +495,7 @@ describe('copilot provider - JSONL parsing', () => {
|
|||
}),
|
||||
])
|
||||
|
||||
const source = { path: eventsPath, project: 'test', provider: 'copilot' }
|
||||
const source = { path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'transcript' }
|
||||
const calls: ParsedProviderCall[] = []
|
||||
for await (const call of copilot.createSessionParser(source, new Set()).parse()) calls.push(call)
|
||||
|
||||
|
|
@ -374,7 +504,7 @@ describe('copilot provider - JSONL parsing', () => {
|
|||
})
|
||||
|
||||
it('chooses the dominant inferred transcript model when prefixes are mixed', async () => {
|
||||
const eventsPath = await createSessionDir('sess-tr-mixed', [
|
||||
const eventsPath = await createTranscriptFile('sess-tr-mixed', [
|
||||
transcriptSessionStart('sess-tr-mixed'),
|
||||
transcriptUserMessage('mixed'),
|
||||
transcriptAssistantMessage({
|
||||
|
|
@ -394,7 +524,7 @@ describe('copilot provider - JSONL parsing', () => {
|
|||
}),
|
||||
])
|
||||
|
||||
const source = { path: eventsPath, project: 'test', provider: 'copilot' }
|
||||
const source = { path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'transcript' }
|
||||
const calls: ParsedProviderCall[] = []
|
||||
for await (const call of copilot.createSessionParser(source, new Set()).parse()) calls.push(call)
|
||||
|
||||
|
|
@ -402,8 +532,35 @@ describe('copilot provider - JSONL parsing', () => {
|
|||
expect(calls.every(c => c.model === 'copilot-openai-auto')).toBe(true)
|
||||
})
|
||||
|
||||
it('parses a producerless transcript with explicit model info and no tool calls', async () => {
|
||||
// Prefix inference has nothing to work with here; the explicit
|
||||
// session.model_change must still establish the model, and the shutdown
|
||||
// rollup must stay ignored — provenance, not the producer field, gates it.
|
||||
const eventsPath = await createTranscriptFile('sess-tr-explicit', [
|
||||
JSON.stringify({ type: 'session.start', data: { sessionId: 'sess-tr-explicit' } }),
|
||||
modelChange('gpt-4.1'),
|
||||
transcriptUserMessage('hi'),
|
||||
JSON.stringify({
|
||||
type: 'assistant.message',
|
||||
timestamp: '2026-04-15T10:00:15Z',
|
||||
data: { messageId: 'msg-1', outputTokens: 80, toolRequests: [] },
|
||||
}),
|
||||
shutdownEvent({
|
||||
modelMetrics: {
|
||||
'gpt-4.1': { inputTokens: 1000, outputTokens: 80, cacheReadTokens: 500, cacheWriteTokens: 200 },
|
||||
},
|
||||
}),
|
||||
])
|
||||
|
||||
const calls = await collectCalls({ path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'transcript' })
|
||||
expect(calls).toHaveLength(1)
|
||||
expect(calls[0]!.model).toBe('gpt-4.1')
|
||||
expect(calls[0]!.outputTokens).toBe(80)
|
||||
expect(calls.every(c => !c.deduplicationKey.includes(':shutdown:'))).toBe(true)
|
||||
})
|
||||
|
||||
it('normalizes Copilot MCP tool names from VS Code transcripts', async () => {
|
||||
const eventsPath = await createSessionDir('sess-tr-mcp-tools', [
|
||||
const eventsPath = await createTranscriptFile('sess-tr-mcp-tools', [
|
||||
transcriptSessionStart('sess-tr-mcp-tools'),
|
||||
transcriptUserMessage('use GitHub MCP'),
|
||||
transcriptAssistantMessage({
|
||||
|
|
@ -414,7 +571,7 @@ describe('copilot provider - JSONL parsing', () => {
|
|||
}),
|
||||
])
|
||||
|
||||
const source = { path: eventsPath, project: 'test', provider: 'copilot' }
|
||||
const source = { path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'transcript' }
|
||||
const calls: ParsedProviderCall[] = []
|
||||
for await (const call of copilot.createSessionParser(source, new Set()).parse()) calls.push(call)
|
||||
|
||||
|
|
@ -458,7 +615,7 @@ describe('copilot provider - session.shutdown token/cost rollup', () => {
|
|||
// One per-turn assistant.message call + one supplementary shutdown call.
|
||||
expect(calls).toHaveLength(2)
|
||||
|
||||
const shutdown = calls.find(c => c.deduplicationKey === 'copilot:sess-shutdown:shutdown:claude-sonnet-4-5')
|
||||
const shutdown = calls.find(c => c.deduplicationKey === 'copilot:sess-shutdown:shutdown:claude-sonnet-4-5:1')
|
||||
expect(shutdown).toBeDefined()
|
||||
expect(shutdown!.model).toBe('claude-sonnet-4-5')
|
||||
expect(shutdown!.inputTokens).toBe(4) // 71282 - 35495 - 35783
|
||||
|
|
@ -546,6 +703,86 @@ describe('copilot provider - session.shutdown token/cost rollup', () => {
|
|||
expect(gpt.costUSD).toBeCloseTo(calculateCost('gpt-5', 50, 0, 0, 5000, 0), 12)
|
||||
})
|
||||
|
||||
it('emits per-leg deltas for a resumed session with cumulative shutdown rollups', async () => {
|
||||
// Numbers from a real resumed CLI 1.0.78 session (3 legs via --resume):
|
||||
// each leg appends a session.shutdown whose modelMetrics are CUMULATIVE.
|
||||
// Emitting deltas keyed by occurrence keeps a growing file append-only
|
||||
// under the durable union-by-key cache merge — re-parsing after each
|
||||
// resume adds only the new leg, never double-counting earlier ones.
|
||||
const legs = [
|
||||
{ inputTokens: 24672, outputTokens: 17, cacheReadTokens: 0, cacheWriteTokens: 24670 },
|
||||
{ inputTokens: 74463, outputTokens: 149, cacheReadTokens: 49489, cacheWriteTokens: 24968 },
|
||||
{ inputTokens: 124783, outputTokens: 243, cacheReadTokens: 99569, cacheWriteTokens: 25204 },
|
||||
]
|
||||
const lines = [modelChange('claude-sonnet-5'), assistantMessage({ messageId: 'msg-1', outputTokens: 17 })]
|
||||
for (const [i, leg] of legs.entries()) {
|
||||
lines.push(shutdownEvent({ modelMetrics: { 'claude-sonnet-5': leg }, timestamp: `2026-08-0${i + 1}T10:00:00Z` }))
|
||||
}
|
||||
const eventsPath = await createSessionDir('sess-resumed', lines)
|
||||
const calls = await collectCalls({ path: eventsPath, project: 'myproject', provider: 'copilot', sourceType: 'jsonl' })
|
||||
|
||||
const shutdowns = calls.filter(c => c.deduplicationKey.includes(':shutdown:'))
|
||||
expect(shutdowns.map(c => c.deduplicationKey)).toEqual([
|
||||
'copilot:sess-resumed:shutdown:claude-sonnet-5:1',
|
||||
'copilot:sess-resumed:shutdown:claude-sonnet-5:2',
|
||||
'copilot:sess-resumed:shutdown:claude-sonnet-5:3',
|
||||
])
|
||||
// Each leg lands on its own shutdown timestamp (a resumed session can
|
||||
// span days; whole-rollup emission would collapse them onto one).
|
||||
expect(shutdowns.map(c => c.timestamp)).toEqual([
|
||||
'2026-08-01T10:00:00Z', '2026-08-02T10:00:00Z', '2026-08-03T10:00:00Z',
|
||||
])
|
||||
// Per-leg deltas sum exactly to the final cumulative rollup.
|
||||
const sum = (k: 'inputTokens' | 'cacheReadInputTokens' | 'cacheCreationInputTokens') =>
|
||||
shutdowns.reduce((a, c) => a + c[k], 0)
|
||||
expect(sum('cacheReadInputTokens')).toBe(99569)
|
||||
expect(sum('cacheCreationInputTokens')).toBe(25204)
|
||||
expect(sum('inputTokens')).toBe(124783 - 99569 - 25204)
|
||||
|
||||
// A later re-parse of the grown file (prior legs already cached) emits
|
||||
// only what the seen-key set lacks.
|
||||
const seen = new Set(calls.map(c => c.deduplicationKey))
|
||||
const again = await collectCalls({ path: eventsPath, project: 'myproject', provider: 'copilot', sourceType: 'jsonl' }, seen)
|
||||
expect(again).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('starts a fresh delta baseline when a cumulative rollup goes backwards (counter reset)', async () => {
|
||||
// Hypothetical but cheap to guard: if the CLI ever resets its counters
|
||||
// mid-session, the post-reset epoch must be billed from zero — a stale
|
||||
// high-water baseline would clamp it away (and the reset leg's real usage
|
||||
// with it).
|
||||
const eventsPath = await createSessionDir('sess-reset', [
|
||||
modelChange('claude-sonnet-5'),
|
||||
assistantMessage({ messageId: 'msg-1', outputTokens: 10 }),
|
||||
shutdownEvent({
|
||||
modelMetrics: { 'claude-sonnet-5': { inputTokens: 10000, outputTokens: 10, cacheReadTokens: 0, cacheWriteTokens: 5000 } },
|
||||
timestamp: '2026-08-01T10:00:00Z',
|
||||
}),
|
||||
// Reset: cumulative drops below the previous rollup → new epoch.
|
||||
shutdownEvent({
|
||||
modelMetrics: { 'claude-sonnet-5': { inputTokens: 2000, outputTokens: 5, cacheReadTokens: 0, cacheWriteTokens: 1000 } },
|
||||
timestamp: '2026-08-02T10:00:00Z',
|
||||
}),
|
||||
shutdownEvent({
|
||||
modelMetrics: { 'claude-sonnet-5': { inputTokens: 5000, outputTokens: 8, cacheReadTokens: 2000, cacheWriteTokens: 1500 } },
|
||||
timestamp: '2026-08-03T10:00:00Z',
|
||||
}),
|
||||
])
|
||||
const calls = await collectCalls({ path: eventsPath, project: 'myproject', provider: 'copilot', sourceType: 'jsonl' })
|
||||
const shutdowns = calls.filter(c => c.deduplicationKey.includes(':shutdown:'))
|
||||
expect(shutdowns).toHaveLength(3)
|
||||
// Leg 1: epoch-1 usage in full.
|
||||
expect(shutdowns[0]!.inputTokens).toBe(5000) // 10000 − 0 − 5000
|
||||
expect(shutdowns[0]!.cacheCreationInputTokens).toBe(5000)
|
||||
// Leg 2 (reset): billed from zero, not clamped away against the old baseline.
|
||||
expect(shutdowns[1]!.inputTokens).toBe(1000) // 2000 − 0 − 1000
|
||||
expect(shutdowns[1]!.cacheCreationInputTokens).toBe(1000)
|
||||
// Leg 3: normal delta within the new epoch.
|
||||
expect(shutdowns[2]!.inputTokens).toBe(500) // (5000−2000) − 2000 − 500
|
||||
expect(shutdowns[2]!.cacheReadInputTokens).toBe(2000)
|
||||
expect(shutdowns[2]!.cacheCreationInputTokens).toBe(500)
|
||||
})
|
||||
|
||||
it('keeps shutdown dedup keys stable across re-parses', async () => {
|
||||
const eventsPath = await createSessionDir('sess-reparse', [
|
||||
modelChange('claude-sonnet-4-5'),
|
||||
|
|
@ -594,7 +831,7 @@ describe('copilot provider - session.shutdown token/cost rollup', () => {
|
|||
})
|
||||
|
||||
it('ignores session.shutdown for VS Code transcript sessions', async () => {
|
||||
const eventsPath = await createSessionDir('sess-tr-shutdown', [
|
||||
const eventsPath = await createTranscriptFile('sess-tr-shutdown', [
|
||||
transcriptSessionStart('sess-tr-shutdown'),
|
||||
transcriptUserMessage('hi'),
|
||||
transcriptAssistantMessage({ messageId: 'msg-1', content: 'done', toolCallIds: ['call_abc'] }),
|
||||
|
|
@ -604,7 +841,7 @@ describe('copilot provider - session.shutdown token/cost rollup', () => {
|
|||
},
|
||||
}),
|
||||
])
|
||||
const source = { path: eventsPath, project: 'test', provider: 'copilot' }
|
||||
const source = { path: eventsPath, project: 'test', provider: 'copilot', sourceType: 'transcript' }
|
||||
const calls = await collectCalls(source)
|
||||
|
||||
// Only the transcript assistant call; the shutdown rollup is CLI-only.
|
||||
|
|
@ -612,6 +849,165 @@ describe('copilot provider - session.shutdown token/cost rollup', () => {
|
|||
expect(calls.every(c => !c.deduplicationKey.includes(':shutdown:'))).toBe(true)
|
||||
expect(calls[0]!.model).toBe('copilot-openai-auto')
|
||||
})
|
||||
|
||||
// Regression test for #944: events are redacted copies of a real Copilot CLI
|
||||
// 1.0.78 session. The CLI writes the same producer ('copilot-agent') as VS
|
||||
// Code transcripts, so content sniffing skipped this session's shutdown
|
||||
// rollup — reporting 100 of its 49,573 tokens and zero input/cache.
|
||||
it('parses a CLI session whose session.start carries producer copilot-agent (issue #944)', async () => {
|
||||
const eventsPath = await createSessionDir('sess-cli-producer', [
|
||||
JSON.stringify({
|
||||
type: 'session.start',
|
||||
timestamp: '2026-08-07T17:56:35.573Z',
|
||||
data: {
|
||||
sessionId: 'sess-cli-producer',
|
||||
version: 1,
|
||||
producer: 'copilot-agent',
|
||||
copilotVersion: '1.0.78',
|
||||
startTime: '2026-08-07T17:56:35.554Z',
|
||||
context: { cwd: '/home/user/myproject' },
|
||||
},
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: 'session.model_change',
|
||||
timestamp: '2026-08-07T17:56:36.725Z',
|
||||
data: { newModel: 'claude-sonnet-5', reasoningEffort: null },
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: 'user.message',
|
||||
timestamp: '2026-08-07T17:56:36.732Z',
|
||||
data: { content: 'Run echo and summarize the output.' },
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: 'assistant.message',
|
||||
timestamp: '2026-08-07T17:56:38.763Z',
|
||||
data: {
|
||||
messageId: 'a982a391-9ee3-4fbd-89a9-26d5af78c890',
|
||||
model: 'claude-sonnet-5',
|
||||
content: '',
|
||||
toolRequests: [{
|
||||
toolCallId: 'toolu_017eL3f5aeGiLoALignYMZEN',
|
||||
name: 'bash',
|
||||
arguments: { command: 'echo codeburn-repro-944', description: 'Echo test string' },
|
||||
type: 'function',
|
||||
}],
|
||||
turnId: '0',
|
||||
outputTokens: 81,
|
||||
},
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: 'assistant.message',
|
||||
timestamp: '2026-08-07T17:56:40.417Z',
|
||||
data: {
|
||||
messageId: '8758ea51-797f-4285-972c-495911e2839f',
|
||||
model: 'claude-sonnet-5',
|
||||
content: 'The command printed the string "codeburn-repro-944".',
|
||||
toolRequests: [],
|
||||
turnId: '1',
|
||||
outputTokens: 19,
|
||||
},
|
||||
}),
|
||||
JSON.stringify({
|
||||
type: 'session.shutdown',
|
||||
timestamp: '2026-08-07T17:56:40.591Z',
|
||||
data: {
|
||||
shutdownType: 'routine',
|
||||
sessionStartTime: 1786125395554,
|
||||
modelMetrics: {
|
||||
'claude-sonnet-5': {
|
||||
requests: { count: 2, cost: 1 },
|
||||
usage: { inputTokens: 49473, outputTokens: 100, cacheReadTokens: 24678, cacheWriteTokens: 24791, reasoningTokens: 0 },
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
])
|
||||
|
||||
// Discovery tags session-state files 'jsonl'; provenance, not the shared
|
||||
// producer value, must classify this as a CLI session.
|
||||
const calls = await collectCalls({ path: eventsPath, project: 'myproject', provider: 'copilot', sourceType: 'jsonl' })
|
||||
|
||||
// Two per-turn output calls with the REAL model — not the
|
||||
// 'copilot-anthropic-auto' bucket transcript inference would pick from the
|
||||
// toolu_ toolCallId prefix.
|
||||
const perTurn = calls.filter(c => !c.deduplicationKey.includes(':shutdown:'))
|
||||
expect(perTurn.map(c => c.outputTokens)).toEqual([81, 19])
|
||||
expect(perTurn.every(c => c.model === 'claude-sonnet-5')).toBe(true)
|
||||
|
||||
// The shutdown rollup lands: the tokens the misclassification dropped.
|
||||
const shutdown = calls.find(c => c.deduplicationKey === 'copilot:sess-cli-producer:shutdown:claude-sonnet-5:1')
|
||||
expect(shutdown).toBeDefined()
|
||||
expect(shutdown!.inputTokens).toBe(4) // 49473 − 24678 − 24791 (cache-inclusive)
|
||||
expect(shutdown!.cacheReadInputTokens).toBe(24678)
|
||||
expect(shutdown!.cacheCreationInputTokens).toBe(24791)
|
||||
expect(shutdown!.outputTokens).toBe(0) // owned by the per-turn events
|
||||
expect(shutdown!.costIsEstimated).toBe(false)
|
||||
expect(shutdown!.costUSD).toBeCloseTo(calculateCost('claude-sonnet-5', 4, 0, 24791, 24678, 0), 12)
|
||||
expect(shutdown!.costUSD).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('treats a bare (untagged) source as CLI format, not transcript', async () => {
|
||||
// Producer sniffing must not resurface for sources without a sourceType tag
|
||||
// (the pre-tagging shape): same events, same result as the tagged parse.
|
||||
const eventsPath = await createSessionDir('sess-cli-untagged', [
|
||||
JSON.stringify({
|
||||
type: 'session.start',
|
||||
timestamp: '2026-08-07T17:56:35.573Z',
|
||||
data: { sessionId: 'sess-cli-untagged', producer: 'copilot-agent', copilotVersion: '1.0.78' },
|
||||
}),
|
||||
modelChange('claude-sonnet-5'),
|
||||
userMessage('hello'),
|
||||
assistantMessage({ messageId: 'msg-1', outputTokens: 42 }),
|
||||
shutdownEvent({
|
||||
modelMetrics: {
|
||||
'claude-sonnet-5': { inputTokens: 1000, outputTokens: 42, cacheReadTokens: 600, cacheWriteTokens: 300 },
|
||||
},
|
||||
}),
|
||||
])
|
||||
|
||||
const calls = await collectCalls({ path: eventsPath, project: 'myproject', provider: 'copilot' })
|
||||
expect(calls.some(c => c.deduplicationKey.includes(':shutdown:'))).toBe(true)
|
||||
expect(calls.find(c => c.deduplicationKey.includes(':shutdown:'))!.cacheReadInputTokens).toBe(600)
|
||||
})
|
||||
|
||||
it('wires discovery through parsing: a discovered CLI session keeps its shutdown rollup', async () => {
|
||||
// The full #944 pipeline: discoverSessions must tag the session-state file
|
||||
// so that the parser it hands off to keeps the shutdown tokens.
|
||||
await createSessionDir('sess-wire', [
|
||||
JSON.stringify({
|
||||
type: 'session.start',
|
||||
timestamp: '2026-08-07T17:56:35.573Z',
|
||||
data: { sessionId: 'sess-wire', producer: 'copilot-agent', copilotVersion: '1.0.78' },
|
||||
}),
|
||||
modelChange('claude-sonnet-5'),
|
||||
userMessage('hello'),
|
||||
assistantMessage({ messageId: 'msg-1', outputTokens: 42 }),
|
||||
shutdownEvent({
|
||||
modelMetrics: {
|
||||
'claude-sonnet-5': { inputTokens: 5000, outputTokens: 42, cacheReadTokens: 3000, cacheWriteTokens: 1500 },
|
||||
},
|
||||
}),
|
||||
])
|
||||
|
||||
// Keep discovery hermetic: a real agent-traces.db on the host must not leak in.
|
||||
vi.stubEnv('CODEBURN_COPILOT_DISABLE_OTEL', '1')
|
||||
try {
|
||||
const provider = createCopilotProvider(tmpDir, '/nonexistent/vscode', '/nonexistent/global', '/nonexistent/jetbrains')
|
||||
const sessions = await provider.discoverSessions()
|
||||
expect(sessions).toHaveLength(1)
|
||||
|
||||
const calls: ParsedProviderCall[] = []
|
||||
for await (const call of provider.createSessionParser(sessions[0]!, new Set()).parse()) calls.push(call)
|
||||
|
||||
const shutdown = calls.find(c => c.deduplicationKey === 'copilot:sess-wire:shutdown:claude-sonnet-5:1')
|
||||
expect(shutdown).toBeDefined()
|
||||
expect(shutdown!.inputTokens).toBe(500) // 5000 − 3000 − 1500
|
||||
expect(shutdown!.cacheReadInputTokens).toBe(3000)
|
||||
expect(shutdown!.cacheCreationInputTokens).toBe(1500)
|
||||
} finally {
|
||||
vi.unstubAllEnvs()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('copilot provider - chatSessions parsing', () => {
|
||||
|
|
@ -811,6 +1207,9 @@ describe('copilot provider - discoverSessions', () => {
|
|||
expect(sessions).toHaveLength(2)
|
||||
expect(sessions.every(s => s.provider === 'copilot')).toBe(true)
|
||||
expect(sessions.every(s => s.path.endsWith('events.jsonl'))).toBe(true)
|
||||
// Session-state files are tagged as CLI sources — the tag (not the file's
|
||||
// producer value) decides transcript vs CLI parsing (#944).
|
||||
expect(sessions.every(s => (s as { sourceType?: string }).sourceType === 'jsonl')).toBe(true)
|
||||
})
|
||||
|
||||
it('reads project name from workspace.yaml cwd', async () => {
|
||||
|
|
@ -864,6 +1263,7 @@ describe('copilot provider - discoverSessions', () => {
|
|||
expect(sessions).toHaveLength(1)
|
||||
expect(sessions[0]!.project).toBe('myapp')
|
||||
expect(sessions[0]!.path).toContain('session-1.jsonl')
|
||||
expect((sessions[0] as { sourceType?: string }).sourceType).toBe('transcript')
|
||||
})
|
||||
|
||||
it('includes VSCodium workspaceStorage paths on all supported platforms', () => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue