From cfdaa685a0218897f1f7d930535dfdd4166b8807 Mon Sep 17 00:00:00 2001 From: Aditya Vikram Singh <247195684+avs-io@users.noreply.github.com> Date: Sun, 23 Aug 2026 17:31:12 +0530 Subject: [PATCH 1/6] fix(report): route displayed output through billableOutputTokens Cost already used the helper. Report, sessions, overview and optimize did not, so exclusive providers dropped reasoning and optimize double-counted it for claude/codex/copilot. Daily cache v28 re-derives. Closes #1115 --- CHANGELOG.md | 1 + src/daily-cache.ts | 8 +- src/day-aggregator.ts | 14 +++- src/optimize.ts | 9 +- src/overview.ts | 3 +- src/session-output.ts | 40 +++++++++ src/sessions-report.ts | 22 +---- src/usage-aggregator.ts | 5 +- tests/billable-output-1115.test.ts | 129 +++++++++++++++++++++++++++++ 9 files changed, 200 insertions(+), 31 deletions(-) create mode 100644 src/session-output.ts create mode 100644 tests/billable-output-1115.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index e21dddda..e7486694 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ - **DeepSeek Harness (`dsh`) is now a supported provider.** Reads DeepSeek's open-source agent harness from `~/.dsh/sessions` (`DSH_HOME` relocates the root), both the default zstd logs and the uncompressed `session.jsonl` variant. A `.zstd` log is a concatenation of independent zstd frames, one per write batch, so it is decoded frame by frame behind a structural frame scan and a torn trailing frame from a crashed writer is ignored rather than failing the file (needs Node 22.15+ for `zlib` zstd; below that dsh is skipped with a notice instead of counted as $0). One call per `(turn, step)`, with the step's final `assistant/message` usage superseding the streamed `assistant/chunk` sample of the same call rather than adding to it, the model taken from the message that served the step, and reasoning tokens billed at the output rate. DSH records tokens but no cost, so calls are priced from the shared tables. The events a forked session replays from its parent are skipped, since codeburn already counts the parent's own log. The session format is pinned at version 0 upstream with no compatibility implied, so a log stamped with any other version is skipped with a notice instead of read under today's assumptions. ### Changed +- **Report, sessions, overview and optimize now use `billableOutputTokens` for displayed output.** Cost already did. Exclusive providers (Grok and the rest) were under-reporting output by exactly their reasoning tokens; optimize added reasoning on top of output for the inclusive set `{claude, codex, copilot}` and double-counted it. Daily cache v28 re-derives finalized days so `report` matches the live parse. Thanks @saulcanina. (#1115) - **The discovery sweep issues its metadata syscalls concurrently.** Every dated command re-walks and re-stats every provider tree before it can decide what is already cached, and that sweep was strictly serial: one `readdir`, one `stat`, one `state.json` read at a time, per provider, one provider after another. On a 21k-file / 9-provider corpus it owned most of a warm run's wall clock while the machine sat idle waiting on the kernel. Provider discovery now runs across providers at once, and the four walks that dominate it (claude, codex, kimicode, grok) plus the Claude project-dir walk and both fingerprint passes fan out through a shared bounded-concurrency helper. Order is unchanged everywhere — each level is re-concatenated in registry/`readdir` order before anything reconciles against the cache — so what is discovered, in what sequence, is byte-identical to the serial walk. Two smaller cuts ride along: the Claude walk reads directory entries with their types so a plain file no longer costs a wasted `subagents/` probe, and the Codex result cache (a single file that can reach hundreds of MB) now shares one in-flight load between concurrent readers instead of letting each one re-read and re-parse it. Warm `codeburn today` on that corpus: 5.30s to 2.99s; a cold parse 34.7s to 31.1s. (#1104) - **A kill mid-way through a non-Claude provider phase no longer restarts that whole phase.** `scanProjectDirs` (Claude) has long taken a throttled `saveProgress` callback so a killed cold parse resumes from a warm cache; `parseProviderSources` (every other provider — codex, cursor, gemini, and the rest) did not, and only persisted at the whole-provider boundary. On a large single-provider corpus (a multi-GB codex history is the common case) an app-timeout SIGKILL, crash, or force-quit during that phase discarded everything parsed since the last provider finished, forcing the entire phase to re-parse from zero on the next run. `parseProviderSources` now takes the same callback, invoked once per source right after that source's cache entry lands (mirroring `scanProjectDirs`' placement, outside the per-file try/catch), on the same file-count/wall-clock throttle. A file only ever gets a fingerprint once it has fully parsed, so a mid-file kill can never leave a half-parsed file's entry looking complete on resume. - **Routed model ids price as the model they wrap, and an unknown vendor prefix no longer prices by blind stripping.** Token-plan and gateway spellings of the same model (`omniroute:`, `cp/`, `cline-pass/`, `cline-free/`, `cmd/`, `antigravity/`) are peeled and the remaining id is priced, so a Cline Pass or OmniRoute session shows a `~` estimate instead of $0. In exchange, `provider/model` is no longer treated as authority on its own: the leading segment is stripped only when it is a namespace the bundled pricing catalog itself uses (`anthropic/`, `openai/`, `google/`, `x-ai/`, `qwen/`, `moonshotai/`, `nousresearch/`, `xiaomi/`, `z-ai/`, and every other vendor prefix in the LiteLLM snapshot), one of the routing wrappers above, or one of the client-side spellings `kimi/`, `mimo/`, `zhipu/`, `litellm_proxy/` and `openai_like/`. Anything else stays unpriced and is reported as unpriced rather than inheriting the price of a same-named cloud row, and local-runner prefixes (`ollama/`, `lmstudio/`, `hosted_vllm/`, `local/`) are excluded on purpose so an unlisted local tag can never invent cloud spend. A user price override for the bare id wins over the catalog row a routed spelling would otherwise hit. diff --git a/src/daily-cache.ts b/src/daily-cache.ts index b59561cc..400806bc 100644 --- a/src/daily-cache.ts +++ b/src/daily-cache.ts @@ -168,8 +168,12 @@ import type { DateRange, ProjectSummary } from './types.js' // v25: #1047 activity-id pricing. v24 on main already shipped #1090. // v26: #946 copilot session-store accounting (see the top of this ladder). // v27: #1093 claude-haiku-4.5 alias (see top). -export const DAILY_CACHE_VERSION = 27 -const MIN_SUPPORTED_VERSION = 27 +// v28: #1115 report/optimize output tokens go through billableOutputTokens. +// Exclusive providers (Grok and the rest) were under-counted by reasoningTokens +// in finalized daily rows; inclusive {claude,codex,copilot} were correct there +// but optimize added reasoning again. Re-derive so report matches the live parse. +export const DAILY_CACHE_VERSION = 28 +const MIN_SUPPORTED_VERSION = 28 /// Providers whose per-day CALL COUNT means something different at /// DAILY_CACHE_VERSION 26 than it did before it. Copilot's supplementary diff --git a/src/day-aggregator.ts b/src/day-aggregator.ts index bab24617..101d6097 100644 --- a/src/day-aggregator.ts +++ b/src/day-aggregator.ts @@ -2,6 +2,7 @@ import type { DailyEntry, ProjectDayStats, ProviderDaySlice } from './daily-cach import type { PeriodData } from './menubar-json.js' import { CATEGORY_LABELS, type ProjectSummary, type TaskCategory } from './types.js' import { isBehavioralCall, isBehavioralTurn } from './behavioral-weight.js' +import { billableOutputTokens } from './models.js' function emptyEntry(date: string): DailyEntry { return { @@ -189,8 +190,13 @@ export function aggregateProjectsIntoDays(projects: ProjectSummary[], dateKeyFn: callDay.cost += call.costUSD callDay.savingsUSD += callSavings callDay.calls += callWeight + const billableOut = billableOutputTokens( + call.provider, + call.usage.outputTokens, + call.usage.reasoningTokens, + ) callDay.inputTokens += call.usage.inputTokens - callDay.outputTokens += call.usage.outputTokens + callDay.outputTokens += billableOut callDay.cacheReadTokens += call.usage.cacheReadInputTokens callDay.cacheWriteTokens += call.usage.cacheCreationInputTokens @@ -208,7 +214,7 @@ export function aggregateProjectsIntoDays(projects: ProjectSummary[], dateKeyFn: model.cost += call.costUSD model.savingsUSD += callSavings model.inputTokens += call.usage.inputTokens - model.outputTokens += call.usage.outputTokens + model.outputTokens += billableOut model.cacheReadTokens += call.usage.cacheReadInputTokens model.cacheWriteTokens += call.usage.cacheCreationInputTokens callDay.models[call.model] = model @@ -218,7 +224,7 @@ export function aggregateProjectsIntoDays(projects: ProjectSummary[], dateKeyFn: slice.cost += call.costUSD slice.savingsUSD += callSavings slice.inputTokens! += call.usage.inputTokens - slice.outputTokens! += call.usage.outputTokens + slice.outputTokens! += billableOut slice.cacheReadTokens! += call.usage.cacheReadInputTokens slice.cacheWriteTokens! += call.usage.cacheCreationInputTokens @@ -236,7 +242,7 @@ export function aggregateProjectsIntoDays(projects: ProjectSummary[], dateKeyFn: sliceModel.cost += call.costUSD sliceModel.savingsUSD += callSavings sliceModel.inputTokens += call.usage.inputTokens - sliceModel.outputTokens += call.usage.outputTokens + sliceModel.outputTokens += billableOut sliceModel.cacheReadTokens += call.usage.cacheReadInputTokens sliceModel.cacheWriteTokens += call.usage.cacheCreationInputTokens slice.models![call.model] = sliceModel diff --git a/src/optimize.ts b/src/optimize.ts index 134424c4..77de130a 100644 --- a/src/optimize.ts +++ b/src/optimize.ts @@ -16,6 +16,7 @@ import { formatTokens } from './format.js' import { recommendModelDefault, type ModelDefaultRecommendation } from './act/model-defaults.js' import { appliedFixGlyph, formatAppliedFix, type AppliedFix } from './act/types.js' import { isUserStartedSession, userStartedProjects } from './session-population.js' +import { sessionBillableOutputTokens } from './session-output.js' import { aggregateFileChurn, buildCoachingNotes, scanUserCorrections, medianTimeToFirstEditMs, worstOneShotCategory, type ReworkedFile } from './workflow-insights.js' // ============================================================================ @@ -3030,7 +3031,7 @@ export function detectRecurringContext(openers: SessionOpener[]): WasteFinding | function sessionTokenTotal(session: ProjectSummary['sessions'][number]): number { return session.totalInputTokens - + session.totalOutputTokens + + sessionBillableOutputTokens(session) + session.totalCacheReadTokens + session.totalCacheWriteTokens } @@ -3268,9 +3269,9 @@ export function findContextBloatCandidates(projects: ProjectSummary[]): ContextB for (const session of sessions) { const inputTokens = sessionEffectiveContextTokens(session) - // Reasoning is stored separately from ordinary output, but both are - // generated tokens for this detector. Reports already use their sum. - const outputTokens = session.totalOutputTokens + session.totalReasoningTokens + // Generated tokens: exclusive providers add reasoning; inclusive ones + // already folded it into totalOutputTokens (#1078 / #1115). + const outputTokens = sessionBillableOutputTokens(session) const ratio = inputTokens / Math.max(outputTokens, 1) const currentMs = new Date(session.firstTimestamp).getTime() const gapMs = previousTimestampMs !== null ? currentMs - previousTimestampMs : null diff --git a/src/overview.ts b/src/overview.ts index 52992464..064625f2 100644 --- a/src/overview.ts +++ b/src/overview.ts @@ -5,6 +5,7 @@ import { homedir } from 'os' import { CATEGORY_LABELS, type ProjectSummary, type TaskCategory } from './types.js' import { formatCost as baseCost, getCurrency } from './currency.js' import { findUnpricedModels, getShortModelName, unpricedModelHint } from './models.js' +import { sessionBillableOutputTokens } from './session-output.js' import { markEstimated } from './format.js' import { dateKey } from './day-aggregator.js' import type { DailyEntry } from './daily-cache.js' @@ -140,7 +141,7 @@ export function renderOverview( byProject.set(pname, pe) for (const s of p.sessions) { inTok += s.totalInputTokens - outTok += s.totalOutputTokens + outTok += sessionBillableOutputTokens(s) cacheR += s.totalCacheReadTokens cacheW += s.totalCacheWriteTokens for (const [m, d] of Object.entries(s.modelBreakdown)) { diff --git a/src/session-output.ts b/src/session-output.ts new file mode 100644 index 00000000..ed75f6b1 --- /dev/null +++ b/src/session-output.ts @@ -0,0 +1,40 @@ +import { billableOutputTokens } from './models.js' +import type { SessionSummary } from './types.js' + +/** First on-call provider, then a model-name fallback. Sessions are usually one provider. */ +export function inferSessionProvider(session: SessionSummary): string { + for (const turn of session.turns) { + const provider = turn.assistantCalls[0]?.provider + if (provider) return provider + } + + const models = Object.keys(session.modelBreakdown) + const model = models[0]?.toLowerCase() ?? '' + if (model.startsWith('claude')) return 'claude' + if (model.startsWith('gpt-') || model.startsWith('o1') || model.startsWith('o3') || model.startsWith('o4')) return 'codex' + if (model.startsWith('gemini')) return 'gemini' + if (model.includes('/')) return model.split('/', 1)[0] || 'unknown' + return 'unknown' +} + +/** Display/report output: exclusive providers add reasoning; inclusive ones do not. */ +export function sessionBillableOutputTokens(session: SessionSummary): number { + let fromCalls = 0 + let sawCall = false + for (const turn of session.turns) { + for (const call of turn.assistantCalls) { + sawCall = true + fromCalls += billableOutputTokens( + call.provider, + call.usage.outputTokens, + call.usage.reasoningTokens, + ) + } + } + if (sawCall) return fromCalls + return billableOutputTokens( + inferSessionProvider(session), + session.totalOutputTokens, + session.totalReasoningTokens, + ) +} diff --git a/src/sessions-report.ts b/src/sessions-report.ts index 84fb5a98..cba88b88 100644 --- a/src/sessions-report.ts +++ b/src/sessions-report.ts @@ -1,5 +1,6 @@ import { behavioralCallCount, behavioralTurnCount } from './behavioral-weight.js' import { getShortModelName } from './models.js' +import { inferSessionProvider, sessionBillableOutputTokens } from './session-output.js' import { CATEGORY_LABELS } from './types.js' import type { ProjectSummary, SessionSummary, TaskCategory } from './types.js' @@ -23,21 +24,6 @@ export type SessionRow = { durationMs: number } -function inferProvider(session: SessionSummary): string { - for (const turn of session.turns) { - const provider = turn.assistantCalls[0]?.provider - if (provider) return provider - } - - const models = Object.keys(session.modelBreakdown) - const model = models[0]?.toLowerCase() ?? '' - if (model.startsWith('claude')) return 'claude' - if (model.startsWith('gpt-') || model.startsWith('o1') || model.startsWith('o3') || model.startsWith('o4')) return 'codex' - if (model.startsWith('gemini')) return 'gemini' - if (model.includes('/')) return model.split('/', 1)[0] || 'unknown' - return 'unknown' -} - function durationMs(startedAt: string, endedAt: string): number { const duration = new Date(endedAt).getTime() - new Date(startedAt).getTime() return Number.isFinite(duration) ? duration : 0 @@ -48,14 +34,14 @@ export function aggregateSessions(projects: ProjectSummary[]): SessionRow[] { sessionId: session.sessionId, title: session.title ?? '', project: session.project || project.project, - provider: inferProvider(session), + provider: inferSessionProvider(session), models: Object.keys(session.modelBreakdown), cost: session.totalCostUSD, savingsUSD: session.totalSavingsUSD, calls: session.apiCalls, turns: behavioralTurnCount(session.turns), inputTokens: session.totalInputTokens, - outputTokens: session.totalOutputTokens, + outputTokens: sessionBillableOutputTokens(session), cacheReadTokens: session.totalCacheReadTokens, cacheWriteTokens: session.totalCacheWriteTokens, startedAt: session.firstTimestamp, @@ -374,7 +360,7 @@ const KEY_SEP = String.fromCharCode(0) function linkageProvider(session: SessionSummary): string { if (session.parentSessionId || session.agentSpawnLinks || session.spawnPrSets) return 'claude' - return inferProvider(session) + return inferSessionProvider(session) } function providerSessionKey(session: SessionSummary): string { return `${linkageProvider(session)}${KEY_SEP}${session.sessionId}` diff --git a/src/usage-aggregator.ts b/src/usage-aggregator.ts index 60a02429..c035db84 100644 --- a/src/usage-aggregator.ts +++ b/src/usage-aggregator.ts @@ -13,6 +13,7 @@ import { aggregateModels } from './models-report.js' import { scanUserCorrections, medianTimeToFirstEditMs, aggregateFileChurn, computePricingCoverage } from './workflow-insights.js' import { buildPrAttribution, aggregateByBranch } from './sessions-report.js' import { scanAndDetect } from './optimize.js' +import { sessionBillableOutputTokens } from './session-output.js' import { getDaysInRange, ensureCacheHydrated, emptyCache, BACKFILL_DAYS, toDateString, type DailyCache, type DailyEntry, type ProjectDayStats, type ProviderDaySlice } from './daily-cache.js' import { buildGranularHistory } from './granular-history.js' @@ -27,7 +28,7 @@ export function buildPeriodData(label: string, projects: ProjectSummary[]): Peri for (const sess of sessions) { inputTokens += sess.totalInputTokens - outputTokens += sess.totalOutputTokens + outputTokens += sessionBillableOutputTokens(sess) cacheReadTokens += sess.totalCacheReadTokens cacheWriteTokens += sess.totalCacheWriteTokens for (const [cat, d] of Object.entries(sess.categoryBreakdown)) { @@ -748,7 +749,7 @@ export async function buildMenubarPayloadForRange(periodInfo: PeriodInfo, opts: savingsUSD: s.totalSavingsUSD, calls: s.apiCalls, inputTokens: s.totalInputTokens, - outputTokens: s.totalOutputTokens, + outputTokens: sessionBillableOutputTokens(s), date: s.firstTimestamp?.split('T')[0] ?? '', models: Object.entries(s.modelBreakdown) .map(([name, m]) => ({ name, cost: m.costUSD, savingsUSD: m.savingsUSD })) diff --git a/tests/billable-output-1115.test.ts b/tests/billable-output-1115.test.ts new file mode 100644 index 00000000..328774e1 --- /dev/null +++ b/tests/billable-output-1115.test.ts @@ -0,0 +1,129 @@ +import { describe, expect, it } from 'vitest' + +import { aggregateProjectsIntoDays } from '../src/day-aggregator.js' +import { billableOutputTokens } from '../src/models.js' +import { findContextBloatCandidates } from '../src/optimize.js' +import { sessionBillableOutputTokens } from '../src/session-output.js' +import { aggregateSessions } from '../src/sessions-report.js' +import { buildPeriodData } from '../src/usage-aggregator.js' +import type { ProjectSummary, SessionSummary } from '../src/types.js' + +function makeCall(provider: string, outputTokens: number, reasoningTokens: number) { + return { + provider, + model: provider === 'codex' ? 'gpt-5.4' : 'grok-4', + usage: { + inputTokens: 0, + outputTokens, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens, + webSearchRequests: 0, + }, + costUSD: 0, + tools: [], + mcpTools: [], + skills: [], + hasAgentSpawn: false, + hasPlanMode: false, + speed: 'standard' as const, + timestamp: '2026-08-01T12:00:00Z', + bashCommands: [], + deduplicationKey: `${provider}-${outputTokens}-${reasoningTokens}`, + } +} + +function makeSession(provider: string, outputTokens: number, reasoningTokens: number): SessionSummary { + const call = makeCall(provider, outputTokens, reasoningTokens) + return { + sessionId: `${provider}-s`, + project: 'p', + firstTimestamp: call.timestamp, + lastTimestamp: call.timestamp, + totalCostUSD: 0, + totalSavingsUSD: 0, + totalInputTokens: 0, + totalOutputTokens: outputTokens, + totalReasoningTokens: reasoningTokens, + totalCacheReadTokens: 0, + totalCacheWriteTokens: 0, + apiCalls: 1, + turns: [{ + userMessage: 'x', + timestamp: call.timestamp, + sessionId: `${provider}-s`, + category: 'coding', + retries: 0, + hasEdits: true, + assistantCalls: [call], + }], + modelBreakdown: {}, + toolBreakdown: {}, + mcpBreakdown: {}, + bashBreakdown: {}, + categoryBreakdown: {} as never, + skillBreakdown: {} as never, + } as SessionSummary +} + +function makeProject(session: SessionSummary): ProjectSummary { + return { + project: 'p', + projectPath: '/p', + totalCostUSD: 0, + totalSavingsUSD: 0, + totalProxiedCostUSD: 0, + totalApiCalls: 1, + sessions: [session], + } as ProjectSummary +} + +describe('#1115 billableOutputTokens on report/optimize totals', () => { + it('exclusive grok: 10 output + 3 reasoning = 13', () => { + expect(billableOutputTokens('grok', 10, 3)).toBe(13) + expect(sessionBillableOutputTokens(makeSession('grok', 10, 3))).toBe(13) + }) + + it('inclusive codex: 10 output already contains reasoning = 10', () => { + expect(billableOutputTokens('codex', 10, 3)).toBe(10) + expect(sessionBillableOutputTokens(makeSession('codex', 10, 3))).toBe(10) + }) + + it('day-aggregator uses billable output per call', () => { + const grokDays = aggregateProjectsIntoDays([makeProject(makeSession('grok', 10, 3))]) + expect(grokDays[0]!.outputTokens).toBe(13) + expect(grokDays[0]!.providers.grok!.outputTokens).toBe(13) + + const codexDays = aggregateProjectsIntoDays([makeProject(makeSession('codex', 10, 3))]) + expect(codexDays[0]!.outputTokens).toBe(10) + expect(codexDays[0]!.providers.codex!.outputTokens).toBe(10) + }) + + it('sessions report and period data match the helper', () => { + const grok = makeProject(makeSession('grok', 10, 3)) + const codex = makeProject(makeSession('codex', 10, 3)) + expect(aggregateSessions([grok])[0]!.outputTokens).toBe(13) + expect(aggregateSessions([codex])[0]!.outputTokens).toBe(10) + expect(buildPeriodData('t', [grok]).outputTokens).toBe(13) + expect(buildPeriodData('t', [codex]).outputTokens).toBe(10) + }) + + it('optimize context-bloat denominator does not double-count inclusive reasoning', () => { + const inclusive = makeSession('codex', 100_000, 50_000) + inclusive.totalInputTokens = 2_000_000 + const exclusive = makeSession('grok', 100_000, 50_000) + exclusive.totalInputTokens = 2_000_000 + + const inc = findContextBloatCandidates([makeProject(inclusive)]) + const exc = findContextBloatCandidates([makeProject(exclusive)]) + // ratio = input / billableOut. Inclusive 2e6/1e5 = 20; exclusive 2e6/1.5e5 ≈ 13.3 + // Both clear CONTEXT_BLOAT_MIN_RATIO if that threshold is below 13. + if (inc.length && exc.length) { + expect(inc[0]!.growthRatio === null || typeof inc[0]!.growthRatio === 'number').toBe(true) + } + // Direct contract: helper is what the detector uses. + expect(sessionBillableOutputTokens(inclusive)).toBe(100_000) + expect(sessionBillableOutputTokens(exclusive)).toBe(150_000) + }) +}) From f816c5e3de541f1ff7ffde9eaa55df5aa87c8b4f Mon Sep 17 00:00:00 2001 From: Aditya Vikram Singh <247195684+avs-io@users.noreply.github.com> Date: Sun, 23 Aug 2026 18:18:57 +0530 Subject: [PATCH 2/6] fix(report): bill remaining displayed output per call Compare, export, report JSON, overview Top-model/By-tool/day, and menubar localModelSavings still summed raw output. Session totals stay defined on aggregate-only and stub calls. Upgrade verifier expects daily-cache.v28.json. --- CHANGELOG.md | 2 +- scripts/upgrade-path/run.mjs | 2 +- src/compare-stats.ts | 3 +- src/export.ts | 30 +++-- src/main.ts | 13 ++- src/overview.ts | 11 +- src/session-output.ts | 48 +++++--- src/usage-aggregator.ts | 4 +- tests/billable-output-1115.test.ts | 88 ++++++++++++++- tests/menubar-json.test.ts | 25 ++++- .../usage-aggregator-billable-output.test.ts | 103 ++++++++++++++++++ 11 files changed, 292 insertions(+), 37 deletions(-) create mode 100644 tests/usage-aggregator-billable-output.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index e7486694..73aacfe3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ - **DeepSeek Harness (`dsh`) is now a supported provider.** Reads DeepSeek's open-source agent harness from `~/.dsh/sessions` (`DSH_HOME` relocates the root), both the default zstd logs and the uncompressed `session.jsonl` variant. A `.zstd` log is a concatenation of independent zstd frames, one per write batch, so it is decoded frame by frame behind a structural frame scan and a torn trailing frame from a crashed writer is ignored rather than failing the file (needs Node 22.15+ for `zlib` zstd; below that dsh is skipped with a notice instead of counted as $0). One call per `(turn, step)`, with the step's final `assistant/message` usage superseding the streamed `assistant/chunk` sample of the same call rather than adding to it, the model taken from the message that served the step, and reasoning tokens billed at the output rate. DSH records tokens but no cost, so calls are priced from the shared tables. The events a forked session replays from its parent are skipped, since codeburn already counts the parent's own log. The session format is pinned at version 0 upstream with no compatibility implied, so a log stamped with any other version is skipped with a notice instead of read under today's assumptions. ### Changed -- **Report, sessions, overview and optimize now use `billableOutputTokens` for displayed output.** Cost already did. Exclusive providers (Grok and the rest) were under-reporting output by exactly their reasoning tokens; optimize added reasoning on top of output for the inclusive set `{claude, codex, copilot}` and double-counted it. Daily cache v28 re-derives finalized days so `report` matches the live parse. Thanks @saulcanina. (#1115) +- **Report, sessions, overview, compare, export, report JSON and menubar `localModelSavings` now use `billableOutputTokens` for displayed output.** Cost already did. Exclusive providers (Grok and the rest) were under-reporting output by exactly their reasoning tokens; optimize added reasoning on top of output for the inclusive set `{claude, codex, copilot}` and double-counted it. Displayed aggregates now bill per call while the provider is known. Daily cache v28 re-derives finalized days so `report` matches the live parse. Thanks @saulcanina. (#1115) - **The discovery sweep issues its metadata syscalls concurrently.** Every dated command re-walks and re-stats every provider tree before it can decide what is already cached, and that sweep was strictly serial: one `readdir`, one `stat`, one `state.json` read at a time, per provider, one provider after another. On a 21k-file / 9-provider corpus it owned most of a warm run's wall clock while the machine sat idle waiting on the kernel. Provider discovery now runs across providers at once, and the four walks that dominate it (claude, codex, kimicode, grok) plus the Claude project-dir walk and both fingerprint passes fan out through a shared bounded-concurrency helper. Order is unchanged everywhere — each level is re-concatenated in registry/`readdir` order before anything reconciles against the cache — so what is discovered, in what sequence, is byte-identical to the serial walk. Two smaller cuts ride along: the Claude walk reads directory entries with their types so a plain file no longer costs a wasted `subagents/` probe, and the Codex result cache (a single file that can reach hundreds of MB) now shares one in-flight load between concurrent readers instead of letting each one re-read and re-parse it. Warm `codeburn today` on that corpus: 5.30s to 2.99s; a cold parse 34.7s to 31.1s. (#1104) - **A kill mid-way through a non-Claude provider phase no longer restarts that whole phase.** `scanProjectDirs` (Claude) has long taken a throttled `saveProgress` callback so a killed cold parse resumes from a warm cache; `parseProviderSources` (every other provider — codex, cursor, gemini, and the rest) did not, and only persisted at the whole-provider boundary. On a large single-provider corpus (a multi-GB codex history is the common case) an app-timeout SIGKILL, crash, or force-quit during that phase discarded everything parsed since the last provider finished, forcing the entire phase to re-parse from zero on the next run. `parseProviderSources` now takes the same callback, invoked once per source right after that source's cache entry lands (mirroring `scanProjectDirs`' placement, outside the per-file try/catch), on the same file-count/wall-clock throttle. A file only ever gets a fingerprint once it has fully parsed, so a mid-file kill can never leave a half-parsed file's entry looking complete on resume. - **Routed model ids price as the model they wrap, and an unknown vendor prefix no longer prices by blind stripping.** Token-plan and gateway spellings of the same model (`omniroute:`, `cp/`, `cline-pass/`, `cline-free/`, `cmd/`, `antigravity/`) are peeled and the remaining id is priced, so a Cline Pass or OmniRoute session shows a `~` estimate instead of $0. In exchange, `provider/model` is no longer treated as authority on its own: the leading segment is stripped only when it is a namespace the bundled pricing catalog itself uses (`anthropic/`, `openai/`, `google/`, `x-ai/`, `qwen/`, `moonshotai/`, `nousresearch/`, `xiaomi/`, `z-ai/`, and every other vendor prefix in the LiteLLM snapshot), one of the routing wrappers above, or one of the client-side spellings `kimi/`, `mimo/`, `zhipu/`, `litellm_proxy/` and `openai_like/`. Anything else stays unpriced and is reported as unpriced rather than inheriting the price of a same-named cloud row, and local-runner prefixes (`ollama/`, `lmstudio/`, `hosted_vllm/`, `local/`) are excluded on purpose so an unlisted local tag can never invent cloud spend. A user price override for the bare id wins over the catalog row a routed spelling would otherwise hit. diff --git a/scripts/upgrade-path/run.mjs b/scripts/upgrade-path/run.mjs index ac848043..c7af2816 100644 --- a/scripts/upgrade-path/run.mjs +++ b/scripts/upgrade-path/run.mjs @@ -33,7 +33,7 @@ const WORK = process.env['UPGRADE_PATH_WORK'] || join(tmpdir(), 'codeburn upgrad const OLD_SESSION_CACHE = 'session-cache.v7.json' const OLD_DAILY_CACHE = 'daily-cache.v17.json' const NEW_SESSION_CACHE_DIR = 'session-cache.v9' -const NEW_DAILY_CACHE = 'daily-cache.v27.json' +const NEW_DAILY_CACHE = 'daily-cache.v28.json' const HOME = join(WORK, 'user home') const PAYLOADS = join(WORK, 'payloads') diff --git a/src/compare-stats.ts b/src/compare-stats.ts index 5faabf73..2b4b9931 100644 --- a/src/compare-stats.ts +++ b/src/compare-stats.ts @@ -4,6 +4,7 @@ import { join } from 'path' import type { ClassifiedTurn, ProjectSummary } from './types.js' import { isBehavioralCall } from './behavioral-weight.js' import { getShortModelName } from './models.js' +import { callBillableOutputTokens } from './session-output.js' const PLANNING_TOOLS = new Set(['TaskCreate', 'TaskUpdate', 'TodoWrite', 'EnterPlanMode', 'ExitPlanMode']) @@ -67,7 +68,7 @@ export function aggregateModelStats(projects: ProjectSummary[]): ModelStats[] { const cs = call.model === primaryModel ? ms : ensure(call.model) if (isBehavioralCall(call)) cs.calls++ cs.cost += call.costUSD - cs.outputTokens += call.usage.outputTokens + cs.outputTokens += callBillableOutputTokens(call) cs.inputTokens += call.usage.inputTokens cs.cacheReadTokens += call.usage.cacheReadInputTokens cs.cacheWriteTokens += call.usage.cacheCreationInputTokens diff --git a/src/export.ts b/src/export.ts index 733dff72..ba294fa6 100644 --- a/src/export.ts +++ b/src/export.ts @@ -6,6 +6,7 @@ import { getCurrency, convertCost, roundForActiveCurrency } from './currency.js' import { dateKey } from './day-aggregator.js' import { behavioralTurnCount, isBehavioralCall } from './behavioral-weight.js' import { aggregateModelEfficiency } from './model-efficiency.js' +import { callBillableOutputTokens } from './session-output.js' function escCsv(s: string): string { const sanitized = /^[\t\r=+\-@]/.test(s) ? `'${s}` : s @@ -65,7 +66,7 @@ function buildDailyRows(projects: ProjectSummary[], period: string): Row[] { // so daily.csv call counts must reconcile with summary.csv. if (isBehavioralCall(call)) daily[day].calls++ daily[day].input += call.usage.inputTokens - daily[day].output += call.usage.outputTokens + daily[day].output += callBillableOutputTokens(call) daily[day].cacheRead += call.usage.cacheReadInputTokens daily[day].cacheWrite += call.usage.cacheCreationInputTokens } @@ -149,17 +150,28 @@ function buildActivityRows(projects: ProjectSummary[], period: string): Row[] { function buildModelRows(projects: ProjectSummary[], period: string): Row[] { const modelTotals: Record = {} const modelEfficiency = aggregateModelEfficiency(projects) + const ensure = (model: string) => { + if (!modelTotals[model]) modelTotals[model] = { calls: 0, cost: 0, savings: 0, input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } + return modelTotals[model] + } for (const project of projects) { for (const session of project.sessions) { for (const [model, d] of Object.entries(session.modelBreakdown)) { - if (!modelTotals[model]) modelTotals[model] = { calls: 0, cost: 0, savings: 0, input: 0, output: 0, cacheRead: 0, cacheWrite: 0 } - modelTotals[model].calls += d.calls - modelTotals[model].cost += d.costUSD - modelTotals[model].savings += d.savingsUSD - modelTotals[model].input += d.tokens.inputTokens - modelTotals[model].output += d.tokens.outputTokens - modelTotals[model].cacheRead += d.tokens.cacheReadInputTokens ?? 0 - modelTotals[model].cacheWrite += d.tokens.cacheCreationInputTokens ?? 0 + const acc = ensure(model) + acc.calls += d.calls + acc.cost += d.costUSD + acc.savings += d.savingsUSD + acc.input += d.tokens.inputTokens + acc.cacheRead += d.tokens.cacheReadInputTokens ?? 0 + acc.cacheWrite += d.tokens.cacheCreationInputTokens ?? 0 + } + // Output must be billed per call while provider identity is still known. + // Grouped modelBreakdown tokens cannot distinguish exclusive vs inclusive. + for (const turn of session.turns) { + for (const call of turn.assistantCalls) { + if (!call.model) continue + ensure(call.model).output += callBillableOutputTokens(call) + } } } } diff --git a/src/main.ts b/src/main.ts index 61efb9ba..7cc43058 100644 --- a/src/main.ts +++ b/src/main.ts @@ -10,6 +10,7 @@ import { convertCost, formatCost } from './currency.js' import { renderStatusBar } from './format.js' import { toDateString } from './daily-cache.js' import { dateKey } from './day-aggregator.js' +import { callBillableOutputTokens } from './session-output.js' import { isBehavioralCall } from './behavioral-weight.js' import { CATEGORY_LABELS, type DateRange, type ProjectSummary, type TaskCategory } from './types.js' import type { AppliedFix } from './act/types.js' @@ -542,10 +543,20 @@ function buildJsonReport(projects: ProjectSummary[], period: string, periodKey: modelMap[model].savings += d.savingsUSD modelMap[model].estimatedCost += d.estimatedCostUSD ?? 0 modelMap[model].inputTokens += d.tokens.inputTokens - modelMap[model].outputTokens += d.tokens.outputTokens modelMap[model].cacheReadTokens += d.tokens.cacheReadInputTokens modelMap[model].cacheWriteTokens += d.tokens.cacheCreationInputTokens } + // Output must be billed per call while provider identity is still known. + // Inferring after modelBreakdown grouping is not safe for mixed-provider data. + for (const turn of sess.turns) { + for (const call of turn.assistantCalls) { + if (!call.model) continue + if (!modelMap[call.model]) { + modelMap[call.model] = { calls: 0, cost: 0, savings: 0, estimatedCost: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, baselineModel: '' } + } + modelMap[call.model].outputTokens += callBillableOutputTokens(call) + } + } } // Pull the active baseline model name out of the savings config so the // report can show what the local calls were mapped against without diff --git a/src/overview.ts b/src/overview.ts index 064625f2..ed3c09c7 100644 --- a/src/overview.ts +++ b/src/overview.ts @@ -5,7 +5,7 @@ import { homedir } from 'os' import { CATEGORY_LABELS, type ProjectSummary, type TaskCategory } from './types.js' import { formatCost as baseCost, getCurrency } from './currency.js' import { findUnpricedModels, getShortModelName, unpricedModelHint } from './models.js' -import { sessionBillableOutputTokens } from './session-output.js' +import { callBillableOutputTokens, sessionBillableOutputTokens } from './session-output.js' import { markEstimated } from './format.js' import { dateKey } from './day-aggregator.js' import type { DailyEntry } from './daily-cache.js' @@ -149,7 +149,7 @@ export function renderOverview( e.cost += d.costUSD e.calls += d.calls e.estimatedCost += d.estimatedCostUSD ?? 0 - e.tokens += d.tokens.inputTokens + d.tokens.outputTokens + d.tokens.cacheReadInputTokens + d.tokens.cacheCreationInputTokens + e.tokens += d.tokens.inputTokens + d.tokens.cacheReadInputTokens + d.tokens.cacheCreationInputTokens byModel.set(m, e) } for (const [cat, d] of Object.entries(s.categoryBreakdown)) { @@ -164,7 +164,12 @@ export function renderOverview( for (const t of s.turns) { const day = dateKey(t.timestamp || t.assistantCalls[0]?.timestamp || '') for (const call of t.assistantCalls) { - const tk = call.usage.inputTokens + call.usage.outputTokens + call.usage.cacheReadInputTokens + call.usage.cacheCreationInputTokens + const usage = call.usage + const billableOut = callBillableOutputTokens(call) + const tk = (usage?.inputTokens ?? 0) + billableOut + (usage?.cacheReadInputTokens ?? 0) + (usage?.cacheCreationInputTokens ?? 0) + const me = byModel.get(call.model) ?? { cost: 0, calls: 0, tokens: 0, estimatedCost: 0 } + me.tokens += billableOut + byModel.set(call.model, me) const pv = byProvider.get(call.provider) ?? { cost: 0, tokens: 0 } pv.cost += call.costUSD pv.tokens += tk diff --git a/src/session-output.ts b/src/session-output.ts index ed75f6b1..e7c85e6b 100644 --- a/src/session-output.ts +++ b/src/session-output.ts @@ -1,14 +1,24 @@ import { billableOutputTokens } from './models.js' import type { SessionSummary } from './types.js' +type UsageLike = { + outputTokens?: number + reasoningTokens?: number +} + +type CallLike = { + provider?: string + usage?: UsageLike +} + /** First on-call provider, then a model-name fallback. Sessions are usually one provider. */ export function inferSessionProvider(session: SessionSummary): string { - for (const turn of session.turns) { - const provider = turn.assistantCalls[0]?.provider + for (const turn of session.turns ?? []) { + const provider = turn.assistantCalls?.[0]?.provider if (provider) return provider } - const models = Object.keys(session.modelBreakdown) + const models = Object.keys(session.modelBreakdown ?? {}) const model = models[0]?.toLowerCase() ?? '' if (model.startsWith('claude')) return 'claude' if (model.startsWith('gpt-') || model.startsWith('o1') || model.startsWith('o3') || model.startsWith('o4')) return 'codex' @@ -17,24 +27,32 @@ export function inferSessionProvider(session: SessionSummary): string { return 'unknown' } +/** Per-call displayed output. Missing usage/fields are 0 so aggregate-only and stub calls cannot crash. */ +export function callBillableOutputTokens(call: CallLike): number { + const usage = call.usage + if (!usage) return 0 + return billableOutputTokens( + call.provider ?? 'unknown', + usage.outputTokens ?? 0, + usage.reasoningTokens ?? 0, + ) +} + /** Display/report output: exclusive providers add reasoning; inclusive ones do not. */ export function sessionBillableOutputTokens(session: SessionSummary): number { let fromCalls = 0 - let sawCall = false - for (const turn of session.turns) { - for (const call of turn.assistantCalls) { - sawCall = true - fromCalls += billableOutputTokens( - call.provider, - call.usage.outputTokens, - call.usage.reasoningTokens, - ) + let sawUsage = false + for (const turn of session.turns ?? []) { + for (const call of turn.assistantCalls ?? []) { + if (!call.usage) continue + sawUsage = true + fromCalls += callBillableOutputTokens(call) } } - if (sawCall) return fromCalls + if (sawUsage) return fromCalls return billableOutputTokens( inferSessionProvider(session), - session.totalOutputTokens, - session.totalReasoningTokens, + session.totalOutputTokens ?? 0, + session.totalReasoningTokens ?? 0, ) } diff --git a/src/usage-aggregator.ts b/src/usage-aggregator.ts index c035db84..c7dd7d2b 100644 --- a/src/usage-aggregator.ts +++ b/src/usage-aggregator.ts @@ -13,7 +13,7 @@ import { aggregateModels } from './models-report.js' import { scanUserCorrections, medianTimeToFirstEditMs, aggregateFileChurn, computePricingCoverage } from './workflow-insights.js' import { buildPrAttribution, aggregateByBranch } from './sessions-report.js' import { scanAndDetect } from './optimize.js' -import { sessionBillableOutputTokens } from './session-output.js' +import { callBillableOutputTokens, sessionBillableOutputTokens } from './session-output.js' import { getDaysInRange, ensureCacheHydrated, emptyCache, BACKFILL_DAYS, toDateString, type DailyCache, type DailyEntry, type ProjectDayStats, type ProviderDaySlice } from './daily-cache.js' import { buildGranularHistory } from './granular-history.js' @@ -926,7 +926,7 @@ export async function buildMenubarPayloadForRange(periodInfo: PeriodInfo, opts: acc.savingsUSD += call.savingsUSD acc.baselineModel = acc.baselineModel || (call.savingsBaselineModel ?? '') acc.inputTokens += call.usage.inputTokens - acc.outputTokens += call.usage.outputTokens + acc.outputTokens += callBillableOutputTokens(call) savingsByModel.set(modelKey, acc) const provAcc = savingsByProvider.get(call.provider) ?? { calls: 0, savingsUSD: 0 } provAcc.calls += callWeight diff --git a/tests/billable-output-1115.test.ts b/tests/billable-output-1115.test.ts index 328774e1..ea89b6d2 100644 --- a/tests/billable-output-1115.test.ts +++ b/tests/billable-output-1115.test.ts @@ -1,9 +1,15 @@ +import { mkdtemp, readFile, rm } from 'fs/promises' +import { tmpdir } from 'os' +import { join } from 'path' import { describe, expect, it } from 'vitest' +import { aggregateModelStats, computeComparison } from '../src/compare-stats.js' import { aggregateProjectsIntoDays } from '../src/day-aggregator.js' +import { exportJson } from '../src/export.js' import { billableOutputTokens } from '../src/models.js' import { findContextBloatCandidates } from '../src/optimize.js' -import { sessionBillableOutputTokens } from '../src/session-output.js' +import { renderOverview } from '../src/overview.js' +import { callBillableOutputTokens, sessionBillableOutputTokens } from '../src/session-output.js' import { aggregateSessions } from '../src/sessions-report.js' import { buildPeriodData } from '../src/usage-aggregator.js' import type { ProjectSummary, SessionSummary } from '../src/types.js' @@ -31,6 +37,8 @@ function makeCall(provider: string, outputTokens: number, reasoningTokens: numbe timestamp: '2026-08-01T12:00:00Z', bashCommands: [], deduplicationKey: `${provider}-${outputTokens}-${reasoningTokens}`, + savingsUSD: 1, + savingsBaselineModel: 'gpt-4o', } } @@ -58,13 +66,20 @@ function makeSession(provider: string, outputTokens: number, reasoningTokens: nu hasEdits: true, assistantCalls: [call], }], - modelBreakdown: {}, + modelBreakdown: { + [call.model]: { + calls: 1, + costUSD: 0, + savingsUSD: 0, + tokens: call.usage, + }, + }, toolBreakdown: {}, mcpBreakdown: {}, bashBreakdown: {}, categoryBreakdown: {} as never, skillBreakdown: {} as never, - } as SessionSummary + } as unknown as SessionSummary } function makeProject(session: SessionSummary): ProjectSummary { @@ -126,4 +141,71 @@ describe('#1115 billableOutputTokens on report/optimize totals', () => { expect(sessionBillableOutputTokens(inclusive)).toBe(100_000) expect(sessionBillableOutputTokens(exclusive)).toBe(150_000) }) + + it('compare-stats Output tok/call uses per-call billable output', () => { + const grok = aggregateModelStats([makeProject(makeSession('grok', 10, 3))]) + const codex = aggregateModelStats([makeProject(makeSession('codex', 10, 3))]) + expect(grok[0]!.outputTokens).toBe(13) + expect(codex[0]!.outputTokens).toBe(10) + const rows = computeComparison(grok[0]!, codex[0]!) + const outputRow = rows.find(r => r.label === 'Output tok / call')! + expect(outputRow.valueA).toBe(13) + expect(outputRow.valueB).toBe(10) + }) + + it('export daily/model Output Tokens use per-call billable output', async () => { + const dir = await mkdtemp(join(tmpdir(), 'cb-1115-export-')) + try { + const path = await exportJson([ + { label: '30 Days', projects: [makeProject(makeSession('grok', 10, 3)), makeProject(makeSession('codex', 10, 3))] }, + ], join(dir, 'out.json')) + const data = JSON.parse(await readFile(path, 'utf-8')) as { + periods: Array<{ daily: Array<{ 'Output Tokens': number }>; models: Array<{ Model: string; 'Output Tokens': number }> }> + records: Array<{ outputTokens: number; reasoningTokens: number }> + } + expect(data.periods[0]!.daily[0]!['Output Tokens']).toBe(23) + const grokRow = data.periods[0]!.models.find(r => r.Model === 'grok-4')! + const codexRow = data.periods[0]!.models.find(r => r.Model === 'gpt-5.4')! + expect(grokRow['Output Tokens']).toBe(13) + expect(codexRow['Output Tokens']).toBe(10) + // Record-level export stays raw columns, not the billable sum. + expect(data.records.map(r => r.outputTokens).sort()).toEqual([10, 10]) + expect(data.records.map(r => r.reasoningTokens).sort()).toEqual([3, 3]) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) + + it('overview Top-model / By-tool / day totals use per-call billable output', () => { + const out = renderOverview( + [makeProject(makeSession('grok', 10, 3)), makeProject(makeSession('codex', 10, 3))], + { label: 'August 2026', color: false }, + ) + // Exclusive 13 + inclusive 10. Combined token totals (input/cache are 0). + expect(out).toContain('13') + expect(out).toContain('10') + expect(out).toMatch(/Output[\s\S]*13/) + expect(out).toContain('grok') + expect(out).toContain('codex') + }) + + it('session helper does not crash on aggregate-only or minimal calls', () => { + const aggregate = makeSession('grok', 10, 3) + aggregate.turns = [] + expect(sessionBillableOutputTokens(aggregate)).toBe(13) + + const noReasoning = makeSession('grok', 10, 3) + noReasoning.turns = [] + delete (noReasoning as { totalReasoningTokens?: number }).totalReasoningTokens + const noReasoningOut = sessionBillableOutputTokens(noReasoning) + expect(Number.isFinite(noReasoningOut)).toBe(true) + expect(noReasoningOut).toBe(10) + + const stub = makeSession('codex', 10, 3) + stub.turns[0]!.assistantCalls = [{ costUSD: 1, tools: [], bashCommands: [], timestamp: stub.firstTimestamp } as never] + expect(sessionBillableOutputTokens(stub)).toBe(10) + + expect(callBillableOutputTokens({} as never)).toBe(0) + expect(callBillableOutputTokens({ provider: 'grok', usage: { outputTokens: 10, reasoningTokens: 3 } })).toBe(13) + }) }) diff --git a/tests/menubar-json.test.ts b/tests/menubar-json.test.ts index 393482a2..c499bee3 100644 --- a/tests/menubar-json.test.ts +++ b/tests/menubar-json.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' -import { buildMenubarPayload, type CombinedUsage, type PeriodData, type ProviderCost } from '../src/menubar-json.js' +import { buildMenubarPayload, type CombinedUsage, type LocalModelSavings, type PeriodData, type ProviderCost } from '../src/menubar-json.js' import type { OptimizeResult } from '../src/optimize.js' function emptyPeriod(label: string): PeriodData { @@ -423,4 +423,27 @@ describe('buildMenubarPayload', () => { const payload = buildMenubarPayload(emptyPeriod('Today'), [], null) expect(payload.stale).toBeUndefined() }) + + it('passes localModelSavings.byModel.outputTokens through as billable output', () => { + // usage-aggregator writes billableOutputTokens into this field while the + // provider is still known. Exclusive 10+3 → 13; inclusive 10+3 → 10. + const localModelSavings: LocalModelSavings = { + totalUSD: 2, + calls: 2, + byModel: [ + { name: 'Grok 4', calls: 1, actualUSD: 0, savingsUSD: 1, baselineModel: 'gpt-4o', inputTokens: 0, outputTokens: 13 }, + { name: 'GPT-5.4', calls: 1, actualUSD: 0, savingsUSD: 1, baselineModel: 'gpt-4o', inputTokens: 0, outputTokens: 10 }, + ], + byProvider: [ + { name: 'grok', calls: 1, savingsUSD: 1 }, + { name: 'codex', calls: 1, savingsUSD: 1 }, + ], + } + const period: PeriodData = { ...emptyPeriod('Today'), outputTokens: 23 } + const payload = buildMenubarPayload(period, [], null, undefined, undefined, undefined, { localModelSavings }) + expect(payload.current.outputTokens).toBe(23) + expect(payload.current.localModelSavings.byModel).toEqual(localModelSavings.byModel) + expect(payload.current.localModelSavings.byModel[0]!.outputTokens).toBe(13) + expect(payload.current.localModelSavings.byModel[1]!.outputTokens).toBe(10) + }) }) diff --git a/tests/usage-aggregator-billable-output.test.ts b/tests/usage-aggregator-billable-output.test.ts new file mode 100644 index 00000000..c7627572 --- /dev/null +++ b/tests/usage-aggregator-billable-output.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it, beforeAll, vi } from 'vitest' + +import { buildMenubarPayloadForRange } from '../src/usage-aggregator.js' +import { getDateRange } from '../src/cli-date.js' +import { loadPricing } from '../src/models.js' +import type { ProjectSummary } from '../src/types.js' + +const ts = new Date().toISOString() + +function makeCall(provider: string, outputTokens: number, reasoningTokens: number) { + return { + provider, + model: provider === 'codex' ? 'gpt-5.4' : 'grok-4', + usage: { + inputTokens: 0, + outputTokens, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens, + webSearchRequests: 0, + }, + costUSD: 0, + savingsUSD: 1, + savingsBaselineModel: 'gpt-4o', + tools: [], + mcpTools: [], + skills: [], + subagentTypes: [], + hasAgentSpawn: false, + hasPlanMode: false, + speed: 'standard' as const, + timestamp: ts, + bashCommands: [], + deduplicationKey: `${provider}-sav`, + } +} + +const emptyCat = { turns: 0, costUSD: 0, savingsUSD: 0, retries: 0, editTurns: 0, oneShotTurns: 0 } + +function fixtureProjects(): ProjectSummary[] { + const grok = makeCall('grok', 10, 3) + const codex = makeCall('codex', 10, 3) + return [{ + project: 'proj', + projectPath: 'proj', + sessions: [{ + sessionId: 'sess-billable', + project: 'proj', + firstTimestamp: ts, + lastTimestamp: ts, + totalCostUSD: 0, + totalSavingsUSD: 2, + totalInputTokens: 0, + totalOutputTokens: 20, + totalReasoningTokens: 6, + totalCacheReadTokens: 0, + totalCacheWriteTokens: 0, + apiCalls: 2, + turns: [{ + userMessage: 'hi', + timestamp: ts, + sessionId: 'sess-billable', + category: 'coding', + retries: 0, + hasEdits: false, + assistantCalls: [grok, codex], + }], + modelBreakdown: {}, + toolBreakdown: {}, + mcpBreakdown: {}, + bashBreakdown: {}, + subagentBreakdown: {}, + categoryBreakdown: { coding: { ...emptyCat, turns: 1, savingsUSD: 2 } }, + skillBreakdown: {}, + }], + totalCostUSD: 0, + totalSavingsUSD: 2, + totalApiCalls: 2, + }] as unknown as ProjectSummary[] +} + +vi.mock('../src/parser.js', async (importOriginal) => { + const mod = await importOriginal() + return { ...mod, parseAllSessions: vi.fn(async () => fixtureProjects()) } +}) + +describe('buildMenubarPayloadForRange: localModelSavings billable output', () => { + beforeAll(async () => { + await loadPricing() + }) + + it('writes per-call billableOutputTokens into byModel.outputTokens', async () => { + const payload = await buildMenubarPayloadForRange(getDateRange('today'), { provider: 'all', optimize: false }) + const byModel = payload.current.localModelSavings.byModel + const grok = byModel.find(m => m.outputTokens === 13) + const codex = byModel.find(m => m.outputTokens === 10) + expect(grok).toBeDefined() + expect(codex).toBeDefined() + expect(grok!.outputTokens).toBe(13) + expect(codex!.outputTokens).toBe(10) + }) +}) From ece5548126d29c90dc25bad942d0f6112577c6c2 Mon Sep 17 00:00:00 2001 From: Aditya Vikram Singh <247195684+avs-io@users.noreply.github.com> Date: Sun, 23 Aug 2026 18:43:37 +0530 Subject: [PATCH 3/6] fix(report): join model output on modelBreakdown keys Per-call billable output was written under raw call.model while parser buckets use getShortModelName. Display aliases split into a zero-output named row and a phantom raw-id row. Aggregate-only sessions fall back to each existing bucket. --- CHANGELOG.md | 2 +- src/export.ts | 11 ++--- src/main.ts | 15 +++--- src/overview.ts | 12 +++-- src/session-output.ts | 39 +++++++++++++++- tests/billable-output-1115.test.ts | 73 ++++++++++++++++++++++++++++-- 6 files changed, 125 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73aacfe3..3007475d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ - **DeepSeek Harness (`dsh`) is now a supported provider.** Reads DeepSeek's open-source agent harness from `~/.dsh/sessions` (`DSH_HOME` relocates the root), both the default zstd logs and the uncompressed `session.jsonl` variant. A `.zstd` log is a concatenation of independent zstd frames, one per write batch, so it is decoded frame by frame behind a structural frame scan and a torn trailing frame from a crashed writer is ignored rather than failing the file (needs Node 22.15+ for `zlib` zstd; below that dsh is skipped with a notice instead of counted as $0). One call per `(turn, step)`, with the step's final `assistant/message` usage superseding the streamed `assistant/chunk` sample of the same call rather than adding to it, the model taken from the message that served the step, and reasoning tokens billed at the output rate. DSH records tokens but no cost, so calls are priced from the shared tables. The events a forked session replays from its parent are skipped, since codeburn already counts the parent's own log. The session format is pinned at version 0 upstream with no compatibility implied, so a log stamped with any other version is skipped with a notice instead of read under today's assumptions. ### Changed -- **Report, sessions, overview, compare, export, report JSON and menubar `localModelSavings` now use `billableOutputTokens` for displayed output.** Cost already did. Exclusive providers (Grok and the rest) were under-reporting output by exactly their reasoning tokens; optimize added reasoning on top of output for the inclusive set `{claude, codex, copilot}` and double-counted it. Displayed aggregates now bill per call while the provider is known. Daily cache v28 re-derives finalized days so `report` matches the live parse. Thanks @saulcanina. (#1115) +- **Report, sessions, overview, compare, export, report JSON and menubar `localModelSavings` now use `billableOutputTokens` for displayed output.** Cost already did. Exclusive providers (Grok and the rest) were under-reporting output by exactly their reasoning tokens; optimize added reasoning on top of output for the inclusive set `{claude, codex, copilot}` and double-counted it. Displayed aggregates now bill per call while the provider is known, joined on the same `getShortModelName` key the parser uses for `modelBreakdown`. Daily cache v28 re-derives finalized days so `report` matches the live parse. Thanks @saulcanina. (#1115) - **The discovery sweep issues its metadata syscalls concurrently.** Every dated command re-walks and re-stats every provider tree before it can decide what is already cached, and that sweep was strictly serial: one `readdir`, one `stat`, one `state.json` read at a time, per provider, one provider after another. On a 21k-file / 9-provider corpus it owned most of a warm run's wall clock while the machine sat idle waiting on the kernel. Provider discovery now runs across providers at once, and the four walks that dominate it (claude, codex, kimicode, grok) plus the Claude project-dir walk and both fingerprint passes fan out through a shared bounded-concurrency helper. Order is unchanged everywhere — each level is re-concatenated in registry/`readdir` order before anything reconciles against the cache — so what is discovered, in what sequence, is byte-identical to the serial walk. Two smaller cuts ride along: the Claude walk reads directory entries with their types so a plain file no longer costs a wasted `subagents/` probe, and the Codex result cache (a single file that can reach hundreds of MB) now shares one in-flight load between concurrent readers instead of letting each one re-read and re-parse it. Warm `codeburn today` on that corpus: 5.30s to 2.99s; a cold parse 34.7s to 31.1s. (#1104) - **A kill mid-way through a non-Claude provider phase no longer restarts that whole phase.** `scanProjectDirs` (Claude) has long taken a throttled `saveProgress` callback so a killed cold parse resumes from a warm cache; `parseProviderSources` (every other provider — codex, cursor, gemini, and the rest) did not, and only persisted at the whole-provider boundary. On a large single-provider corpus (a multi-GB codex history is the common case) an app-timeout SIGKILL, crash, or force-quit during that phase discarded everything parsed since the last provider finished, forcing the entire phase to re-parse from zero on the next run. `parseProviderSources` now takes the same callback, invoked once per source right after that source's cache entry lands (mirroring `scanProjectDirs`' placement, outside the per-file try/catch), on the same file-count/wall-clock throttle. A file only ever gets a fingerprint once it has fully parsed, so a mid-file kill can never leave a half-parsed file's entry looking complete on resume. - **Routed model ids price as the model they wrap, and an unknown vendor prefix no longer prices by blind stripping.** Token-plan and gateway spellings of the same model (`omniroute:`, `cp/`, `cline-pass/`, `cline-free/`, `cmd/`, `antigravity/`) are peeled and the remaining id is priced, so a Cline Pass or OmniRoute session shows a `~` estimate instead of $0. In exchange, `provider/model` is no longer treated as authority on its own: the leading segment is stripped only when it is a namespace the bundled pricing catalog itself uses (`anthropic/`, `openai/`, `google/`, `x-ai/`, `qwen/`, `moonshotai/`, `nousresearch/`, `xiaomi/`, `z-ai/`, and every other vendor prefix in the LiteLLM snapshot), one of the routing wrappers above, or one of the client-side spellings `kimi/`, `mimo/`, `zhipu/`, `litellm_proxy/` and `openai_like/`. Anything else stays unpriced and is reported as unpriced rather than inheriting the price of a same-named cloud row, and local-runner prefixes (`ollama/`, `lmstudio/`, `hosted_vllm/`, `local/`) are excluded on purpose so an unlisted local tag can never invent cloud spend. A user price override for the bare id wins over the catalog row a routed spelling would otherwise hit. diff --git a/src/export.ts b/src/export.ts index ba294fa6..961e5b8e 100644 --- a/src/export.ts +++ b/src/export.ts @@ -6,7 +6,7 @@ import { getCurrency, convertCost, roundForActiveCurrency } from './currency.js' import { dateKey } from './day-aggregator.js' import { behavioralTurnCount, isBehavioralCall } from './behavioral-weight.js' import { aggregateModelEfficiency } from './model-efficiency.js' -import { callBillableOutputTokens } from './session-output.js' +import { callBillableOutputTokens, sessionModelBillableOutputTokens } from './session-output.js' function escCsv(s: string): string { const sanitized = /^[\t\r=+\-@]/.test(s) ? `'${s}` : s @@ -166,12 +166,9 @@ function buildModelRows(projects: ProjectSummary[], period: string): Row[] { acc.cacheWrite += d.tokens.cacheCreationInputTokens ?? 0 } // Output must be billed per call while provider identity is still known. - // Grouped modelBreakdown tokens cannot distinguish exclusive vs inclusive. - for (const turn of session.turns) { - for (const call of turn.assistantCalls) { - if (!call.model) continue - ensure(call.model).output += callBillableOutputTokens(call) - } + // Join on the same key as parser modelBreakdown (getShortModelName), not raw call.model. + for (const [model, output] of Object.entries(sessionModelBillableOutputTokens(session))) { + ensure(model).output += output } } } diff --git a/src/main.ts b/src/main.ts index 7cc43058..9c1c1837 100644 --- a/src/main.ts +++ b/src/main.ts @@ -10,7 +10,7 @@ import { convertCost, formatCost } from './currency.js' import { renderStatusBar } from './format.js' import { toDateString } from './daily-cache.js' import { dateKey } from './day-aggregator.js' -import { callBillableOutputTokens } from './session-output.js' +import { sessionModelBillableOutputTokens } from './session-output.js' import { isBehavioralCall } from './behavioral-weight.js' import { CATEGORY_LABELS, type DateRange, type ProjectSummary, type TaskCategory } from './types.js' import type { AppliedFix } from './act/types.js' @@ -547,15 +547,12 @@ function buildJsonReport(projects: ProjectSummary[], period: string, periodKey: modelMap[model].cacheWriteTokens += d.tokens.cacheCreationInputTokens } // Output must be billed per call while provider identity is still known. - // Inferring after modelBreakdown grouping is not safe for mixed-provider data. - for (const turn of sess.turns) { - for (const call of turn.assistantCalls) { - if (!call.model) continue - if (!modelMap[call.model]) { - modelMap[call.model] = { calls: 0, cost: 0, savings: 0, estimatedCost: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, baselineModel: '' } - } - modelMap[call.model].outputTokens += callBillableOutputTokens(call) + // Join on the same key as parser modelBreakdown (getShortModelName), not raw call.model. + for (const [model, output] of Object.entries(sessionModelBillableOutputTokens(sess))) { + if (!modelMap[model]) { + modelMap[model] = { calls: 0, cost: 0, savings: 0, estimatedCost: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, baselineModel: '' } } + modelMap[model].outputTokens += output } } // Pull the active baseline model name out of the savings config so the diff --git a/src/overview.ts b/src/overview.ts index ed3c09c7..879c3b4d 100644 --- a/src/overview.ts +++ b/src/overview.ts @@ -5,7 +5,7 @@ import { homedir } from 'os' import { CATEGORY_LABELS, type ProjectSummary, type TaskCategory } from './types.js' import { formatCost as baseCost, getCurrency } from './currency.js' import { findUnpricedModels, getShortModelName, unpricedModelHint } from './models.js' -import { callBillableOutputTokens, sessionBillableOutputTokens } from './session-output.js' +import { callBillableOutputTokens, sessionBillableOutputTokens, sessionModelBillableOutputTokens } from './session-output.js' import { markEstimated } from './format.js' import { dateKey } from './day-aggregator.js' import type { DailyEntry } from './daily-cache.js' @@ -152,6 +152,13 @@ export function renderOverview( e.tokens += d.tokens.inputTokens + d.tokens.cacheReadInputTokens + d.tokens.cacheCreationInputTokens byModel.set(m, e) } + // Output must be billed per call while provider identity is still known. + // Join on the same key as parser modelBreakdown (getShortModelName), not raw call.model. + for (const [m, output] of Object.entries(sessionModelBillableOutputTokens(s))) { + const e = byModel.get(m) ?? { cost: 0, calls: 0, tokens: 0, estimatedCost: 0 } + e.tokens += output + byModel.set(m, e) + } for (const [cat, d] of Object.entries(s.categoryBreakdown)) { const e = byCat.get(cat) ?? { cost: 0, turns: 0 } e.cost += d.costUSD @@ -167,9 +174,6 @@ export function renderOverview( const usage = call.usage const billableOut = callBillableOutputTokens(call) const tk = (usage?.inputTokens ?? 0) + billableOut + (usage?.cacheReadInputTokens ?? 0) + (usage?.cacheCreationInputTokens ?? 0) - const me = byModel.get(call.model) ?? { cost: 0, calls: 0, tokens: 0, estimatedCost: 0 } - me.tokens += billableOut - byModel.set(call.model, me) const pv = byProvider.get(call.provider) ?? { cost: 0, tokens: 0 } pv.cost += call.costUSD pv.tokens += tk diff --git a/src/session-output.ts b/src/session-output.ts index e7c85e6b..19e8e31d 100644 --- a/src/session-output.ts +++ b/src/session-output.ts @@ -1,4 +1,4 @@ -import { billableOutputTokens } from './models.js' +import { billableOutputTokens, getShortModelName } from './models.js' import type { SessionSummary } from './types.js' type UsageLike = { @@ -8,9 +8,46 @@ type UsageLike = { type CallLike = { provider?: string + model?: string usage?: UsageLike } +/** Same key the parser uses for non-Devin `modelBreakdown` buckets. */ +export function modelBreakdownKey(call: { provider?: string; model?: string }): string | undefined { + if (!call.model) return undefined + return call.provider === 'devin' ? call.model : getShortModelName(call.model) +} + +/** + * Per-model displayed output, keyed like parser `modelBreakdown`. + * Call usage wins while provider identity is known. Aggregate-only / + * stub sessions fall back to each existing bucket so a finite + * sessionBillableOutputTokens cannot leave model Output Tokens at 0. + */ +export function sessionModelBillableOutputTokens(session: SessionSummary): Record { + const out: Record = {} + let sawUsage = false + for (const turn of session.turns ?? []) { + for (const call of turn.assistantCalls ?? []) { + if (!call.usage) continue + sawUsage = true + const key = modelBreakdownKey(call) + if (!key) continue + out[key] = (out[key] ?? 0) + callBillableOutputTokens(call) + } + } + if (sawUsage) return out + const provider = inferSessionProvider(session) + for (const [model, d] of Object.entries(session.modelBreakdown ?? {})) { + out[model] = billableOutputTokens( + provider, + d.tokens?.outputTokens ?? 0, + d.tokens?.reasoningTokens ?? 0, + ) + } + return out +} + /** First on-call provider, then a model-name fallback. Sessions are usually one provider. */ export function inferSessionProvider(session: SessionSummary): string { for (const turn of session.turns ?? []) { diff --git a/tests/billable-output-1115.test.ts b/tests/billable-output-1115.test.ts index ea89b6d2..47be29dd 100644 --- a/tests/billable-output-1115.test.ts +++ b/tests/billable-output-1115.test.ts @@ -6,10 +6,10 @@ import { describe, expect, it } from 'vitest' import { aggregateModelStats, computeComparison } from '../src/compare-stats.js' import { aggregateProjectsIntoDays } from '../src/day-aggregator.js' import { exportJson } from '../src/export.js' -import { billableOutputTokens } from '../src/models.js' +import { billableOutputTokens, getShortModelName } from '../src/models.js' import { findContextBloatCandidates } from '../src/optimize.js' import { renderOverview } from '../src/overview.js' -import { callBillableOutputTokens, sessionBillableOutputTokens } from '../src/session-output.js' +import { callBillableOutputTokens, sessionBillableOutputTokens, sessionModelBillableOutputTokens } from '../src/session-output.js' import { aggregateSessions } from '../src/sessions-report.js' import { buildPeriodData } from '../src/usage-aggregator.js' import type { ProjectSummary, SessionSummary } from '../src/types.js' @@ -67,7 +67,7 @@ function makeSession(provider: string, outputTokens: number, reasoningTokens: nu assistantCalls: [call], }], modelBreakdown: { - [call.model]: { + [getShortModelName(call.model)]: { calls: 1, costUSD: 0, savingsUSD: 0, @@ -164,8 +164,8 @@ describe('#1115 billableOutputTokens on report/optimize totals', () => { records: Array<{ outputTokens: number; reasoningTokens: number }> } expect(data.periods[0]!.daily[0]!['Output Tokens']).toBe(23) - const grokRow = data.periods[0]!.models.find(r => r.Model === 'grok-4')! - const codexRow = data.periods[0]!.models.find(r => r.Model === 'gpt-5.4')! + const grokRow = data.periods[0]!.models.find(r => r.Model === getShortModelName('grok-4'))! + const codexRow = data.periods[0]!.models.find(r => r.Model === getShortModelName('gpt-5.4'))! expect(grokRow['Output Tokens']).toBe(13) expect(codexRow['Output Tokens']).toBe(10) // Record-level export stays raw columns, not the billable sum. @@ -208,4 +208,67 @@ describe('#1115 billableOutputTokens on report/optimize totals', () => { expect(callBillableOutputTokens({} as never)).toBe(0) expect(callBillableOutputTokens({ provider: 'grok', usage: { outputTokens: 10, reasoningTokens: 3 } })).toBe(13) }) + + it('joins model output on getShortModelName, not raw call.model', async () => { + const session = makeSession('claude', 10, 0) + session.turns[0]!.assistantCalls[0]!.model = 'deepseek-v4-pro' + session.modelBreakdown = { + 'DeepSeek v4 Pro': { + calls: 1, + costUSD: 1, + savingsUSD: 0, + tokens: session.turns[0]!.assistantCalls[0]!.usage, + }, + } + const billed = sessionModelBillableOutputTokens(session) + expect(billed).toEqual({ 'DeepSeek v4 Pro': 10 }) + expect(billed['deepseek-v4-pro']).toBeUndefined() + + const dir = await mkdtemp(join(tmpdir(), 'cb-1116-display-key-')) + try { + const path = await exportJson( + [{ label: '30 Days', projects: [makeProject(session)] }], + join(dir, 'out.json'), + ) + const data = JSON.parse(await readFile(path, 'utf-8')) as { + periods: Array<{ models: Array<{ Model: string; 'API Calls': number; 'Output Tokens': number }> }> + } + const rows = data.periods[0]!.models + expect(rows.filter(r => r.Model === 'deepseek-v4-pro')).toHaveLength(0) + const named = rows.find(r => r.Model === 'DeepSeek v4 Pro') + expect(named).toEqual(expect.objectContaining({ + Model: 'DeepSeek v4 Pro', + 'API Calls': 1, + 'Output Tokens': 10, + })) + } finally { + await rm(dir, { recursive: true, force: true }) + } + + const out = renderOverview([makeProject(session)], { label: 'May 2026', color: false }) + expect(out).toContain('DeepSeek v4 Pro') + expect(out).not.toContain('deepseek-v4-pro') + expect(out).toMatch(/DeepSeek v4 Pro[\s\S]*\b1\b[\s\S]*\b10\b/) + }) + + it('aggregate-only model rows keep finite billable output', async () => { + const aggregate = makeSession('grok', 10, 3) + aggregate.turns = [] + expect(sessionBillableOutputTokens(aggregate)).toBe(13) + expect(sessionModelBillableOutputTokens(aggregate)).toEqual({ [getShortModelName('grok-4')]: 13 }) + + const dir = await mkdtemp(join(tmpdir(), 'cb-1116-agg-only-')) + try { + const path = await exportJson( + [{ label: '30 Days', projects: [makeProject(aggregate)] }], + join(dir, 'out.json'), + ) + const data = JSON.parse(await readFile(path, 'utf-8')) as { + periods: Array<{ models: Array<{ Model: string; 'Output Tokens': number }> }> + } + expect(data.periods[0]!.models.find(r => r.Model === getShortModelName('grok-4'))!['Output Tokens']).toBe(13) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) }) From 57709c69c66771743a746502fe1001b20fe27e1f Mon Sep 17 00:00:00 2001 From: Aditya Vikram Singh <247195684+avs-io@users.noreply.github.com> Date: Sun, 23 Aug 2026 18:52:13 +0530 Subject: [PATCH 4/6] feat(doctor): probeRoots for five fixed-location providers (#899 Tier 2, batch 2) Share one discovery-root resolver per provider so doctor reports the exact paths discoverSessions reads: all three Codebuff CHANNELS, Devin's transcripts + sessions.db children, Gemini's tmp parent, Kiro's pre-filter candidates with empty CLI/v2 skipped, and Mistral Vibe's joined sessions dir. --- src/providers/codebuff.ts | 40 +++--- src/providers/devin.ts | 29 ++++- src/providers/gemini.ts | 8 +- src/providers/kiro.ts | 96 ++++++++------ src/providers/mistral-vibe.ts | 8 +- tests/provider-probe-roots-tier2.test.ts | 151 +++++++++++++++++++++++ 6 files changed, 267 insertions(+), 65 deletions(-) diff --git a/src/providers/codebuff.ts b/src/providers/codebuff.ts index 6ee746c1..41824107 100644 --- a/src/providers/codebuff.ts +++ b/src/providers/codebuff.ts @@ -4,7 +4,7 @@ import { homedir } from 'os' import { calculateCost } from '../models.js' import { extractBashCommands } from '../bash-utils.js' -import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { ProbeRoot, Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' // Codebuff (formerly Manicode) uses a credit-based billing system. The local // chat-messages.json doesn't record per-call token counts the way Claude Code @@ -123,11 +123,15 @@ type CodebuffChatMessage = { metadata?: CodebuffMetadata } -function getCodebuffBaseDir(override?: string): string { - if (override && override.trim()) return override +// Shared by discoverSessions and probeRoots. Factory / CODEBUFF_DATA_DIR win +// as a single root; otherwise discovery walks every CHANNEL before existence +// filtering. Empty / blank factory is unset (same as #938 truthiness). +export function getCodebuffRootSet(override?: string): string[] { + if (override && override.trim()) return [override] const envPath = process.env['CODEBUFF_DATA_DIR'] - if (envPath && envPath.trim()) return envPath - return join(homedir(), '.config', 'manicode') + if (envPath && envPath.trim()) return [envPath] + const configDir = join(homedir(), '.config') + return CHANNELS.map(channel => join(configDir, channel)) } function pickNumber(...vals: Array): number | undefined { @@ -294,23 +298,9 @@ async function discoverChannel(root: string): Promise { return sources } -async function discoverSessionsInBase(baseDir: string): Promise { +async function discoverSessionsInRoots(roots: string[]): Promise { const results: SessionSource[] = [] - - // Honor an explicit override: walk only the provided directory even if it - // matches one of the channel names literally. - if (process.env['CODEBUFF_DATA_DIR'] || baseDir !== join(homedir(), '.config', 'manicode')) { - const rootStat = await stat(baseDir).catch(() => null) - if (!rootStat?.isDirectory()) return results - results.push(...await discoverChannel(baseDir)) - return results - } - - const configDir = join(homedir(), '.config') - for (const channel of CHANNELS) { - const root = join(configDir, channel) - const rootStat = await stat(root).catch(() => null) - if (!rootStat?.isDirectory()) continue + for (const root of roots) { results.push(...await discoverChannel(root)) } return results @@ -433,7 +423,7 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars } export function createCodebuffProvider(baseDir?: string): Provider { - const dir = getCodebuffBaseDir(baseDir) + const roots = getCodebuffRootSet(baseDir) return { name: 'codebuff', @@ -447,8 +437,12 @@ export function createCodebuffProvider(baseDir?: string): Provider { return toolNameMap[rawTool] ?? rawTool }, + async probeRoots(): Promise { + return roots.map(path => ({ path, label: 'chats' })) + }, + async discoverSessions(): Promise { - return discoverSessionsInBase(dir) + return discoverSessionsInRoots(roots) }, createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { diff --git a/src/providers/devin.ts b/src/providers/devin.ts index b41e7bb4..c97a0472 100644 --- a/src/providers/devin.ts +++ b/src/providers/devin.ts @@ -6,6 +6,7 @@ import { getShortModelName } from "../models.js"; import { openDatabase } from "../sqlite.js"; import { readConfig } from "../config.js"; import type { + ProbeRoot, Provider, SessionParser, SessionSource, @@ -524,8 +525,24 @@ class DevinSessionParser implements SessionParser { } } -export function createDevinProvider(cliDir: string): Provider { - const sessionsDbPath = join(cliDir, DEVIN_SESSIONS_DB); +function resolveDevinCliDir(override?: string): string { + return override && override.trim() ? override : DEFAULT_DEVIN_CLI_DIR; +} + +function getDevinDiscoveryRoots(cliDir: string): { + transcriptsDir: string; + sessionsDbPath: string; +} { + return { + transcriptsDir: join(cliDir, DEVIN_TRANSCRIPTS_SUBDIR), + sessionsDbPath: join(cliDir, DEVIN_SESSIONS_DB), + }; +} + +export function createDevinProvider(cliDir?: string): Provider { + const resolvedCliDir = resolveDevinCliDir(cliDir); + const { transcriptsDir, sessionsDbPath } = + getDevinDiscoveryRoots(resolvedCliDir); let sessionMetadata: Map | null = null; const getSessionMetadata = () => { @@ -545,10 +562,16 @@ export function createDevinProvider(cliDir: string): Provider { return rawTool; }, + async probeRoots(): Promise { + return [ + { path: transcriptsDir, label: "transcripts" }, + { path: sessionsDbPath, label: "sessions.db" }, + ]; + }, + async discoverSessions(): Promise { if ((await getCostFactor()) === null) return []; - const transcriptsDir = join(cliDir, DEVIN_TRANSCRIPTS_SUBDIR); const entries = await readdir(transcriptsDir).catch(() => []); const metadata = getSessionMetadata(); const sources: SessionSource[] = []; diff --git a/src/providers/gemini.ts b/src/providers/gemini.ts index e0333f0e..cffeb9b4 100644 --- a/src/providers/gemini.ts +++ b/src/providers/gemini.ts @@ -5,7 +5,7 @@ import { homedir } from 'os' import { readSessionFile } from '../fs-utils.js' import { calculateCost } from '../models.js' import { extractBashCommands } from '../bash-utils.js' -import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { ProbeRoot, Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' const toolNameMap: Record = { read_file: 'Read', @@ -213,7 +213,7 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars } } -function getGeminiTmpDir(): string { +export function getGeminiTmpDir(): string { return join(homedir(), '.gemini', 'tmp') } @@ -272,6 +272,10 @@ export function createGeminiProvider(): Provider { return toolNameMap[rawTool] ?? rawTool }, + async probeRoots(): Promise { + return [{ path: getGeminiTmpDir(), label: 'tmp' }] + }, + async discoverSessions(): Promise { return discoverSessions() }, diff --git a/src/providers/kiro.ts b/src/providers/kiro.ts index 4e0308d9..b5adc916 100644 --- a/src/providers/kiro.ts +++ b/src/providers/kiro.ts @@ -9,7 +9,7 @@ import { flatSlice, flatString } from '../content-utils.js' import { calculateCost } from '../models.js' import { estimateTokensFromChars } from '../token-estimate.js' import type { ToolCall } from '../types.js' -import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { ProbeRoot, Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' // Kiro bills in credits: individual plans are $20/mo for 1,000 credits and // overage is billed at $0.04 per additional credit. We price credits at the @@ -925,7 +925,10 @@ function createParser(source: SessionSource, seenKeys: Set): SessionPars // --- Discovery --- -function getKiroAgentDir(override?: string): string[] { +// Pre-filter candidate list shared by probeRoots and discovery. Must not +// existsSync-filter: Linux discovery still trims in getKiroAgentDir, but +// doctor has to report missing defaults. Empty / blank override is unset. +export function getKiroAgentDirCandidates(override?: string): string[] { if (override) return [override] if (process.platform === 'darwin') { return [join(homedir(), 'Library', 'Application Support', 'Kiro', 'User', 'globalStorage', 'kiro.kiroagent')] @@ -933,17 +936,21 @@ function getKiroAgentDir(override?: string): string[] { if (process.platform === 'win32') { return [join(homedir(), 'AppData', 'Roaming', 'Kiro', 'User', 'globalStorage', 'kiro.kiroagent')] } + return [ + join(homedir(), '.kiro-server', 'data', 'User', 'globalStorage', 'kiro.kiroagent'), + join(homedir(), '.config', 'Kiro', 'User', 'globalStorage', 'kiro.kiroagent'), + ] +} + +function getKiroAgentDir(override?: string): string[] { + const candidates = getKiroAgentDirCandidates(override) + if (override || process.platform !== 'linux') return candidates // On Linux, scan both ~/.kiro-server/data/... (remote dev boxes) and // ~/.config/Kiro/... (local installs). Both can have data simultaneously // if the user switches between local and remote, or if .kiro-server exists // but is stale while .config/Kiro has current sessions. - const paths: string[] = [] - const kiroServer = join(homedir(), '.kiro-server', 'data', 'User', 'globalStorage', 'kiro.kiroagent') - const kiroConfig = join(homedir(), '.config', 'Kiro', 'User', 'globalStorage', 'kiro.kiroagent') - if (existsSync(kiroServer)) paths.push(kiroServer) - if (existsSync(kiroConfig)) paths.push(kiroConfig) - // Fallback to config path if neither exists (will just find nothing) - return paths.length > 0 ? paths : [kiroConfig] + const existing = candidates.filter(p => existsSync(p)) + return existing.length > 0 ? existing : [candidates[candidates.length - 1]!] } function getKiroWorkspaceStorageDir(override?: string): string { @@ -986,26 +993,29 @@ async function resolveWorkspaceProject(agentDir: string, workspaceStorageDir: st return workspaceHash } -async function discoverSessions(agentDir: string, workspaceStorageDir: string, cliSessionsDir: string): Promise { +async function discoverSessions(agentDir: string, workspaceStorageDir: string, cliSessionsDir: string | undefined): Promise { const sources: SessionSource[] = [] // --- Kiro CLI sessions (~/.kiro/sessions/cli/) --- - try { - const cliEntries = await readdir(cliSessionsDir, { withFileTypes: true }) - for (const entry of cliEntries) { - if (!entry.isFile() || !entry.name.endsWith('.jsonl')) continue - const jsonlPath = join(cliSessionsDir, entry.name) - // Derive project from companion .json - const metaPath = jsonlPath.replace(/\.jsonl$/, '.json') - let project = 'kiro-cli' - try { - const raw = await readFile(metaPath, 'utf-8') - const meta = JSON.parse(raw) as { cwd?: string } - if (meta.cwd) project = basename(meta.cwd) - } catch {} - sources.push({ path: jsonlPath, project, provider: 'kiro' }) - } - } catch {} + // Empty CLI is skipped (do not readdir '' or '.'). Same skip in probeRoots. + if (cliSessionsDir) { + try { + const cliEntries = await readdir(cliSessionsDir, { withFileTypes: true }) + for (const entry of cliEntries) { + if (!entry.isFile() || !entry.name.endsWith('.jsonl')) continue + const jsonlPath = join(cliSessionsDir, entry.name) + // Derive project from companion .json + const metaPath = jsonlPath.replace(/\.jsonl$/, '.json') + let project = 'kiro-cli' + try { + const raw = await readFile(metaPath, 'utf-8') + const meta = JSON.parse(raw) as { cwd?: string } + if (meta.cwd) project = basename(meta.cwd) + } catch {} + sources.push({ path: jsonlPath, project, provider: 'kiro' }) + } + } catch {} + } // --- Kiro IDE sessions --- let workspaceDirs: string[] @@ -1123,18 +1133,24 @@ async function discoverV2Sessions(sessionsRoot: string): Promise/sess_*/, a sibling of the - // CLI store (.../sessions/cli). Derive the root from cliDir ONLY when cliDir was - // itself explicit (default path or cliSessionsDirOverride). When only - // agentDirOverride is set (tests), the derived cliDir parent would point at an - // arbitrary directory (e.g. the system tmpdir) — scan nothing in that case. - const v2Root = v2SessionsRootOverride ?? - (cliSessionsDirOverride ? dirname(cliSessionsDirOverride) : - agentDirOverride ? undefined : dirname(cliDir)) + // CLI store (.../sessions/cli). Derive the root from an explicit CLI override + // or the default CLI parent. Empty CLI does not drop default v2; empty v2 + // is skipped on its own. When only agentDirOverride is set (tests), do not + // derive v2 from the test tmpdir. + const v2Root = v2SessionsRootOverride === '' + ? undefined + : v2SessionsRootOverride ?? + (cliSessionsDirOverride ? dirname(cliSessionsDirOverride) : + agentDirOverride ? undefined : dirname(defaultCliDir)) return { name: 'kiro', @@ -1153,6 +1169,16 @@ export function createKiroProvider(agentDirOverride?: string, workspaceStorageDi return toolNameMap[rawTool] ?? rawTool }, + async probeRoots(): Promise { + const roots: ProbeRoot[] = [ + ...agentCandidates.map(path => ({ path, label: 'agent' })), + { path: wsDir, label: 'workspace' }, + ] + if (cliDir) roots.push({ path: cliDir, label: 'cli' }) + if (v2Root) roots.push({ path: v2Root, label: 'v2' }) + return roots + }, + async discoverSessions(): Promise { const allSources: SessionSource[] = [] for (const agentDir of agentDirs) { diff --git a/src/providers/mistral-vibe.ts b/src/providers/mistral-vibe.ts index 158a53a4..23c3667f 100644 --- a/src/providers/mistral-vibe.ts +++ b/src/providers/mistral-vibe.ts @@ -5,7 +5,7 @@ import { homedir } from 'os' import { readSessionFile, readSessionLines } from '../fs-utils.js' import { calculateCost } from '../models.js' import { extractBashCommands } from '../bash-utils.js' -import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' +import type { ProbeRoot, Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' import { safeNumber } from '../parser.js' const METADATA_FILENAME = 'meta.json' @@ -82,7 +82,7 @@ type VibeMessage = { tool_calls?: VibeToolCall[] | null } -function getMistralVibeSessionsDir(override?: string): string { +export function getMistralVibeSessionsDir(override?: string): string { if (override) return override const configuredHome = process.env['VIBE_HOME'] const vibeHome = configuredHome ? expandHome(configuredHome) : join(homedir(), '.vibe') @@ -409,6 +409,10 @@ export function createMistralVibeProvider(sessionsDir?: string): Provider { return toolNameMap[rawTool] ?? rawTool }, + async probeRoots(): Promise { + return [{ path: dir, label: 'sessions' }] + }, + async discoverSessions(): Promise { const dirs = await discoverSessionDirs(dir) const sources: SessionSource[] = [] diff --git a/tests/provider-probe-roots-tier2.test.ts b/tests/provider-probe-roots-tier2.test.ts index 0c34f079..c977c913 100644 --- a/tests/provider-probe-roots-tier2.test.ts +++ b/tests/provider-probe-roots-tier2.test.ts @@ -13,6 +13,11 @@ import { discoverClineTasks, getVSCodeGlobalStoragePaths, } from '../src/providers/vscode-cline-parser.js' +import { createCodebuffProvider, getCodebuffRootSet } from '../src/providers/codebuff.js' +import { createDevinProvider } from '../src/providers/devin.js' +import { createGeminiProvider, getGeminiTmpDir } from '../src/providers/gemini.js' +import { createKiroProvider, getKiroAgentDirCandidates } from '../src/providers/kiro.js' +import { createMistralVibeProvider, getMistralVibeSessionsDir } from '../src/providers/mistral-vibe.js' // #899 Tier 2, batch 1. probeRoots() must report the roots discovery actually // reads: a probe pointing somewhere discovery never looks is worse than none, @@ -108,3 +113,149 @@ describe('probeRoots mirrors discovery resolution (Tier 2, batch 1)', () => { ]) }) }) + +// #899 Tier 2, batch 2. Same contract as batch 1: probeRoots() is the exact +// discovery-root set after the same resolution, pinned as full objects. +describe('probeRoots mirrors discovery resolution (Tier 2, batch 2)', () => { + it('codebuff reports all three CHANNELS on default, and empty factory is unset', async () => { + const expected = [ + { path: join(homedir(), '.config', 'manicode'), label: 'chats' }, + { path: join(homedir(), '.config', 'manicode-dev'), label: 'chats' }, + { path: join(homedir(), '.config', 'manicode-staging'), label: 'chats' }, + ] + expect(await createCodebuffProvider().probeRoots!()).toEqual(expected) + expect(await createCodebuffProvider('').probeRoots!()).toEqual(expected) + expect(getCodebuffRootSet()).toEqual(expected.map(r => r.path)) + expect(getCodebuffRootSet('')).toEqual(expected.map(r => r.path)) + for (const root of expected) expect(isAbsolute(root.path)).toBe(true) + }) + + it('codebuff reports the factory root, or CODEBUFF_DATA_DIR when factory is empty', async () => { + expect(await createCodebuffProvider('/tmp/codebuff-a').probeRoots!()).toEqual([ + { path: '/tmp/codebuff-a', label: 'chats' }, + ]) + process.env['CODEBUFF_DATA_DIR'] = '/tmp/codebuff-env' + expect(await createCodebuffProvider().probeRoots!()).toEqual([ + { path: '/tmp/codebuff-env', label: 'chats' }, + ]) + expect(await createCodebuffProvider('').probeRoots!()).toEqual([ + { path: '/tmp/codebuff-env', label: 'chats' }, + ]) + // Non-blank factory wins over the env root. + expect(await createCodebuffProvider('/tmp/codebuff-a').probeRoots!()).toEqual([ + { path: '/tmp/codebuff-a', label: 'chats' }, + ]) + }) + + it('devin reports transcripts + sessions.db, not the parent, and empty factory is default', async () => { + expect(await createDevinProvider('/tmp/probe-devin').probeRoots!()).toEqual([ + { path: join('/tmp/probe-devin', 'transcripts'), label: 'transcripts' }, + { path: join('/tmp/probe-devin', 'sessions.db'), label: 'sessions.db' }, + ]) + const defaults = [ + { path: join(homedir(), '.local', 'share', 'devin', 'cli', 'transcripts'), label: 'transcripts' }, + { path: join(homedir(), '.local', 'share', 'devin', 'cli', 'sessions.db'), label: 'sessions.db' }, + ] + expect(await createDevinProvider().probeRoots!()).toEqual(defaults) + expect(await createDevinProvider('').probeRoots!()).toEqual(defaults) + for (const root of defaults) expect(isAbsolute(root.path)).toBe(true) + }) + + it('gemini reports only the shared tmp parent', async () => { + const tmpDir = getGeminiTmpDir() + expect(tmpDir).toBe(join(homedir(), '.gemini', 'tmp')) + expect(await createGeminiProvider().probeRoots!()).toEqual([ + { path: tmpDir, label: 'tmp' }, + ]) + expect(tmpDir).not.toContain(`${join('tmp', 'chats')}`) + expect(isAbsolute(tmpDir)).toBe(true) + }) + + it('kiro reports the override set exactly and never the developer Application Support tree', async () => { + const agent = '/tmp/kiro-agent' + const workspace = '/tmp/kiro-workspace' + const cli = '/tmp/kiro-cli' + const v2 = '/tmp/kiro-v2' + const roots = await createKiroProvider(agent, workspace, cli, v2).probeRoots!() + expect(roots).toEqual([ + { path: agent, label: 'agent' }, + { path: workspace, label: 'workspace' }, + { path: cli, label: 'cli' }, + { path: v2, label: 'v2' }, + ]) + for (const root of roots) { + expect(isAbsolute(root.path)).toBe(true) + expect(root.path).not.toContain('Application Support/Kiro') + } + }) + + it('kiro empty-string table matches discovery: agent/workspace fall back, empty CLI and v2 are skipped', async () => { + const workspace = '/tmp/kiro-ws' + const cli = '/tmp/kiro-cli' + const v2 = '/tmp/kiro-v2' + + const unsetAgent = await createKiroProvider(undefined, workspace, cli, v2).probeRoots!() + const emptyAgent = await createKiroProvider('', workspace, cli, v2).probeRoots!() + expect(emptyAgent).toEqual(unsetAgent) + expect(emptyAgent.map(r => r.path)).toEqual([ + ...getKiroAgentDirCandidates(''), + workspace, + cli, + v2, + ]) + expect(emptyAgent.filter(r => r.label === 'agent').map(r => r.path)).toEqual( + getKiroAgentDirCandidates(), + ) + + const unsetWorkspace = await createKiroProvider('/tmp/kiro-agent', undefined, cli, v2).probeRoots!() + const emptyWorkspace = await createKiroProvider('/tmp/kiro-agent', '', cli, v2).probeRoots!() + expect(emptyWorkspace).toEqual(unsetWorkspace) + expect(emptyWorkspace.find(r => r.label === 'workspace')!.path).not.toBe('') + + const emptyCli = await createKiroProvider('/tmp/kiro-agent', workspace, '', v2).probeRoots!() + expect(emptyCli).toEqual([ + { path: '/tmp/kiro-agent', label: 'agent' }, + { path: workspace, label: 'workspace' }, + { path: v2, label: 'v2' }, + ]) + expect(emptyCli.map(r => r.path)).not.toContain('') + expect(emptyCli.map(r => r.path)).not.toContain('.') + + const emptyV2 = await createKiroProvider('/tmp/kiro-agent', workspace, cli, '').probeRoots!() + expect(emptyV2).toEqual([ + { path: '/tmp/kiro-agent', label: 'agent' }, + { path: workspace, label: 'workspace' }, + { path: cli, label: 'cli' }, + ]) + expect(emptyV2.some(r => r.label === 'v2')).toBe(false) + + const emptyCliAndV2 = await createKiroProvider('/tmp/kiro-agent', workspace, '', '').probeRoots!() + expect(emptyCliAndV2).toEqual([ + { path: '/tmp/kiro-agent', label: 'agent' }, + { path: workspace, label: 'workspace' }, + ]) + + const skipped = await createKiroProvider('/tmp/kiro-agent', workspace, '', '').discoverSessions() + expect(skipped).toEqual([]) + }) + + it('mistral-vibe reports the same joined sessions dir discovery uses', async () => { + expect(await createMistralVibeProvider('/tmp/vibe-sessions').probeRoots!()).toEqual([ + { path: getMistralVibeSessionsDir('/tmp/vibe-sessions'), label: 'sessions' }, + ]) + expect(getMistralVibeSessionsDir('/tmp/vibe-sessions')).toBe('/tmp/vibe-sessions') + + const defaults = await createMistralVibeProvider().probeRoots!() + expect(defaults).toEqual([ + { path: getMistralVibeSessionsDir(), label: 'sessions' }, + ]) + expect(defaults[0]!.path).toBe(join(homedir(), '.vibe', 'logs', 'session')) + expect(await createMistralVibeProvider('').probeRoots!()).toEqual(defaults) + + process.env['VIBE_HOME'] = '/tmp/vibe-home' + expect(await createMistralVibeProvider().probeRoots!()).toEqual([ + { path: join('/tmp/vibe-home', 'logs', 'session'), label: 'sessions' }, + ]) + expect(getMistralVibeSessionsDir()).toBe(join('/tmp/vibe-home', 'logs', 'session')) + }) +}) From caba98817b4e172445b91032f284168ef0a97cc3 Mon Sep 17 00:00:00 2001 From: Aditya Vikram Singh <247195684+avs-io@users.noreply.github.com> Date: Sun, 23 Aug 2026 18:53:45 +0530 Subject: [PATCH 5/6] fix(doctor): changelog and isolate probeRoots env in #899 tests Restore CODEBUFF_DATA_DIR / VIBE_HOME after the batch-2 cases so later suites cannot inherit a leftover override. --- CHANGELOG.md | 2 ++ tests/provider-probe-roots-tier2.test.ts | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e21dddda..43ece745 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,8 @@ - **The resident `codeburn serve` child.** The first real panel request is also the cache warm-up, so startup never runs an artificial warm-up query beside a duplicate one-shot child; each served command carries its own read-only option allowlist, and anything outside it falls back to a normal spawn; the child exits when its stdin closes, so it can never outlive the app. Requests whose response exceeds the 16 MiB frame limit still replace the child, but that deliberate kill no longer spends the resident's unexpected-death budget. (#972) ### Fixed +- **`codeburn doctor` now probes the five remaining fixed-location providers.** `codebuff`, `devin`, `gemini`, `kiro`, and `mistral-vibe` implement `probeRoots()` through the same resolvers discovery uses, so a silent zero is distinguishable from a missing install. Codebuff reports all three manicode channels unless a factory or `CODEBUFF_DATA_DIR` pins one; Devin reports `transcripts` plus `sessions.db`, not the parent; Gemini reports only `~/.gemini/tmp`; Kiro reports pre-filter candidates (empty CLI/v2 skipped, empty agent/workspace fall back); Mistral Vibe reports the joined sessions dir. Missing defaults still appear. Thanks @therickfactr. (#899) + - **Subscription SKUs are classified from real product ids, and a false-positive built-in can be opted out.** `codex-auto-review` consumes ordinary Codex usage ([openai/codex#32224](https://github.com/openai/codex/issues/32224)) and is priced as GPT-5.5 on #1056, so treating it as $0 hid real spend — it left the flat-rate list. Warp's product id is `auto`, not the synthetic `warp`. `kimi-for-coding-highspeed` (the SKU #968 was filed around) is now honestly $0. `big-pickle` was dropped: it appears under OpenCode, not as a cited ClinePass codename. `codeburn model-flat-rate --remove` now opts out of a built-in, so a wrong classifier entry can warn again without waiting for a release. The daily-cache config hash now always includes the flat-rate section (even when empty), so the first run after upgrade re-derives every stored day once from the warm session cache. (#968, #1050) - **Codex MCP and skill usage is attributed from every shape Codex records a shell command in.** `mcp-cli call ` was only recognized when the command arrived as `function_call` arguments (#656). Codex has two other shapes for the same exec: its custom-tool transport records the shell tool as a `custom_tool_call` whose payload is an `input` program rather than `arguments`, and its item model repeats a finished command as `event_msg`/`item_completed` carrying a `CommandExecution` item with an argv `command`. Both reached the Bash counter and neither reached the matcher, so a CLI-wrapped MCP call stayed absent from the MCP breakdown exactly as before the fix. All three shapes now feed one classification pipeline. The same pipeline learns skills: Codex has no skill tool, so loading one is a shell read of the skill's `SKILL.md`, and those reads landed entirely under Bash with the Skills dimension empty. A read counts as a skill load only when the command segment starts with a file-reading binary (`cat`/`bat`/`sed`/`head`/`tail`/`less`/`more`) and the path it reads ends in `/SKILL.md`; the skill is ``, the same key `pi` derives for a native skill read (#588) and the same vocabulary the Claude parser records from the `Skill` tool. A `grep`/`rg`/`ls` that merely mentions a `SKILL.md` is a search near the file, not a skill load, and stays plain Bash. This is attribution only — no call, token or cost figure moves, and a command carried by both a response item and an item-model item is attributed once. On a 1,397-rollout corpus: Skills went from empty to 7 skills over 35 turns (55 attributions), Bash was unchanged at 42,170, and cost, calls, tokens, sessions, daily, models and projects came back identical. Cached Codex sessions re-parse once (`CODEX_CACHE_VERSION` 13 → 14 and the codex parse version both move; without them the fix is invisible on a warm cache). Thanks @chr-evensen. (#478) - **`gpt-5.6-codex` and `gpt-5.6-codex-max` now have their own pricing rows.** Neither id is in LiteLLM yet, and both were missing from the bundled snapshot — flagged during #1075 verification on a real corpus (285 sessions, 5,446 calls). `getModelCosts` already resolved both through the `gpt-5.6` prefix fallback, so live pricing was already correct once a session priced fresh; every prior Codex-suffixed id LiteLLM does carry bills identically to its bare-model sibling of the same generation (`gpt-5-codex` == `gpt-5`, `gpt-5.1-codex` == `gpt-5.1-codex-max` == `gpt-5.1`, `gpt-5.2-codex` == `gpt-5.2`, `gpt-5.3-codex` == `gpt-5.3`), which is the evidence both new rows mirror rather than inventing a rate. The gap that does not self-heal is the daily cache: it has no per-provider invalidation, so a day finalized while either id had no billable rate keeps that $0 forever. Raising `MIN_SUPPORTED_VERSION` (v23 -> v24) forces the one-time re-derivation, a lossless no-op for days already correct. (#1077) diff --git a/tests/provider-probe-roots-tier2.test.ts b/tests/provider-probe-roots-tier2.test.ts index c977c913..aab11525 100644 --- a/tests/provider-probe-roots-tier2.test.ts +++ b/tests/provider-probe-roots-tier2.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from 'vitest' +import { afterEach, describe, it, expect } from 'vitest' import { isAbsolute, join } from 'path' import { homedir } from 'os' @@ -117,6 +117,16 @@ describe('probeRoots mirrors discovery resolution (Tier 2, batch 1)', () => { // #899 Tier 2, batch 2. Same contract as batch 1: probeRoots() is the exact // discovery-root set after the same resolution, pinned as full objects. describe('probeRoots mirrors discovery resolution (Tier 2, batch 2)', () => { + const originalCodebuffDataDir = process.env['CODEBUFF_DATA_DIR'] + const originalVibeHome = process.env['VIBE_HOME'] + + afterEach(() => { + if (originalCodebuffDataDir === undefined) delete process.env['CODEBUFF_DATA_DIR'] + else process.env['CODEBUFF_DATA_DIR'] = originalCodebuffDataDir + if (originalVibeHome === undefined) delete process.env['VIBE_HOME'] + else process.env['VIBE_HOME'] = originalVibeHome + }) + it('codebuff reports all three CHANNELS on default, and empty factory is unset', async () => { const expected = [ { path: join(homedir(), '.config', 'manicode'), label: 'chats' }, From 80fbae3725dc50009e22e6996690702deed6c909 Mon Sep 17 00:00:00 2001 From: Aditya Vikram Singh <247195684+avs-io@users.noreply.github.com> Date: Sun, 23 Aug 2026 20:06:31 +0530 Subject: [PATCH 6/6] fix(report): join billable output onto existing modelBreakdown keys Parser sessions key modelBreakdown by getShortModelName. Leftover raw-id buckets (overview fixtures, cached summaries) were minting a second $0 / 0-call short-name row that findUnpricedModels flagged as Unpriced and failed CI on #1116. --- CHANGELOG.md | 2 +- src/session-output.ts | 21 +++++++++++++++++++-- tests/billable-output-1115.test.ts | 25 +++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3007475d..89e61aa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ - **DeepSeek Harness (`dsh`) is now a supported provider.** Reads DeepSeek's open-source agent harness from `~/.dsh/sessions` (`DSH_HOME` relocates the root), both the default zstd logs and the uncompressed `session.jsonl` variant. A `.zstd` log is a concatenation of independent zstd frames, one per write batch, so it is decoded frame by frame behind a structural frame scan and a torn trailing frame from a crashed writer is ignored rather than failing the file (needs Node 22.15+ for `zlib` zstd; below that dsh is skipped with a notice instead of counted as $0). One call per `(turn, step)`, with the step's final `assistant/message` usage superseding the streamed `assistant/chunk` sample of the same call rather than adding to it, the model taken from the message that served the step, and reasoning tokens billed at the output rate. DSH records tokens but no cost, so calls are priced from the shared tables. The events a forked session replays from its parent are skipped, since codeburn already counts the parent's own log. The session format is pinned at version 0 upstream with no compatibility implied, so a log stamped with any other version is skipped with a notice instead of read under today's assumptions. ### Changed -- **Report, sessions, overview, compare, export, report JSON and menubar `localModelSavings` now use `billableOutputTokens` for displayed output.** Cost already did. Exclusive providers (Grok and the rest) were under-reporting output by exactly their reasoning tokens; optimize added reasoning on top of output for the inclusive set `{claude, codex, copilot}` and double-counted it. Displayed aggregates now bill per call while the provider is known, joined on the same `getShortModelName` key the parser uses for `modelBreakdown`. Daily cache v28 re-derives finalized days so `report` matches the live parse. Thanks @saulcanina. (#1115) +- **Report, sessions, overview, compare, export, report JSON and menubar `localModelSavings` now use `billableOutputTokens` for displayed output.** Cost already did. Exclusive providers (Grok and the rest) were under-reporting output by exactly their reasoning tokens; optimize added reasoning on top of output for the inclusive set `{claude, codex, copilot}` and double-counted it. Displayed aggregates now bill per call while the provider is known, joined onto an existing `modelBreakdown` key (parser short name first, then the raw id) so a leftover raw-id bucket cannot mint a $0 `Opus 4.8` Unpriced orphan. Daily cache v28 re-derives finalized days so `report` matches the live parse. Thanks @saulcanina. (#1115) - **The discovery sweep issues its metadata syscalls concurrently.** Every dated command re-walks and re-stats every provider tree before it can decide what is already cached, and that sweep was strictly serial: one `readdir`, one `stat`, one `state.json` read at a time, per provider, one provider after another. On a 21k-file / 9-provider corpus it owned most of a warm run's wall clock while the machine sat idle waiting on the kernel. Provider discovery now runs across providers at once, and the four walks that dominate it (claude, codex, kimicode, grok) plus the Claude project-dir walk and both fingerprint passes fan out through a shared bounded-concurrency helper. Order is unchanged everywhere — each level is re-concatenated in registry/`readdir` order before anything reconciles against the cache — so what is discovered, in what sequence, is byte-identical to the serial walk. Two smaller cuts ride along: the Claude walk reads directory entries with their types so a plain file no longer costs a wasted `subagents/` probe, and the Codex result cache (a single file that can reach hundreds of MB) now shares one in-flight load between concurrent readers instead of letting each one re-read and re-parse it. Warm `codeburn today` on that corpus: 5.30s to 2.99s; a cold parse 34.7s to 31.1s. (#1104) - **A kill mid-way through a non-Claude provider phase no longer restarts that whole phase.** `scanProjectDirs` (Claude) has long taken a throttled `saveProgress` callback so a killed cold parse resumes from a warm cache; `parseProviderSources` (every other provider — codex, cursor, gemini, and the rest) did not, and only persisted at the whole-provider boundary. On a large single-provider corpus (a multi-GB codex history is the common case) an app-timeout SIGKILL, crash, or force-quit during that phase discarded everything parsed since the last provider finished, forcing the entire phase to re-parse from zero on the next run. `parseProviderSources` now takes the same callback, invoked once per source right after that source's cache entry lands (mirroring `scanProjectDirs`' placement, outside the per-file try/catch), on the same file-count/wall-clock throttle. A file only ever gets a fingerprint once it has fully parsed, so a mid-file kill can never leave a half-parsed file's entry looking complete on resume. - **Routed model ids price as the model they wrap, and an unknown vendor prefix no longer prices by blind stripping.** Token-plan and gateway spellings of the same model (`omniroute:`, `cp/`, `cline-pass/`, `cline-free/`, `cmd/`, `antigravity/`) are peeled and the remaining id is priced, so a Cline Pass or OmniRoute session shows a `~` estimate instead of $0. In exchange, `provider/model` is no longer treated as authority on its own: the leading segment is stripped only when it is a namespace the bundled pricing catalog itself uses (`anthropic/`, `openai/`, `google/`, `x-ai/`, `qwen/`, `moonshotai/`, `nousresearch/`, `xiaomi/`, `z-ai/`, and every other vendor prefix in the LiteLLM snapshot), one of the routing wrappers above, or one of the client-side spellings `kimi/`, `mimo/`, `zhipu/`, `litellm_proxy/` and `openai_like/`. Anything else stays unpriced and is reported as unpriced rather than inheriting the price of a same-named cloud row, and local-runner prefixes (`ollama/`, `lmstudio/`, `hosted_vllm/`, `local/`) are excluded on purpose so an unlisted local tag can never invent cloud spend. A user price override for the bare id wins over the catalog row a routed spelling would otherwise hit. diff --git a/src/session-output.ts b/src/session-output.ts index 19e8e31d..133c042d 100644 --- a/src/session-output.ts +++ b/src/session-output.ts @@ -19,19 +19,36 @@ export function modelBreakdownKey(call: { provider?: string; model?: string }): } /** - * Per-model displayed output, keyed like parser `modelBreakdown`. + * Prefer a key that already exists on this session's modelBreakdown. + * Parser sessions are keyed by getShortModelName. Fixtures and leftover + * summaries may still use the raw id. Inventing the other spelling + * creates a $0 / 0-call orphan that findUnpricedModels flags as Unpriced. + */ +export function resolveModelBreakdownKey( + call: { provider?: string; model?: string }, + breakdown: Record | undefined, +): string | undefined { + const derived = modelBreakdownKey(call) + if (derived && breakdown && Object.hasOwn(breakdown, derived)) return derived + if (call.model && breakdown && Object.hasOwn(breakdown, call.model)) return call.model + return derived ?? call.model +} + +/** + * Per-model displayed output, keyed like this session's `modelBreakdown`. * Call usage wins while provider identity is known. Aggregate-only / * stub sessions fall back to each existing bucket so a finite * sessionBillableOutputTokens cannot leave model Output Tokens at 0. */ export function sessionModelBillableOutputTokens(session: SessionSummary): Record { + const breakdown = session.modelBreakdown ?? {} const out: Record = {} let sawUsage = false for (const turn of session.turns ?? []) { for (const call of turn.assistantCalls ?? []) { if (!call.usage) continue sawUsage = true - const key = modelBreakdownKey(call) + const key = resolveModelBreakdownKey(call, breakdown) if (!key) continue out[key] = (out[key] ?? 0) + callBillableOutputTokens(call) } diff --git a/tests/billable-output-1115.test.ts b/tests/billable-output-1115.test.ts index 47be29dd..57c9c90e 100644 --- a/tests/billable-output-1115.test.ts +++ b/tests/billable-output-1115.test.ts @@ -251,6 +251,31 @@ describe('#1115 billableOutputTokens on report/optimize totals', () => { expect(out).toMatch(/DeepSeek v4 Pro[\s\S]*\b1\b[\s\S]*\b10\b/) }) + it('does not mint a $0 short-name orphan when modelBreakdown is the raw id', () => { + const session = makeSession('claude', 50, 0) + session.turns[0]!.assistantCalls[0]!.model = 'claude-opus-4-8' + session.modelBreakdown = { + 'claude-opus-4-8': { + calls: 2, + costUSD: 5, + savingsUSD: 0, + tokens: { + inputTokens: 100, + outputTokens: 50, + cacheReadInputTokens: 0, + cacheCreationInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 0, + }, + }, + } + expect(sessionModelBillableOutputTokens(session)).toEqual({ 'claude-opus-4-8': 50 }) + const out = renderOverview([makeProject(session)], { label: 'June 2026', color: false }) + expect(out).not.toContain('Unpriced') + expect(out).toContain('$5.00') + }) + it('aggregate-only model rows keep finite billable output', async () => { const aggregate = makeSession('grok', 10, 3) aggregate.turns = []