codeburn/tests/day-aggregator-savings.test.ts
Justin Gheorghe 8ded6ad6fd feat(cli): track local-model cost savings against a paid baseline (#421)
Add a new `localModelSavings` config and `codeburn model-savings` CLI
that maps a local-model name (e.g. llama3.1:8b) to a paid baseline
(e.g. gpt-4o). The local call still costs $0; the new `savingsUSD`
field tracks the counterfactual spend avoided by running locally and
is reported separately from `costUSD` everywhere a number is shown.

* Parser normalization (`applyLocalModelSavings`) runs on Claude
  parse, direct provider calls, and the cached-call path. It forces
  `costUSD` to 0 and attaches `savingsUSD` + `savingsBaselineModel`
  + `isLocalSavings` on the `ParsedApiCall`. Local-savings wins for
  actual cost even when the same model is also in `modelAliases`.
* Session, project, day, model, category, activity, skill, and
  subagent rollups all carry `savingsUSD` alongside `costUSD`.
* `status --format json` adds `today.savings` and `month.savings`.
* `status --format menubar-json` adds a `current.localModelSavings`
  block (totalUSD, calls, byModel, byProvider) plus savings on
  topModels, topProjects, topSessions, topActivities, and history
  daily entries. Schema fields default-decode for backward compat.
* `report --format json` adds savings across overview/daily/
  projects/models/activities/skills/subagents/topSessions, with
  the active paid baseline name on each model row.
* `models` command gains a `Saved` column on table/markdown/CSV
  and a `savingsUSD`/`savingsBaselineModel` pair in JSON. Default
  `--min-cost 0.01` filter now ORs in `savingsUSD >= minCost` so
  local models with $0 actual cost but >0 savings still surface.
* CSV/JSON exports add a `Saved (CODE)` column on summary/daily/
  models/projects/sessions.
* Dashboard TUI shows a green 'saved $X by local models' footer
  line in the overview when any savings are present.
* macOS Swift payload gains a `LocalModelSavings` Codable block
  and savings fields on every model/activity/session/daily
  struct. Hero shows a green leaf 'Saved $X' caption, models
  section gets a green `Saved` column. `swift build` clean.
* GNOME indicator adds 'saved $X' to the hero meta line and a
  `codeburn-model-saved` column to the model row.
* Daily cache schema bumped to v8 (`savingsUSD` on day/model/
  category/provider). `savingsConfigHash` invalidates the cache
  when the user changes their baseline mapping so historical
  saved-spend numbers never lie about a stale baseline.
* Defensive `Object.hasOwn` lookup in `getLocalSavingsBaseline`
  blocks the prototype-pollution test that previously surfaced via
  the savings path with a hostile `__proto__` model name.
* New tests (5 files, 25 tests, 549 lines) cover pricing helpers,
  end-to-end parser normalization, day aggregator savings,
  menubar payload savings, CLI set/list/remove, and
  daily-cache hash invalidation. Existing tests for daily-cache
  / day-aggregator / models-report updated for the new fields.
  Full vitest suite: 1028/1028 passing across 73 test files.
  `tsc --noEmit` clean. `npm run build` clean.
  (Note: `mac/Tests` has a pre-existing `no such module 'Testing'`
  environment error on the installed Swift toolchain, confirmed
  on `main` before this PR; not caused by these changes.)
2026-06-01 11:06:39 +03:00

150 lines
5.6 KiB
TypeScript

import { describe, expect, it } from 'vitest'
import { aggregateProjectsIntoDays, buildPeriodDataFromDays } from '../src/day-aggregator.js'
import type { ParsedApiCall, ProjectSummary, SessionSummary, Turn } from '../src/types.js'
function makeCall(timestamp: string, opts: { costUSD: number; savingsUSD?: number; savingsBaselineModel?: string; model?: string }): ParsedApiCall {
return {
provider: 'claude',
model: opts.model ?? 'local-model',
usage: {
inputTokens: 100,
outputTokens: 200,
cacheCreationInputTokens: 0,
cacheReadInputTokens: 50,
cachedInputTokens: 0,
reasoningTokens: 0,
webSearchRequests: 0,
},
costUSD: opts.costUSD,
savingsUSD: opts.savingsUSD,
savingsBaselineModel: opts.savingsBaselineModel,
tools: [],
mcpTools: [],
skills: [],
subagentTypes: [],
hasAgentSpawn: false,
hasPlanMode: false,
speed: 'standard',
timestamp,
bashCommands: [],
deduplicationKey: `dk-${timestamp}-${opts.costUSD}-${opts.savingsUSD ?? 0}`,
}
}
function makeTurn(timestamp: string, calls: ParsedApiCall[], category: string = 'coding'): Turn {
return {
userMessage: 'u',
timestamp,
sessionId: 's',
category: category as Turn['category'],
retries: 0,
hasEdits: false,
assistantCalls: calls,
} as Turn
}
function makeSession(sessions: SessionSummary[]): ProjectSummary {
const totalCostUSD = sessions.reduce((s, sess) => s + sess.totalCostUSD, 0)
const totalSavingsUSD = sessions.reduce((s, sess) => s + sess.totalSavingsUSD, 0)
const totalApiCalls = sessions.reduce((s, sess) => s + sess.apiCalls, 0)
return {
project: 'p',
projectPath: '/p',
sessions,
totalCostUSD,
totalSavingsUSD,
totalApiCalls,
}
}
describe('aggregateProjectsIntoDays: savings totals', () => {
it('rolls up day, model, category, and provider savings separately from cost', () => {
const turn = makeTurn('2026-04-10T10:00:00', [
makeCall('2026-04-10T10:00:00', { costUSD: 0, savingsUSD: 5, savingsBaselineModel: 'gpt-4o' }),
])
const turn2 = makeTurn('2026-04-10T10:01:00', [
makeCall('2026-04-10T10:01:00', { costUSD: 2, savingsUSD: 0, model: 'gpt-4o' }),
])
const project: ProjectSummary = {
project: 'p',
projectPath: '/p',
sessions: [{
sessionId: 's1',
project: 'p',
firstTimestamp: '2026-04-10T10:00:00',
lastTimestamp: '2026-04-10T10:01:00',
totalCostUSD: 2,
totalSavingsUSD: 5,
totalInputTokens: 200,
totalOutputTokens: 400,
totalCacheReadTokens: 100,
totalCacheWriteTokens: 0,
apiCalls: 2,
turns: [turn, turn2],
modelBreakdown: { 'Local Model': { calls: 1, costUSD: 0, savingsUSD: 5, tokens: { inputTokens: 100, outputTokens: 200, cacheCreationInputTokens: 0, cacheReadInputTokens: 50, cachedInputTokens: 0, reasoningTokens: 0, webSearchRequests: 0 } }, 'gpt-4o': { calls: 1, costUSD: 2, savingsUSD: 0, tokens: { inputTokens: 100, outputTokens: 200, cacheCreationInputTokens: 0, cacheReadInputTokens: 50, cachedInputTokens: 0, reasoningTokens: 0, webSearchRequests: 0 } } },
toolBreakdown: {}, mcpBreakdown: {}, bashBreakdown: {},
categoryBreakdown: { coding: { turns: 1, costUSD: 2, savingsUSD: 5, retries: 0, editTurns: 0, oneShotTurns: 0 } },
skillBreakdown: {}, subagentBreakdown: {},
}],
totalCostUSD: 2,
totalSavingsUSD: 5,
totalApiCalls: 2,
}
const days = aggregateProjectsIntoDays([project])
expect(days).toHaveLength(1)
const day = days[0]!
expect(day.cost).toBe(2)
expect(day.savingsUSD).toBe(5)
expect(day.models['local-model']).toMatchObject({ calls: 1, cost: 0, savingsUSD: 5 })
expect(day.models['gpt-4o']).toMatchObject({ calls: 1, cost: 2, savingsUSD: 0 })
expect(day.providers['claude']).toMatchObject({ calls: 2, cost: 2, savingsUSD: 5 })
expect(day.categories.coding).toMatchObject({ turns: 2, cost: 2, savingsUSD: 5 })
})
})
describe('buildPeriodDataFromDays: savings totals', () => {
it('threads savings through to model and category rollups', () => {
const days = [
{
date: '2026-04-09',
cost: 2,
savingsUSD: 5,
calls: 1,
sessions: 1,
inputTokens: 100,
outputTokens: 200,
cacheReadTokens: 0,
cacheWriteTokens: 0,
editTurns: 0,
oneShotTurns: 0,
models: { 'local-model': { calls: 1, cost: 0, savingsUSD: 5, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 } },
categories: { coding: { turns: 1, cost: 0, savingsUSD: 5, editTurns: 0, oneShotTurns: 0 } },
providers: { claude: { calls: 1, cost: 0, savingsUSD: 5 } },
},
{
date: '2026-04-10',
cost: 3,
savingsUSD: 0,
calls: 1,
sessions: 1,
inputTokens: 100,
outputTokens: 200,
cacheReadTokens: 0,
cacheWriteTokens: 0,
editTurns: 0,
oneShotTurns: 0,
models: { 'gpt-4o': { calls: 1, cost: 3, savingsUSD: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 } },
categories: { coding: { turns: 1, cost: 3, savingsUSD: 0, editTurns: 0, oneShotTurns: 0 } },
providers: { claude: { calls: 1, cost: 3, savingsUSD: 0 } },
},
]
const pd = buildPeriodDataFromDays(days, '7 Days')
expect(pd.savingsUSD).toBe(5)
const coding = pd.categories.find(c => c.name === 'Coding')!
expect(coding.savingsUSD).toBe(5)
const local = pd.models.find(m => m.name === 'local-model')!
expect(local.savingsUSD).toBe(5)
expect(local.cost).toBe(0)
})
})