mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-08-25 08:22:42 +00:00
Add a new `localModelSavings` config and `codeburn model-savings` CLI that maps a local-model name (e.g. llama3.1:8b) to a paid baseline (e.g. gpt-4o). The local call still costs $0; the new `savingsUSD` field tracks the counterfactual spend avoided by running locally and is reported separately from `costUSD` everywhere a number is shown. * Parser normalization (`applyLocalModelSavings`) runs on Claude parse, direct provider calls, and the cached-call path. It forces `costUSD` to 0 and attaches `savingsUSD` + `savingsBaselineModel` + `isLocalSavings` on the `ParsedApiCall`. Local-savings wins for actual cost even when the same model is also in `modelAliases`. * Session, project, day, model, category, activity, skill, and subagent rollups all carry `savingsUSD` alongside `costUSD`. * `status --format json` adds `today.savings` and `month.savings`. * `status --format menubar-json` adds a `current.localModelSavings` block (totalUSD, calls, byModel, byProvider) plus savings on topModels, topProjects, topSessions, topActivities, and history daily entries. Schema fields default-decode for backward compat. * `report --format json` adds savings across overview/daily/ projects/models/activities/skills/subagents/topSessions, with the active paid baseline name on each model row. * `models` command gains a `Saved` column on table/markdown/CSV and a `savingsUSD`/`savingsBaselineModel` pair in JSON. Default `--min-cost 0.01` filter now ORs in `savingsUSD >= minCost` so local models with $0 actual cost but >0 savings still surface. * CSV/JSON exports add a `Saved (CODE)` column on summary/daily/ models/projects/sessions. * Dashboard TUI shows a green 'saved $X by local models' footer line in the overview when any savings are present. * macOS Swift payload gains a `LocalModelSavings` Codable block and savings fields on every model/activity/session/daily struct. Hero shows a green leaf 'Saved $X' caption, models section gets a green `Saved` column. `swift build` clean. * GNOME indicator adds 'saved $X' to the hero meta line and a `codeburn-model-saved` column to the model row. * Daily cache schema bumped to v8 (`savingsUSD` on day/model/ category/provider). `savingsConfigHash` invalidates the cache when the user changes their baseline mapping so historical saved-spend numbers never lie about a stale baseline. * Defensive `Object.hasOwn` lookup in `getLocalSavingsBaseline` blocks the prototype-pollution test that previously surfaced via the savings path with a hostile `__proto__` model name. * New tests (5 files, 25 tests, 549 lines) cover pricing helpers, end-to-end parser normalization, day aggregator savings, menubar payload savings, CLI set/list/remove, and daily-cache hash invalidation. Existing tests for daily-cache / day-aggregator / models-report updated for the new fields. Full vitest suite: 1028/1028 passing across 73 test files. `tsc --noEmit` clean. `npm run build` clean. (Note: `mac/Tests` has a pre-existing `no such module 'Testing'` environment error on the installed Swift toolchain, confirmed on `main` before this PR; not caused by these changes.)
150 lines
5.6 KiB
TypeScript
150 lines
5.6 KiB
TypeScript
import { describe, expect, it } from 'vitest'
|
|
|
|
import { aggregateProjectsIntoDays, buildPeriodDataFromDays } from '../src/day-aggregator.js'
|
|
import type { ParsedApiCall, ProjectSummary, SessionSummary, Turn } from '../src/types.js'
|
|
|
|
function makeCall(timestamp: string, opts: { costUSD: number; savingsUSD?: number; savingsBaselineModel?: string; model?: string }): ParsedApiCall {
|
|
return {
|
|
provider: 'claude',
|
|
model: opts.model ?? 'local-model',
|
|
usage: {
|
|
inputTokens: 100,
|
|
outputTokens: 200,
|
|
cacheCreationInputTokens: 0,
|
|
cacheReadInputTokens: 50,
|
|
cachedInputTokens: 0,
|
|
reasoningTokens: 0,
|
|
webSearchRequests: 0,
|
|
},
|
|
costUSD: opts.costUSD,
|
|
savingsUSD: opts.savingsUSD,
|
|
savingsBaselineModel: opts.savingsBaselineModel,
|
|
tools: [],
|
|
mcpTools: [],
|
|
skills: [],
|
|
subagentTypes: [],
|
|
hasAgentSpawn: false,
|
|
hasPlanMode: false,
|
|
speed: 'standard',
|
|
timestamp,
|
|
bashCommands: [],
|
|
deduplicationKey: `dk-${timestamp}-${opts.costUSD}-${opts.savingsUSD ?? 0}`,
|
|
}
|
|
}
|
|
|
|
function makeTurn(timestamp: string, calls: ParsedApiCall[], category: string = 'coding'): Turn {
|
|
return {
|
|
userMessage: 'u',
|
|
timestamp,
|
|
sessionId: 's',
|
|
category: category as Turn['category'],
|
|
retries: 0,
|
|
hasEdits: false,
|
|
assistantCalls: calls,
|
|
} as Turn
|
|
}
|
|
|
|
function makeSession(sessions: SessionSummary[]): ProjectSummary {
|
|
const totalCostUSD = sessions.reduce((s, sess) => s + sess.totalCostUSD, 0)
|
|
const totalSavingsUSD = sessions.reduce((s, sess) => s + sess.totalSavingsUSD, 0)
|
|
const totalApiCalls = sessions.reduce((s, sess) => s + sess.apiCalls, 0)
|
|
return {
|
|
project: 'p',
|
|
projectPath: '/p',
|
|
sessions,
|
|
totalCostUSD,
|
|
totalSavingsUSD,
|
|
totalApiCalls,
|
|
}
|
|
}
|
|
|
|
describe('aggregateProjectsIntoDays: savings totals', () => {
|
|
it('rolls up day, model, category, and provider savings separately from cost', () => {
|
|
const turn = makeTurn('2026-04-10T10:00:00', [
|
|
makeCall('2026-04-10T10:00:00', { costUSD: 0, savingsUSD: 5, savingsBaselineModel: 'gpt-4o' }),
|
|
])
|
|
const turn2 = makeTurn('2026-04-10T10:01:00', [
|
|
makeCall('2026-04-10T10:01:00', { costUSD: 2, savingsUSD: 0, model: 'gpt-4o' }),
|
|
])
|
|
const project: ProjectSummary = {
|
|
project: 'p',
|
|
projectPath: '/p',
|
|
sessions: [{
|
|
sessionId: 's1',
|
|
project: 'p',
|
|
firstTimestamp: '2026-04-10T10:00:00',
|
|
lastTimestamp: '2026-04-10T10:01:00',
|
|
totalCostUSD: 2,
|
|
totalSavingsUSD: 5,
|
|
totalInputTokens: 200,
|
|
totalOutputTokens: 400,
|
|
totalCacheReadTokens: 100,
|
|
totalCacheWriteTokens: 0,
|
|
apiCalls: 2,
|
|
turns: [turn, turn2],
|
|
modelBreakdown: { 'Local Model': { calls: 1, costUSD: 0, savingsUSD: 5, tokens: { inputTokens: 100, outputTokens: 200, cacheCreationInputTokens: 0, cacheReadInputTokens: 50, cachedInputTokens: 0, reasoningTokens: 0, webSearchRequests: 0 } }, 'gpt-4o': { calls: 1, costUSD: 2, savingsUSD: 0, tokens: { inputTokens: 100, outputTokens: 200, cacheCreationInputTokens: 0, cacheReadInputTokens: 50, cachedInputTokens: 0, reasoningTokens: 0, webSearchRequests: 0 } } },
|
|
toolBreakdown: {}, mcpBreakdown: {}, bashBreakdown: {},
|
|
categoryBreakdown: { coding: { turns: 1, costUSD: 2, savingsUSD: 5, retries: 0, editTurns: 0, oneShotTurns: 0 } },
|
|
skillBreakdown: {}, subagentBreakdown: {},
|
|
}],
|
|
totalCostUSD: 2,
|
|
totalSavingsUSD: 5,
|
|
totalApiCalls: 2,
|
|
}
|
|
const days = aggregateProjectsIntoDays([project])
|
|
expect(days).toHaveLength(1)
|
|
const day = days[0]!
|
|
expect(day.cost).toBe(2)
|
|
expect(day.savingsUSD).toBe(5)
|
|
expect(day.models['local-model']).toMatchObject({ calls: 1, cost: 0, savingsUSD: 5 })
|
|
expect(day.models['gpt-4o']).toMatchObject({ calls: 1, cost: 2, savingsUSD: 0 })
|
|
expect(day.providers['claude']).toMatchObject({ calls: 2, cost: 2, savingsUSD: 5 })
|
|
expect(day.categories.coding).toMatchObject({ turns: 2, cost: 2, savingsUSD: 5 })
|
|
})
|
|
})
|
|
|
|
describe('buildPeriodDataFromDays: savings totals', () => {
|
|
it('threads savings through to model and category rollups', () => {
|
|
const days = [
|
|
{
|
|
date: '2026-04-09',
|
|
cost: 2,
|
|
savingsUSD: 5,
|
|
calls: 1,
|
|
sessions: 1,
|
|
inputTokens: 100,
|
|
outputTokens: 200,
|
|
cacheReadTokens: 0,
|
|
cacheWriteTokens: 0,
|
|
editTurns: 0,
|
|
oneShotTurns: 0,
|
|
models: { 'local-model': { calls: 1, cost: 0, savingsUSD: 5, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 } },
|
|
categories: { coding: { turns: 1, cost: 0, savingsUSD: 5, editTurns: 0, oneShotTurns: 0 } },
|
|
providers: { claude: { calls: 1, cost: 0, savingsUSD: 5 } },
|
|
},
|
|
{
|
|
date: '2026-04-10',
|
|
cost: 3,
|
|
savingsUSD: 0,
|
|
calls: 1,
|
|
sessions: 1,
|
|
inputTokens: 100,
|
|
outputTokens: 200,
|
|
cacheReadTokens: 0,
|
|
cacheWriteTokens: 0,
|
|
editTurns: 0,
|
|
oneShotTurns: 0,
|
|
models: { 'gpt-4o': { calls: 1, cost: 3, savingsUSD: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 } },
|
|
categories: { coding: { turns: 1, cost: 3, savingsUSD: 0, editTurns: 0, oneShotTurns: 0 } },
|
|
providers: { claude: { calls: 1, cost: 3, savingsUSD: 0 } },
|
|
},
|
|
]
|
|
const pd = buildPeriodDataFromDays(days, '7 Days')
|
|
expect(pd.savingsUSD).toBe(5)
|
|
const coding = pd.categories.find(c => c.name === 'Coding')!
|
|
expect(coding.savingsUSD).toBe(5)
|
|
const local = pd.models.find(m => m.name === 'local-model')!
|
|
expect(local.savingsUSD).toBe(5)
|
|
expect(local.cost).toBe(0)
|
|
})
|
|
})
|