mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-08-25 00:14:40 +00:00
feat(cli): track local-model cost savings against a paid baseline (#421)
Add a new `localModelSavings` config and `codeburn model-savings` CLI that maps a local-model name (e.g. llama3.1:8b) to a paid baseline (e.g. gpt-4o). The local call still costs $0; the new `savingsUSD` field tracks the counterfactual spend avoided by running locally and is reported separately from `costUSD` everywhere a number is shown. * Parser normalization (`applyLocalModelSavings`) runs on Claude parse, direct provider calls, and the cached-call path. It forces `costUSD` to 0 and attaches `savingsUSD` + `savingsBaselineModel` + `isLocalSavings` on the `ParsedApiCall`. Local-savings wins for actual cost even when the same model is also in `modelAliases`. * Session, project, day, model, category, activity, skill, and subagent rollups all carry `savingsUSD` alongside `costUSD`. * `status --format json` adds `today.savings` and `month.savings`. * `status --format menubar-json` adds a `current.localModelSavings` block (totalUSD, calls, byModel, byProvider) plus savings on topModels, topProjects, topSessions, topActivities, and history daily entries. Schema fields default-decode for backward compat. * `report --format json` adds savings across overview/daily/ projects/models/activities/skills/subagents/topSessions, with the active paid baseline name on each model row. * `models` command gains a `Saved` column on table/markdown/CSV and a `savingsUSD`/`savingsBaselineModel` pair in JSON. Default `--min-cost 0.01` filter now ORs in `savingsUSD >= minCost` so local models with $0 actual cost but >0 savings still surface. * CSV/JSON exports add a `Saved (CODE)` column on summary/daily/ models/projects/sessions. * Dashboard TUI shows a green 'saved $X by local models' footer line in the overview when any savings are present. * macOS Swift payload gains a `LocalModelSavings` Codable block and savings fields on every model/activity/session/daily struct. Hero shows a green leaf 'Saved $X' caption, models section gets a green `Saved` column. `swift build` clean. * GNOME indicator adds 'saved $X' to the hero meta line and a `codeburn-model-saved` column to the model row. * Daily cache schema bumped to v8 (`savingsUSD` on day/model/ category/provider). `savingsConfigHash` invalidates the cache when the user changes their baseline mapping so historical saved-spend numbers never lie about a stale baseline. * Defensive `Object.hasOwn` lookup in `getLocalSavingsBaseline` blocks the prototype-pollution test that previously surfaced via the savings path with a hostile `__proto__` model name. * New tests (5 files, 25 tests, 549 lines) cover pricing helpers, end-to-end parser normalization, day aggregator savings, menubar payload savings, CLI set/list/remove, and daily-cache hash invalidation. Existing tests for daily-cache / day-aggregator / models-report updated for the new fields. Full vitest suite: 1028/1028 passing across 73 test files. `tsc --noEmit` clean. `npm run build` clean. (Note: `mac/Tests` has a pre-existing `no such module 'Testing'` environment error on the installed Swift toolchain, confirmed on `main` before this PR; not caused by these changes.)
This commit is contained in:
parent
69b1736365
commit
8ded6ad6fd
25 changed files with 1308 additions and 113 deletions
|
|
@ -36,6 +36,7 @@ function pct(n: number, total: number): number {
|
|||
|
||||
type DailyAgg = {
|
||||
cost: number
|
||||
savings: number
|
||||
calls: number
|
||||
input: number
|
||||
output: number
|
||||
|
|
@ -52,11 +53,12 @@ function buildDailyRows(projects: ProjectSummary[], period: string): Row[] {
|
|||
if (!turn.timestamp) continue
|
||||
const day = dateKey(turn.timestamp)
|
||||
if (!daily[day]) {
|
||||
daily[day] = { cost: 0, calls: 0, input: 0, output: 0, cacheRead: 0, cacheWrite: 0, sessions: new Set() }
|
||||
daily[day] = { cost: 0, savings: 0, calls: 0, input: 0, output: 0, cacheRead: 0, cacheWrite: 0, sessions: new Set() }
|
||||
}
|
||||
daily[day].sessions.add(session.sessionId)
|
||||
for (const call of turn.assistantCalls) {
|
||||
daily[day].cost += call.costUSD
|
||||
daily[day].savings += call.savingsUSD ?? 0
|
||||
daily[day].calls++
|
||||
daily[day].input += call.usage.inputTokens
|
||||
daily[day].output += call.usage.outputTokens
|
||||
|
|
@ -71,6 +73,7 @@ function buildDailyRows(projects: ProjectSummary[], period: string): Row[] {
|
|||
Period: period,
|
||||
Date: date,
|
||||
[`Cost (${code})`]: roundForActiveCurrency(convertCost(d.cost)),
|
||||
[`Saved (${code})`]: roundForActiveCurrency(convertCost(d.savings)),
|
||||
'API Calls': d.calls,
|
||||
Sessions: d.sessions.size,
|
||||
'Input Tokens': d.input,
|
||||
|
|
@ -105,14 +108,15 @@ function buildActivityRows(projects: ProjectSummary[], period: string): Row[] {
|
|||
}
|
||||
|
||||
function buildModelRows(projects: ProjectSummary[], period: string): Row[] {
|
||||
const modelTotals: Record<string, { calls: number; cost: number; input: number; output: number; cacheRead: number; cacheWrite: number }> = {}
|
||||
const modelTotals: Record<string, { calls: number; cost: number; savings: number; input: number; output: number; cacheRead: number; cacheWrite: number }> = {}
|
||||
const modelEfficiency = aggregateModelEfficiency(projects)
|
||||
for (const project of projects) {
|
||||
for (const session of project.sessions) {
|
||||
for (const [model, d] of Object.entries(session.modelBreakdown)) {
|
||||
if (!modelTotals[model]) modelTotals[model] = { calls: 0, cost: 0, input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }
|
||||
if (!modelTotals[model]) modelTotals[model] = { calls: 0, cost: 0, savings: 0, input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }
|
||||
modelTotals[model].calls += d.calls
|
||||
modelTotals[model].cost += d.costUSD
|
||||
modelTotals[model].savings += d.savingsUSD
|
||||
modelTotals[model].input += d.tokens.inputTokens
|
||||
modelTotals[model].output += d.tokens.outputTokens
|
||||
modelTotals[model].cacheRead += d.tokens.cacheReadInputTokens ?? 0
|
||||
|
|
@ -124,13 +128,14 @@ function buildModelRows(projects: ProjectSummary[], period: string): Row[] {
|
|||
const { code } = getCurrency()
|
||||
return Object.entries(modelTotals)
|
||||
.filter(([name]) => name !== '<synthetic>')
|
||||
.sort(([, a], [, b]) => b.cost - a.cost)
|
||||
.sort(([, a], [, b]) => (b.cost + b.savings) - (a.cost + a.savings))
|
||||
.map(([model, d]) => {
|
||||
const efficiency = modelEfficiency.get(model)
|
||||
return {
|
||||
Period: period,
|
||||
Model: model,
|
||||
[`Cost (${code})`]: roundForActiveCurrency(convertCost(d.cost)),
|
||||
[`Saved (${code})`]: roundForActiveCurrency(convertCost(d.savings)),
|
||||
'Share (%)': pct(d.cost, totalCost),
|
||||
'API Calls': d.calls,
|
||||
'Edit Turns': efficiency?.editTurns ?? 0,
|
||||
|
|
@ -190,10 +195,11 @@ function buildProjectRows(projects: ProjectSummary[]): Row[] {
|
|||
const total = projects.reduce((s, p) => s + p.totalCostUSD, 0)
|
||||
return projects
|
||||
.slice()
|
||||
.sort((a, b) => b.totalCostUSD - a.totalCostUSD)
|
||||
.sort((a, b) => (b.totalCostUSD + b.totalSavingsUSD) - (a.totalCostUSD + a.totalSavingsUSD))
|
||||
.map(p => ({
|
||||
Project: p.projectPath,
|
||||
[`Cost (${code})`]: roundForActiveCurrency(convertCost(p.totalCostUSD)),
|
||||
[`Saved (${code})`]: roundForActiveCurrency(convertCost(p.totalSavingsUSD)),
|
||||
[`Avg/Session (${code})`]: p.sessions.length > 0 ? roundForActiveCurrency(convertCost(p.totalCostUSD / p.sessions.length)) : '',
|
||||
'Share (%)': pct(p.totalCostUSD, total),
|
||||
'API Calls': p.totalApiCalls,
|
||||
|
|
@ -211,12 +217,13 @@ function buildSessionRows(projects: ProjectSummary[]): Row[] {
|
|||
'Session ID': s.sessionId,
|
||||
'Started At': s.firstTimestamp ?? '',
|
||||
[`Cost (${code})`]: roundForActiveCurrency(convertCost(s.totalCostUSD)),
|
||||
[`Saved (${code})`]: roundForActiveCurrency(convertCost(s.totalSavingsUSD)),
|
||||
'API Calls': s.apiCalls,
|
||||
Turns: s.turns.length,
|
||||
})
|
||||
}
|
||||
}
|
||||
return rows.sort((a, b) => (b[`Cost (${code})`] as number) - (a[`Cost (${code})`] as number))
|
||||
return rows.sort((a, b) => ((b[`Cost (${code})`] as number) + (b[`Saved (${code})`] as number)) - ((a[`Cost (${code})`] as number) + (a[`Saved (${code})`] as number)))
|
||||
}
|
||||
|
||||
export type PeriodExport = {
|
||||
|
|
@ -228,12 +235,14 @@ function buildSummaryRows(periods: PeriodExport[]): Row[] {
|
|||
const { code } = getCurrency()
|
||||
return periods.map(p => {
|
||||
const cost = p.projects.reduce((s, proj) => s + proj.totalCostUSD, 0)
|
||||
const savings = p.projects.reduce((s, proj) => s + proj.totalSavingsUSD, 0)
|
||||
const calls = p.projects.reduce((s, proj) => s + proj.totalApiCalls, 0)
|
||||
const sessions = p.projects.reduce((s, proj) => s + proj.sessions.length, 0)
|
||||
const projectCount = p.projects.filter(proj => proj.totalCostUSD > 0).length
|
||||
const projectCount = p.projects.filter(proj => proj.totalCostUSD > 0 || proj.totalSavingsUSD > 0).length
|
||||
return {
|
||||
Period: p.label,
|
||||
[`Cost (${code})`]: roundForActiveCurrency(convertCost(cost)),
|
||||
[`Saved (${code})`]: roundForActiveCurrency(convertCost(savings)),
|
||||
'API Calls': calls,
|
||||
Sessions: sessions,
|
||||
Projects: projectCount,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue