mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-07-10 01:29:41 +00:00
8ded6ad6fd
Add a new `localModelSavings` config and `codeburn model-savings` CLI that maps a local-model name (e.g. llama3.1:8b) to a paid baseline (e.g. gpt-4o). The local call still costs $0; the new `savingsUSD` field tracks the counterfactual spend avoided by running locally and is reported separately from `costUSD` everywhere a number is shown. * Parser normalization (`applyLocalModelSavings`) runs on Claude parse, direct provider calls, and the cached-call path. It forces `costUSD` to 0 and attaches `savingsUSD` + `savingsBaselineModel` + `isLocalSavings` on the `ParsedApiCall`. Local-savings wins for actual cost even when the same model is also in `modelAliases`. * Session, project, day, model, category, activity, skill, and subagent rollups all carry `savingsUSD` alongside `costUSD`. * `status --format json` adds `today.savings` and `month.savings`. * `status --format menubar-json` adds a `current.localModelSavings` block (totalUSD, calls, byModel, byProvider) plus savings on topModels, topProjects, topSessions, topActivities, and history daily entries. Schema fields default-decode for backward compat. * `report --format json` adds savings across overview/daily/ projects/models/activities/skills/subagents/topSessions, with the active paid baseline name on each model row. * `models` command gains a `Saved` column on table/markdown/CSV and a `savingsUSD`/`savingsBaselineModel` pair in JSON. Default `--min-cost 0.01` filter now ORs in `savingsUSD >= minCost` so local models with $0 actual cost but >0 savings still surface. * CSV/JSON exports add a `Saved (CODE)` column on summary/daily/ models/projects/sessions. * Dashboard TUI shows a green 'saved $X by local models' footer line in the overview when any savings are present. * macOS Swift payload gains a `LocalModelSavings` Codable block and savings fields on every model/activity/session/daily struct. Hero shows a green leaf 'Saved $X' caption, models section gets a green `Saved` column. `swift build` clean. * GNOME indicator adds 'saved $X' to the hero meta line and a `codeburn-model-saved` column to the model row. * Daily cache schema bumped to v8 (`savingsUSD` on day/model/ category/provider). `savingsConfigHash` invalidates the cache when the user changes their baseline mapping so historical saved-spend numbers never lie about a stale baseline. * Defensive `Object.hasOwn` lookup in `getLocalSavingsBaseline` blocks the prototype-pollution test that previously surfaced via the savings path with a hostile `__proto__` model name. * New tests (5 files, 25 tests, 549 lines) cover pricing helpers, end-to-end parser normalization, day aggregator savings, menubar payload savings, CLI set/list/remove, and daily-cache hash invalidation. Existing tests for daily-cache / day-aggregator / models-report updated for the new fields. Full vitest suite: 1028/1028 passing across 73 test files. `tsc --noEmit` clean. `npm run build` clean. (Note: `mac/Tests` has a pre-existing `no such module 'Testing'` environment error on the installed Swift toolchain, confirmed on `main` before this PR; not caused by these changes.)
76 lines
2.6 KiB
TypeScript
76 lines
2.6 KiB
TypeScript
import { mkdtemp, readFile, rm } from 'node:fs/promises'
|
|
import { tmpdir } from 'node:os'
|
|
import { join } from 'node:path'
|
|
import { spawnSync } from 'node:child_process'
|
|
|
|
import { describe, it, expect } from 'vitest'
|
|
|
|
const CLI_TIMEOUT_MS = 10_000
|
|
|
|
function runCli(args: string[], home: string) {
|
|
return spawnSync(process.execPath, ['--import', 'tsx', 'src/cli.ts', ...args], {
|
|
cwd: process.cwd(),
|
|
env: {
|
|
...process.env,
|
|
HOME: home,
|
|
USERPROFILE: home,
|
|
HOMEPATH: home,
|
|
HOMEDRIVE: '',
|
|
},
|
|
encoding: 'utf-8',
|
|
})
|
|
}
|
|
|
|
function readConfig(home: string): Promise<Record<string, unknown>> {
|
|
return readFile(join(home, '.config', 'codeburn', 'config.json'), 'utf-8')
|
|
.then(raw => JSON.parse(raw) as Record<string, unknown>)
|
|
}
|
|
|
|
describe('codeburn model-savings command', () => {
|
|
it('saves, lists, and removes a local-model savings mapping', async () => {
|
|
const home = await mkdtemp(join(tmpdir(), 'codeburn-cli-savings-'))
|
|
try {
|
|
const set = runCli(['model-savings', 'llama3.1:8b', 'gpt-4o'], home)
|
|
expect(set.status).toBe(0)
|
|
expect(set.stdout).toContain('llama3.1:8b -> gpt-4o')
|
|
|
|
const saved = await readConfig(home)
|
|
expect(saved.localModelSavings).toEqual({ 'llama3.1:8b': 'gpt-4o' })
|
|
|
|
const list = runCli(['model-savings', '--list'], home)
|
|
expect(list.status).toBe(0)
|
|
expect(list.stdout).toContain('llama3.1:8b -> gpt-4o')
|
|
|
|
const remove = runCli(['model-savings', '--remove', 'llama3.1:8b'], home)
|
|
expect(remove.status).toBe(0)
|
|
|
|
const after = await readConfig(home)
|
|
expect(after.localModelSavings).toBeUndefined()
|
|
} finally {
|
|
await rm(home, { recursive: true, force: true })
|
|
}
|
|
}, CLI_TIMEOUT_MS)
|
|
|
|
it('warns when the same model is also configured in modelAliases', async () => {
|
|
const home = await mkdtemp(join(tmpdir(), 'codeburn-cli-savings-'))
|
|
try {
|
|
expect(runCli(['model-alias', 'llama3.1:8b', 'gpt-4o'], home).status).toBe(0)
|
|
const set = runCli(['model-savings', 'llama3.1:8b', 'gpt-4o'], home)
|
|
expect(set.status).toBe(0)
|
|
expect(set.stdout).toContain('savings take precedence')
|
|
} finally {
|
|
await rm(home, { recursive: true, force: true })
|
|
}
|
|
}, CLI_TIMEOUT_MS)
|
|
|
|
it('rejects a remove for an unknown mapping', async () => {
|
|
const home = await mkdtemp(join(tmpdir(), 'codeburn-cli-savings-'))
|
|
try {
|
|
const result = runCli(['model-savings', '--remove', 'unknown:1b'], home)
|
|
expect(result.status).toBe(1)
|
|
expect(result.stderr).toContain('No savings mapping found')
|
|
} finally {
|
|
await rm(home, { recursive: true, force: true })
|
|
}
|
|
}, CLI_TIMEOUT_MS)
|
|
})
|