mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-08-10 17:14:48 +00:00
8ded6ad6fd
Add a new `localModelSavings` config and `codeburn model-savings` CLI that maps a local-model name (e.g. llama3.1:8b) to a paid baseline (e.g. gpt-4o). The local call still costs $0; the new `savingsUSD` field tracks the counterfactual spend avoided by running locally and is reported separately from `costUSD` everywhere a number is shown. * Parser normalization (`applyLocalModelSavings`) runs on Claude parse, direct provider calls, and the cached-call path. It forces `costUSD` to 0 and attaches `savingsUSD` + `savingsBaselineModel` + `isLocalSavings` on the `ParsedApiCall`. Local-savings wins for actual cost even when the same model is also in `modelAliases`. * Session, project, day, model, category, activity, skill, and subagent rollups all carry `savingsUSD` alongside `costUSD`. * `status --format json` adds `today.savings` and `month.savings`. * `status --format menubar-json` adds a `current.localModelSavings` block (totalUSD, calls, byModel, byProvider) plus savings on topModels, topProjects, topSessions, topActivities, and history daily entries. Schema fields default-decode for backward compat. * `report --format json` adds savings across overview/daily/ projects/models/activities/skills/subagents/topSessions, with the active paid baseline name on each model row. * `models` command gains a `Saved` column on table/markdown/CSV and a `savingsUSD`/`savingsBaselineModel` pair in JSON. Default `--min-cost 0.01` filter now ORs in `savingsUSD >= minCost` so local models with $0 actual cost but >0 savings still surface. * CSV/JSON exports add a `Saved (CODE)` column on summary/daily/ models/projects/sessions. * Dashboard TUI shows a green 'saved $X by local models' footer line in the overview when any savings are present. * macOS Swift payload gains a `LocalModelSavings` Codable block and savings fields on every model/activity/session/daily struct. Hero shows a green leaf 'Saved $X' caption, models section gets a green `Saved` column. `swift build` clean. * GNOME indicator adds 'saved $X' to the hero meta line and a `codeburn-model-saved` column to the model row. * Daily cache schema bumped to v8 (`savingsUSD` on day/model/ category/provider). `savingsConfigHash` invalidates the cache when the user changes their baseline mapping so historical saved-spend numbers never lie about a stale baseline. * Defensive `Object.hasOwn` lookup in `getLocalSavingsBaseline` blocks the prototype-pollution test that previously surfaced via the savings path with a hostile `__proto__` model name. * New tests (5 files, 25 tests, 549 lines) cover pricing helpers, end-to-end parser normalization, day aggregator savings, menubar payload savings, CLI set/list/remove, and daily-cache hash invalidation. Existing tests for daily-cache / day-aggregator / models-report updated for the new fields. Full vitest suite: 1028/1028 passing across 73 test files. `tsc --noEmit` clean. `npm run build` clean. (Note: `mac/Tests` has a pre-existing `no such module 'Testing'` environment error on the installed Swift toolchain, confirmed on `main` before this PR; not caused by these changes.)
94 lines
3.7 KiB
TypeScript
94 lines
3.7 KiB
TypeScript
import { describe, it, expect, afterEach } from 'vitest'
|
|
|
|
import {
|
|
calculateCost,
|
|
calculateLocalModelSavings,
|
|
getLocalModelSavingsConfigHash,
|
|
getLocalSavingsBaseline,
|
|
loadPricing,
|
|
setLocalModelSavings,
|
|
} from '../src/models.js'
|
|
|
|
afterEach(() => setLocalModelSavings({}))
|
|
|
|
describe('setLocalModelSavings / getLocalSavingsBaseline', () => {
|
|
it('returns undefined when no mapping is configured', () => {
|
|
setLocalModelSavings({})
|
|
expect(getLocalSavingsBaseline('llama3.1:8b')).toBeUndefined()
|
|
})
|
|
|
|
it('returns the baseline name for a configured source model', () => {
|
|
setLocalModelSavings({ 'llama3.1:8b': 'gpt-4o' })
|
|
expect(getLocalSavingsBaseline('llama3.1:8b')).toBe('gpt-4o')
|
|
})
|
|
|
|
it('uses Object.hasOwn so __proto__ cannot be coerced via the prototype chain', () => {
|
|
// Regression for the prototype-pollution test: a hostile model name
|
|
// like `__proto__` used to resolve to Object.prototype because plain
|
|
// object bracket lookup walks the prototype chain.
|
|
setLocalModelSavings({})
|
|
expect(getLocalSavingsBaseline('__proto__')).toBeUndefined()
|
|
expect(getLocalSavingsBaseline('constructor')).toBeUndefined()
|
|
expect(getLocalSavingsBaseline('toString')).toBeUndefined()
|
|
})
|
|
|
|
it('refuses non-string keys defensively', () => {
|
|
setLocalModelSavings({})
|
|
expect(getLocalSavingsBaseline('' as unknown as string)).toBeUndefined()
|
|
expect(getLocalSavingsBaseline(undefined as unknown as string)).toBeUndefined()
|
|
})
|
|
|
|
it('getLocalModelSavingsConfigHash is stable across sort order and empty for no mappings', () => {
|
|
setLocalModelSavings({})
|
|
expect(getLocalModelSavingsConfigHash()).toBe('')
|
|
|
|
setLocalModelSavings({ a: 'gpt-4o', b: 'claude-opus-4-6' })
|
|
const h1 = getLocalModelSavingsConfigHash()
|
|
setLocalModelSavings({ b: 'claude-opus-4-6', a: 'gpt-4o' })
|
|
const h2 = getLocalModelSavingsConfigHash()
|
|
expect(h1).toBe(h2)
|
|
expect(h1).not.toBe('')
|
|
})
|
|
|
|
it('changes the hash when the baseline mapping changes', () => {
|
|
setLocalModelSavings({ 'llama3.1:8b': 'gpt-4o' })
|
|
const h1 = getLocalModelSavingsConfigHash()
|
|
setLocalModelSavings({ 'llama3.1:8b': 'gpt-5' })
|
|
const h2 = getLocalModelSavingsConfigHash()
|
|
expect(h1).not.toBe(h2)
|
|
})
|
|
})
|
|
|
|
describe('calculateLocalModelSavings', () => {
|
|
it('returns null when no mapping is configured for the model', () => {
|
|
setLocalModelSavings({})
|
|
const out = calculateLocalModelSavings('llama3.1:8b', 1_000_000, 200_000, 0, 0, 0)
|
|
expect(out).toBeNull()
|
|
})
|
|
|
|
it('returns null when the baseline model is unknown to the pricing snapshot', () => {
|
|
setLocalModelSavings({ 'llama3.1:8b': 'unknown-paid-model-xyz' })
|
|
const out = calculateLocalModelSavings('llama3.1:8b', 1_000, 1_000, 0, 0, 0)
|
|
expect(out).toBeNull()
|
|
})
|
|
|
|
it('returns the baseline cost as savings for a configured mapping', async () => {
|
|
await loadPricing()
|
|
setLocalModelSavings({ 'llama3.1:8b': 'gpt-4o' })
|
|
const expected = calculateCost('gpt-4o', 1_000_000, 200_000, 50_000, 800_000, 0)
|
|
const out = calculateLocalModelSavings('llama3.1:8b', 1_000_000, 200_000, 50_000, 800_000, 0)
|
|
expect(out).not.toBeNull()
|
|
expect(out!.savingsUSD).toBeCloseTo(expected)
|
|
expect(out!.baselineModel).toBe('gpt-4o')
|
|
})
|
|
|
|
it('respects speed and web-search inputs in the baseline calculation', async () => {
|
|
await loadPricing()
|
|
setLocalModelSavings({ local: 'gpt-4o' })
|
|
const standard = calculateLocalModelSavings('local', 1_000, 500, 0, 0, 2, 'standard')
|
|
expect(standard).not.toBeNull()
|
|
// Web search is a flat $0.01 per request, so the standard path with 2
|
|
// web search requests should include 2 cents of counterfactual spend.
|
|
expect(standard!.savingsUSD).toBeGreaterThan(0.02)
|
|
})
|
|
})
|