mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-08-29 02:13:06 +00:00
Fix pricing/version staleness and optimize-path caching in status snapshot
Second-round review fixes for the persisted menubar status snapshot (getagentseal/codeburn#999, ozymandiashh review 2026-08-21): - queryKey now folds in getPricingGenerationKey() (src/models.ts): the live LiteLLM cache's freshness, the bundled snapshot's own content digest, and the existing CACHE_SCHEMA_VERSION pricing-behavior version. None of these moved the corpus fingerprint before, so a repricing fetch or pricing-logic fix could keep serving old rendered costs against an unchanged session corpus. - The optimize path (default; --no-optimize unaffected) no longer reads or writes the persisted snapshot at all. Optimize findings depend on mutable project/config/prompt/hook state with no single enumerable fingerprint, and computing them already requires the same full parse the snapshot exists to skip, so excluding them from the snapshot loses no additional performance versus fingerprinting that input surface. Adds two regression tests exercising both fixes end-to-end.
This commit is contained in:
parent
e329f0748b
commit
113ebb1ca6
3 changed files with 169 additions and 7 deletions
30
src/main.ts
30
src/main.ts
|
|
@ -2,7 +2,7 @@ import { isAbsolute } from 'path'
|
|||
import { Command, Option } from 'commander'
|
||||
import { installMenubarApp } from './menubar-installer.js'
|
||||
import { exportCsv, exportJson, type PeriodExport } from './export.js'
|
||||
import { findUnpricedModels, loadPricing, sanitizeModelForDisplay, setModelAliases, setPriceOverrides, setLocalModelSavings, setFlatRateModels, setFlatRateRemoved, setProxyPaths, normalizeProxyPath, unpricedModelHint, isBuiltInFlatRateModel, isSameFlatRateModel, getProxyPathsConfigHash, getModelAliasesConfigHash, getPriceOverridesConfigHash, getLocalModelSavingsConfigHash } from './models.js'
|
||||
import { findUnpricedModels, loadPricing, sanitizeModelForDisplay, setModelAliases, setPriceOverrides, setLocalModelSavings, setFlatRateModels, setFlatRateRemoved, setProxyPaths, normalizeProxyPath, unpricedModelHint, isBuiltInFlatRateModel, isSameFlatRateModel, getProxyPathsConfigHash, getModelAliasesConfigHash, getPriceOverridesConfigHash, getLocalModelSavingsConfigHash, getPricingGenerationKey } from './models.js'
|
||||
import { parseAllSessions, filterProjectsByName, filterProjectsByDateRange, clearSessionCache, setInteractiveScanUI, computeCorpusFingerprint, isSessionHydrationComplete } from './parser.js'
|
||||
import { allProviderNames, getAllProviders } from './providers/index.js'
|
||||
import { getProvider } from './providers/index.js'
|
||||
|
|
@ -1149,9 +1149,31 @@ program
|
|||
// serving the old currency's numbers until a session file happens to
|
||||
// change too.
|
||||
currency: getCurrency(),
|
||||
// Upstream/bundled pricing DATA and CODE version, as opposed to the
|
||||
// four hashes above (user-editable pricing CONFIG): the live LiteLLM
|
||||
// cache's freshness, the bundled snapshot's own content, and the
|
||||
// parser/pricing logic's semantic version. None of these move the
|
||||
// corpus fingerprint or any config hash, so without this a repricing
|
||||
// fetch or a pricing-logic fix can keep serving old rendered costs
|
||||
// indefinitely against an unchanged session corpus.
|
||||
pricingGenerationKey: getPricingGenerationKey(),
|
||||
})
|
||||
const corpus = await computeCorpusFingerprint(pf)
|
||||
const snapshot = await loadStatusSnapshot(corpus.hash, corpus.newestMtimeMs, queryKey)
|
||||
// Optimize findings (the default; see --no-optimize) depend on mutable
|
||||
// project/config/prompt/hook state — ~/.claude and project-level
|
||||
// settings.json, CLAUDE.md, defined skills/agents/commands, MCP config
|
||||
// — that computeCorpusFingerprint and queryKey never observe and that
|
||||
// has no single enumerable fingerprint. Persisting THIS class of output
|
||||
// would mean editing a hook or removing an unused skill leaves the
|
||||
// menubar showing stale findings with no session change to ever
|
||||
// invalidate them. Simplest correct fix: the optimize path never reads
|
||||
// or writes the disk snapshot at all, it always recomputes fresh.
|
||||
// (Computing scanAndDetect's findings requires the same parsed project
|
||||
// data the snapshot exists to avoid recomputing, so skipping the
|
||||
// snapshot loses no additional work versus fingerprinting these inputs
|
||||
// — that path would still force a fresh parse to re-scan them.)
|
||||
const useSnapshot = !queryScope.optimize
|
||||
const corpus = useSnapshot ? await computeCorpusFingerprint(pf) : null
|
||||
const snapshot = corpus ? await loadStatusSnapshot(corpus.hash, corpus.newestMtimeMs, queryKey) : null
|
||||
const payload = (snapshot ?? await buildMenubarPayloadForRange(periodInfo, {
|
||||
...queryScope,
|
||||
daysSelection,
|
||||
|
|
@ -1162,7 +1184,7 @@ program
|
|||
// fingerprint would make that degraded answer look authoritative to
|
||||
// every future poll that matches this fingerprint — never checkpoint a
|
||||
// partial hydration as if it were a real, complete parse.
|
||||
if (!snapshot && isSessionHydrationComplete()) await saveStatusSnapshot(corpus.hash, corpus.newestMtimeMs, queryKey, payload)
|
||||
if (useSnapshot && corpus && !snapshot && isSessionHydrationComplete()) await saveStatusSnapshot(corpus.hash, corpus.newestMtimeMs, queryKey, payload)
|
||||
if (opts.scope === 'combined') {
|
||||
// Combined multi-device usage is best-effort enrichment on the menubar's
|
||||
// hot path. Never let pulling peers (or a corrupt remotes store) take
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import { readFile, writeFile, mkdir } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
import { createHash } from 'crypto'
|
||||
|
||||
import { getCodeburnCacheDir } from './cache-dir.js'
|
||||
import snapshotData from './data/litellm-snapshot.json' with { type: 'json' }
|
||||
|
|
@ -67,7 +68,10 @@ const CACHE_TTL_MS = 24 * 60 * 60 * 1000
|
|||
// added in #1075/#1078). A cache written under an older/missing version is treated as a
|
||||
// miss instead of read verbatim, so a stale on-disk file can't reintroduce a killed bug
|
||||
// for up to CACHE_TTL_MS after an upgrade.
|
||||
const CACHE_SCHEMA_VERSION = 2
|
||||
// Also folded into getPricingGenerationKey() below: a resident/snapshot-caching
|
||||
// consumer needs the same "pricing behavior changed" signal this already gives
|
||||
// the on-disk LiteLLM cache, not just the on-disk cache itself.
|
||||
export const CACHE_SCHEMA_VERSION = 2
|
||||
const WEB_SEARCH_COST = 0.01
|
||||
const ONE_HOUR_CACHE_WRITE_MULTIPLIER_FROM_FIVE_MINUTE_RATE = 1.6
|
||||
|
||||
|
|
@ -209,6 +213,16 @@ export function parseLiteLLMEntry(entry: LiteLLMEntry): ModelCosts | null {
|
|||
)
|
||||
}
|
||||
|
||||
// Timestamp of whichever live LiteLLM data (freshly fetched or read back from
|
||||
// the on-disk cache) is currently loaded into pricingCache; null when nothing
|
||||
// live is loaded and pricing is purely the bundled snapshot (offline/first
|
||||
// run, CODEBURN_PRICING_SNAPSHOT_ONLY, or a failed fetch with no cache hit).
|
||||
// Read by getPricingGenerationKey() so a consumer that persists rendered
|
||||
// costs across process invocations (the menubar's status snapshot) can tell
|
||||
// "the live pricing data actually changed" apart from "nothing changed" —
|
||||
// this module has no other way to signal that across a fresh CLI process.
|
||||
let livePricingTimestamp: number | null = null
|
||||
|
||||
async function fetchAndCachePricing(): Promise<Map<string, ModelCosts>> {
|
||||
// Bounded: runs on every CLI invocation (the menubar shells out and blocks on
|
||||
// it). Without a timeout a half-open network after wake-from-sleep makes
|
||||
|
|
@ -230,12 +244,14 @@ async function fetchAndCachePricing(): Promise<Map<string, ModelCosts>> {
|
|||
if (stripped !== name && !pricing.has(stripped)) pricing.set(stripped, costs)
|
||||
}
|
||||
|
||||
const timestamp = Date.now()
|
||||
await mkdir(getCodeburnCacheDir(), { recursive: true })
|
||||
await writeFile(getCachePath(), JSON.stringify({
|
||||
version: CACHE_SCHEMA_VERSION,
|
||||
timestamp: Date.now(),
|
||||
timestamp,
|
||||
data: Object.fromEntries(pricing),
|
||||
}))
|
||||
livePricingTimestamp = timestamp
|
||||
|
||||
return pricing
|
||||
}
|
||||
|
|
@ -246,6 +262,7 @@ async function loadCachedPricing(): Promise<Map<string, ModelCosts> | null> {
|
|||
const cached = JSON.parse(raw) as { version?: number; timestamp: number; data: Record<string, ModelCosts> }
|
||||
if (cached.version !== CACHE_SCHEMA_VERSION) return null
|
||||
if (Date.now() - cached.timestamp > CACHE_TTL_MS) return null
|
||||
livePricingTimestamp = cached.timestamp
|
||||
return new Map(Object.entries(cached.data))
|
||||
} catch {
|
||||
return null
|
||||
|
|
@ -277,6 +294,7 @@ export async function loadPricing(): Promise<void> {
|
|||
// tests/setup/env-isolation.ts: skip the live LiteLLM fetch and price purely
|
||||
// off the bundled snapshot, so an upstream reprice can't turn tests red.
|
||||
if (process.env['CODEBURN_PRICING_SNAPSHOT_ONLY']) {
|
||||
livePricingTimestamp = null
|
||||
setPricingCache(mergeSnapshotFallbacks(new Map()))
|
||||
return
|
||||
}
|
||||
|
|
@ -285,9 +303,41 @@ export async function loadPricing(): Promise<void> {
|
|||
setPricingCache(mergeSnapshotFallbacks(await fetchAndCachePricing()))
|
||||
} catch {
|
||||
// snapshot already loaded at init; nothing more to do
|
||||
livePricingTimestamp = null
|
||||
}
|
||||
}
|
||||
|
||||
// Content digest of the two bundled pricing files, computed once and memoized
|
||||
// (they're static imports; nothing in-process can change them). Changes only
|
||||
// when `scripts/bundle-litellm.mjs` regenerates litellm-snapshot.json /
|
||||
// pricing-fallback.json and that regeneration ships in a new codeburn build —
|
||||
// exactly the "bundled data" staleness class getPricingGenerationKey exists
|
||||
// to catch, distinct from the live cache's own timestamp.
|
||||
let bundledPricingDigest: string | null = null
|
||||
function getBundledPricingDigest(): string {
|
||||
if (bundledPricingDigest === null) {
|
||||
bundledPricingDigest = createHash('sha256')
|
||||
.update(JSON.stringify(snapshotData))
|
||||
.update(JSON.stringify(fallbackData))
|
||||
.digest('hex')
|
||||
}
|
||||
return bundledPricingDigest
|
||||
}
|
||||
|
||||
/// Stable signature of everything that can silently change a session's
|
||||
/// RENDERED cost with no session file ever changing: the live LiteLLM cache's
|
||||
/// freshness (a repricing fetch, or its absence), the bundled snapshot's own
|
||||
/// content (a repriced model shipped in a new build), and this module's
|
||||
/// pricing-behavior version (CACHE_SCHEMA_VERSION). A caller that persists a
|
||||
/// fully-rendered payload across process invocations (the menubar's status
|
||||
/// snapshot) must fold this into its own cache key the same way it already
|
||||
/// folds the four *ConfigHash getters above — those cover user-editable
|
||||
/// pricing CONFIG, this covers upstream/bundled pricing DATA and code version,
|
||||
/// a different staleness gap with no other invalidation path of its own.
|
||||
export function getPricingGenerationKey(): string {
|
||||
return `${CACHE_SCHEMA_VERSION}:${livePricingTimestamp ?? 'bundled'}:${getBundledPricingDigest()}`
|
||||
}
|
||||
|
||||
// Known model name variants that providers emit but LiteLLM/fallback don't index under.
|
||||
// OMP emits 'anthropic--claude-4.6-opus' (double-dash, dot version, tier-last).
|
||||
// getCanonicalName strips a KNOWN vendor/router prefix first unless the
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { statSync } from 'node:fs'
|
||||
import { existsSync, statSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { delimiter as pathDelimiter, join } from 'node:path'
|
||||
import { spawnSync } from 'node:child_process'
|
||||
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { CACHE_SCHEMA_VERSION } from '../src/models.js'
|
||||
|
||||
// Every case here spawns the real CLI and does genuine multi-provider parse
|
||||
// work; the 5s default is fine on a dev laptop and not on a shared 2-core
|
||||
|
|
@ -713,6 +714,95 @@ describe('codeburn status --format menubar-json', () => {
|
|||
}
|
||||
})
|
||||
|
||||
it('reprices from an updated live LiteLLM cache instead of serving a stale snapshot', async () => {
|
||||
const home = await mkdtemp(join(tmpdir(), 'codeburn-menubar-pricing-gen-'))
|
||||
|
||||
try {
|
||||
const projectDir = join(home, '.claude', 'projects', 'myapp')
|
||||
await mkdir(projectDir, { recursive: true })
|
||||
const now = new Date()
|
||||
const todayUtcMidnight = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())
|
||||
const base = new Date(Math.max(todayUtcMidnight, now.getTime() - 2 * 3600_000))
|
||||
const ts = (offset: number) => new Date(base.getTime() + offset).toISOString().replace(/\.\d+Z$/, 'Z')
|
||||
await writeFile(
|
||||
join(projectDir, 'session.jsonl'),
|
||||
[userLine('s1', ts(0)), assistantLine('s1', ts(60_000), 'msg-1')].join('\n'),
|
||||
)
|
||||
|
||||
const cacheDir = join(home, '.cache', 'codeburn')
|
||||
await mkdir(cacheDir, { recursive: true })
|
||||
const liveCachePath = join(cacheDir, 'litellm-pricing.json')
|
||||
const writeLivePricing = (timestamp: number, inputCost: number, outputCost: number) =>
|
||||
writeFile(liveCachePath, JSON.stringify({
|
||||
version: CACHE_SCHEMA_VERSION,
|
||||
timestamp,
|
||||
data: { 'claude-sonnet-4-5': { inputCostPerToken: inputCost, outputCostPerToken: outputCost, cacheWriteCostPerToken: inputCost * 1.25, cacheReadCostPerToken: inputCost * 0.1, webSearchCostPerRequest: 0.01, fastMultiplier: 1 } },
|
||||
}))
|
||||
|
||||
const args = ['status', '--format', 'menubar-json', '--period', 'today', '--provider', 'claude', '--no-optimize']
|
||||
|
||||
await writeLivePricing(Date.now(), 1e-6, 5e-6)
|
||||
const first = runCli(args, home)
|
||||
expect(first.status, `stderr: ${first.stderr}`).toBe(0)
|
||||
const firstCost = (JSON.parse(first.stdout) as { current: { cost: number } }).current.cost
|
||||
|
||||
// Same session corpus, no config/currency change — only the live
|
||||
// LiteLLM cache's price and timestamp move. Before the fix, the
|
||||
// persisted status snapshot has no way to observe this and would keep
|
||||
// serving `firstCost` indefinitely.
|
||||
await writeLivePricing(Date.now() + 1000, 2e-6, 10e-6)
|
||||
const second = runCli(args, home)
|
||||
expect(second.status, `stderr: ${second.stderr}`).toBe(0)
|
||||
const secondCost = (JSON.parse(second.stdout) as { current: { cost: number } }).current.cost
|
||||
|
||||
expect(secondCost).toBeCloseTo(firstCost * 2, 5)
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('never persists or serves the default optimize-enabled payload from the status snapshot', async () => {
|
||||
const home = await mkdtemp(join(tmpdir(), 'codeburn-menubar-optimize-cache-'))
|
||||
|
||||
try {
|
||||
const projectDir = join(home, '.claude', 'projects', 'myapp')
|
||||
await mkdir(projectDir, { recursive: true })
|
||||
const now = new Date()
|
||||
const todayUtcMidnight = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())
|
||||
const base = new Date(Math.max(todayUtcMidnight, now.getTime() - 2 * 3600_000))
|
||||
const ts = (offset: number) => new Date(base.getTime() + offset).toISOString().replace(/\.\d+Z$/, 'Z')
|
||||
await writeFile(
|
||||
join(projectDir, 'session.jsonl'),
|
||||
[userLine('s1', ts(0)), assistantLine('s1', ts(60_000), 'msg-1')].join('\n'),
|
||||
)
|
||||
|
||||
const args = ['status', '--format', 'menubar-json', '--period', 'today', '--provider', 'claude']
|
||||
|
||||
const before = runCli(args, home)
|
||||
expect(before.status, `stderr: ${before.stderr}`).toBe(0)
|
||||
const findingsBefore = (JSON.parse(before.stdout) as { optimize: { findingCount: number } }).optimize.findingCount
|
||||
|
||||
const snapshotPath = join(home, '.cache', 'codeburn', 'status-snapshot.json')
|
||||
expect(existsSync(snapshotPath)).toBe(false)
|
||||
|
||||
// Mutable, non-fingerprinted optimize input: an unused custom agent
|
||||
// definition. The session corpus is untouched, so a corpus-fingerprint-
|
||||
// keyed snapshot would never notice this changed.
|
||||
const agentsDir = join(home, '.claude', 'agents')
|
||||
await mkdir(agentsDir, { recursive: true })
|
||||
await writeFile(join(agentsDir, 'ghost-agent.md'), '# never invoked this period')
|
||||
|
||||
const after = runCli(args, home)
|
||||
expect(after.status, `stderr: ${after.stderr}`).toBe(0)
|
||||
const findingsAfter = (JSON.parse(after.stdout) as { optimize: { findingCount: number } }).optimize.findingCount
|
||||
|
||||
expect(findingsAfter).toBe(findingsBefore + 1)
|
||||
expect(existsSync(snapshotPath)).toBe(false)
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('still emits a valid combined menubar payload when the remotes store is corrupt', async () => {
|
||||
const home = await mkdtemp(join(tmpdir(), 'codeburn-menubar-corrupt-remotes-'))
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue