mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-11-22 12:59:06 +00:00
fix(hermes): count helper-model usage recorded in session_model_usage (#1849)
* fix(hermes): count helper-model usage recorded in session_model_usage Hermes records auxiliary calls (compression, title generation, vision and other helper tasks) only in session_model_usage, under the helper model and a task name, and never adds them to the sessions totals the provider reads. Those tokens and dollars were missing. Each helper row now becomes a call on its own session, priced through the same cost path with the helper model id, tracked by the session ledger so later growth lands as a delta, and marked as supplementary accounting so it adds no requests or turns. Rows with an empty task are the main loop and voice_chat is already in the sessions totals, so both are skipped. Measured on this machine's Hermes state.db: 12 sessions, 12 calls, $2.0834 before and after. Its 12 session_model_usage rows are all main-loop rows that equal the sessions totals, so nothing changes there. On a copy of that DB with a compression row and a title row added (plus a voice_chat row that must stay out): before 12 calls, $2.0834; after 14 calls, $2.1660, helper share $0.0826. Growing the compression row by 10,000 input tokens added one $0.0100 delta. * fix(hermes): keep a recorded zero helper cost session_model_usage stores actual_cost_usd as NOT NULL DEFAULT 0, while the sessions column is nullable, so a 0 on a helper row usually means nothing was recorded. When the row's cost_status says the amount is actual, that 0 is a recorded free line and now stays $0 (metered) like a sessions row, instead of being re-priced from tokens.
This commit is contained in:
parent
75d3f0a9b6
commit
fb95c4bca8
3 changed files with 160 additions and 3 deletions
|
|
@ -74,6 +74,14 @@ type TableInfoRow = {
|
|||
|
||||
type TableColumn = keyof HermesSessionRow | keyof HermesMessageRow
|
||||
|
||||
type HermesHelperUsageRow = HermesTokenTotals & Pick<HermesSessionRow, 'billing_provider' | 'estimated_cost_usd' | 'actual_cost_usd' | 'cost_status' | 'cost_source'> & {
|
||||
task: string
|
||||
model: string
|
||||
billing_base_url: string
|
||||
billing_mode: string
|
||||
first_seen: number | null
|
||||
}
|
||||
|
||||
const toolNameMap: Record<string, string> = {
|
||||
terminal: 'Bash',
|
||||
execute_code: 'CodeExecution',
|
||||
|
|
@ -282,6 +290,23 @@ function getMessageColumns(db: SqliteDatabase): Set<string> {
|
|||
return new Set(db.query<TableInfoRow>('PRAGMA table_info(messages)').map(row => row.name))
|
||||
}
|
||||
|
||||
function readHelperUsage(db: SqliteDatabase, sessionId: string): HermesHelperUsageRow[] {
|
||||
const columns = new Set(db.query<TableInfoRow>('PRAGMA table_info(session_model_usage)').map(row => row.name))
|
||||
if (!columns.has('task')) return []
|
||||
// Auxiliary calls (compression, titles, ...) land only here, never in the sessions totals;
|
||||
// voice_chat is the one task Hermes also adds to the sessions totals.
|
||||
return db.query<HermesHelperUsageRow>(
|
||||
`SELECT task, model, billing_provider, billing_base_url, billing_mode,
|
||||
input_tokens AS inputTokens, output_tokens AS outputTokens,
|
||||
cache_read_tokens AS cacheReadTokens, cache_write_tokens AS cacheWriteTokens,
|
||||
reasoning_tokens AS reasoningTokens, estimated_cost_usd,
|
||||
CASE WHEN cost_status = 'actual' THEN actual_cost_usd ELSE NULLIF(actual_cost_usd, 0) END AS actual_cost_usd, cost_status, cost_source, first_seen
|
||||
FROM session_model_usage
|
||||
WHERE session_id = ? AND task NOT IN ('', 'voice_chat')`,
|
||||
[sessionId],
|
||||
)
|
||||
}
|
||||
|
||||
function usageExpression(columns: Set<string>): string {
|
||||
const usageColumns: Array<keyof HermesSessionRow> = [
|
||||
'input_tokens',
|
||||
|
|
@ -418,7 +443,7 @@ function tokenSum(tokens: HermesTokenTotals): number {
|
|||
}
|
||||
|
||||
function resolveHermesCost(
|
||||
row: HermesSessionRow | undefined,
|
||||
row: Pick<HermesSessionRow, 'estimated_cost_usd' | 'actual_cost_usd' | 'cost_status' | 'cost_source'> | undefined,
|
||||
model: string,
|
||||
tokens: HermesTokenTotals,
|
||||
): { costUSD: number; costIsEstimated: boolean; costBasis: HermesCostBasis } {
|
||||
|
|
@ -580,7 +605,7 @@ async function discoverFromDb(dbPath: string, profile: string): Promise<SessionS
|
|||
const discoveredIds = new Set(rows.map(row => row.id))
|
||||
const ledger = loadHermesSessionLedger()
|
||||
for (const sessionId of listHermesCursorSessionIds(ledger, profile)) {
|
||||
if (discoveredIds.has(sessionId)) continue
|
||||
if (discoveredIds.has(sessionId) || sessionId.includes('#helper:')) continue
|
||||
sources.push({
|
||||
path: encodeSourcePath(dbPath, sessionId),
|
||||
project: displayProjectForProfile(profile),
|
||||
|
|
@ -767,6 +792,49 @@ function createParser(source: SessionSource, seenKeys: Set<string>, hermesHome:
|
|||
route: routeFromProviderField(row.billing_provider)?.id,
|
||||
})),
|
||||
}
|
||||
for (const helper of readHelperUsage(db, row.id)) {
|
||||
const helperId = `${row.id}#helper:${[helper.task, helper.model, helper.billing_provider, helper.billing_base_url, helper.billing_mode].join('|')}`
|
||||
const helperKey = hermesBaselineKey(profile, helperId)
|
||||
if (seenKeys.has(helperKey)) continue
|
||||
seenKeys.add(helperKey)
|
||||
const helperCost = resolveHermesCost(helper, helper.model, helper)
|
||||
const helperCursor = await recordHermesSnapshot({
|
||||
profile,
|
||||
sessionId: helperId,
|
||||
startedAt: parseTimestamp(helper.first_seen) || timestamp || hermesLedgerNow().toISOString(),
|
||||
observedAt: hermesLedgerNow().toISOString(),
|
||||
tokens: {
|
||||
inputTokens: helper.inputTokens,
|
||||
outputTokens: helper.outputTokens,
|
||||
cacheReadTokens: helper.cacheReadTokens,
|
||||
cacheWriteTokens: helper.cacheWriteTokens,
|
||||
reasoningTokens: helper.reasoningTokens,
|
||||
},
|
||||
costUSD: helperCost.costUSD,
|
||||
costBasis: helperCost.costBasis,
|
||||
})
|
||||
for (const observation of helperCursor.observations) {
|
||||
if (observation.index > 0) seenKeys.add(hermesObservationKey(profile, helperId, observation.index))
|
||||
result.calls.push({
|
||||
...observationToCall(observation, {
|
||||
profile,
|
||||
sessionId: helperId,
|
||||
model: helper.model,
|
||||
tools: [],
|
||||
bashCommands: [],
|
||||
toolSequence: [],
|
||||
userMessage: '',
|
||||
project: workspace.project,
|
||||
projectPath: workspace.projectPath,
|
||||
workingDirectory: workspace.workingDirectory,
|
||||
costIsEstimated: helperCost.costIsEstimated,
|
||||
route: routeFromProviderField(helper.billing_provider)?.id,
|
||||
}),
|
||||
sessionId: row.id,
|
||||
supplementaryAccounting: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
// A transient lock on the live state.db, or a ledger publication
|
||||
|
|
|
|||
|
|
@ -530,7 +530,8 @@ export const PROVIDER_PARSE_VERSIONS: Record<string, string> = {
|
|||
// billing-mode-v1: the resolved cost basis now rides on each call as
|
||||
// `billing` (`included` -> subscription, `actual` -> metered). Cached calls
|
||||
// hold none, so they must re-parse.
|
||||
hermes: 'reasoning-output-accounting-v1-est-cost-routed-ids-workspace-pr-v5-cost-provenance-v3-billing-route-v1-billing-mode-v1',
|
||||
// helper-usage-v1: auxiliary-model rows in session_model_usage now become calls.
|
||||
hermes: 'reasoning-output-accounting-v1-est-cost-routed-ids-workspace-pr-v5-cost-provenance-v3-billing-route-v1-billing-mode-v1-helper-usage-v1',
|
||||
// reported-cost-v1: OpenClaw's per-message `usage.cost.total` is now
|
||||
// preserved through the cache via `costFromBilling`. This is OpenClaw's
|
||||
// first parse version; adding it moves the provider's env fingerprint,
|
||||
|
|
|
|||
|
|
@ -1283,4 +1283,92 @@ skipUnlessSqlite('hermes provider', () => {
|
|||
const calls = await collectCalls(tmpDir, `${dbPath}#hermes-session=tool-result-session`)
|
||||
expect(calls[0]!.tools).toContain('Read')
|
||||
})
|
||||
|
||||
it('counts helper-model usage from session_model_usage on its own session, beside the main call', async () => {
|
||||
const dbPath = createHermesDb(tmpDir)
|
||||
withTestDb(dbPath, (db) => {
|
||||
db.exec(`
|
||||
CREATE TABLE session_model_usage (
|
||||
session_id TEXT NOT NULL, model TEXT NOT NULL,
|
||||
billing_provider TEXT NOT NULL DEFAULT '', billing_base_url TEXT NOT NULL DEFAULT '',
|
||||
billing_mode TEXT NOT NULL DEFAULT '', task TEXT NOT NULL DEFAULT '',
|
||||
api_call_count INTEGER NOT NULL DEFAULT 0, input_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
output_tokens INTEGER NOT NULL DEFAULT 0, cache_read_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
cache_write_tokens INTEGER NOT NULL DEFAULT 0, reasoning_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
estimated_cost_usd REAL NOT NULL DEFAULT 0, actual_cost_usd REAL NOT NULL DEFAULT 0,
|
||||
cost_status TEXT, cost_source TEXT, first_seen REAL, last_seen REAL,
|
||||
PRIMARY KEY (session_id, model, billing_provider, billing_base_url, billing_mode, task)
|
||||
)
|
||||
`)
|
||||
const usage = db.prepare(`INSERT INTO session_model_usage
|
||||
(session_id, model, billing_provider, task, input_tokens, output_tokens, first_seen) VALUES (?, ?, ?, ?, ?, ?, ?)`)
|
||||
for (const id of ['with-helpers', 'without-helpers']) {
|
||||
insertSession(db, {
|
||||
id, model: 'claude-sonnet-4-20250514', billingProvider: 'anthropic',
|
||||
inputTokens: 1000, outputTokens: 200, cacheReadTokens: 0, cacheWriteTokens: 0, reasoningTokens: 0, startedAt: 1779549200,
|
||||
})
|
||||
usage.run(id, 'claude-sonnet-4-20250514', 'anthropic', '', 1000, 200, 1779549200)
|
||||
}
|
||||
usage.run('with-helpers', 'claude-haiku-4-5', 'anthropic', 'compression', 5000, 300, 1779549300)
|
||||
usage.run('with-helpers', 'gpt-5.5', 'openrouter', 'title_generation', 40, 8, 1779549400)
|
||||
// Already inside the sessions totals.
|
||||
usage.run('with-helpers', 'claude-sonnet-4-20250514', 'anthropic', 'voice_chat', 70, 7, 1779549500)
|
||||
})
|
||||
|
||||
const main = calculateCost('claude-sonnet-4-20250514', 1000, 200, 0, 0, 0)
|
||||
const compression = calculateCost('claude-haiku-4-5', 5000, 300, 0, 0, 0)
|
||||
const title = calculateCost('gpt-5.5', 40, 8, 0, 0, 0)
|
||||
expect(compression).toBeGreaterThan(0)
|
||||
expect(title).toBeGreaterThan(0)
|
||||
|
||||
const plain = await collectCalls(tmpDir, `${dbPath}#hermes-session=without-helpers`)
|
||||
expect(plain).toHaveLength(1)
|
||||
expect(plain[0]).toMatchObject({ model: 'claude-sonnet-4-20250514', inputTokens: 1000, outputTokens: 200, costUSD: main })
|
||||
expect(plain[0]!.supplementaryAccounting).toBeUndefined()
|
||||
|
||||
const calls = await collectCalls(tmpDir, `${dbPath}#hermes-session=with-helpers`)
|
||||
expect(calls.map(call => call.model)).toEqual(['claude-sonnet-4-20250514', 'claude-haiku-4-5', 'gpt-5.5'])
|
||||
expect(calls[0]).toEqual({ ...plain[0], sessionId: 'with-helpers', deduplicationKey: 'hermes:default:with-helpers', turnId: 'with-helpers:session' })
|
||||
expect(calls[1]).toMatchObject({
|
||||
sessionId: 'with-helpers', inputTokens: 5000, outputTokens: 300, costUSD: compression,
|
||||
timestamp: new Date(1779549300 * 1000).toISOString(), supplementaryAccounting: true, userMessage: '', tools: [],
|
||||
})
|
||||
expect(calls[2]).toMatchObject({ sessionId: 'with-helpers', costUSD: title, route: 'openrouter', supplementaryAccounting: true })
|
||||
expect(new Set(calls.map(call => call.deduplicationKey)).size).toBe(3)
|
||||
expect(calls.reduce((sum, call) => sum + call.inputTokens, 0)).toBe(1000 + 5000 + 40)
|
||||
expect(calls.reduce((sum, call) => sum + call.costUSD, 0)).toBeCloseTo(main + compression + title, 12)
|
||||
})
|
||||
|
||||
it('keeps a recorded actual $0 on a helper row instead of pricing its tokens', async () => {
|
||||
const dbPath = createHermesDb(tmpDir)
|
||||
withTestDb(dbPath, (db) => {
|
||||
db.exec(`
|
||||
CREATE TABLE session_model_usage (
|
||||
session_id TEXT NOT NULL, model TEXT NOT NULL,
|
||||
billing_provider TEXT NOT NULL DEFAULT '', billing_base_url TEXT NOT NULL DEFAULT '',
|
||||
billing_mode TEXT NOT NULL DEFAULT '', task TEXT NOT NULL DEFAULT '',
|
||||
api_call_count INTEGER NOT NULL DEFAULT 0, input_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
output_tokens INTEGER NOT NULL DEFAULT 0, cache_read_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
cache_write_tokens INTEGER NOT NULL DEFAULT 0, reasoning_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
estimated_cost_usd REAL NOT NULL DEFAULT 0, actual_cost_usd REAL NOT NULL DEFAULT 0,
|
||||
cost_status TEXT, cost_source TEXT, first_seen REAL, last_seen REAL,
|
||||
PRIMARY KEY (session_id, model, billing_provider, billing_base_url, billing_mode, task)
|
||||
)
|
||||
`)
|
||||
insertSession(db, {
|
||||
id: 'free-helper', model: 'claude-sonnet-4-20250514', billingProvider: 'anthropic',
|
||||
inputTokens: 1000, outputTokens: 200, cacheReadTokens: 0, cacheWriteTokens: 0, reasoningTokens: 0, startedAt: 1779549200,
|
||||
})
|
||||
const usage = db.prepare(`INSERT INTO session_model_usage
|
||||
(session_id, model, billing_provider, task, input_tokens, output_tokens, cost_status, first_seen) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
usage.run('free-helper', 'claude-haiku-4-5', 'openrouter', 'compression', 5000, 300, 'actual', 1779549300)
|
||||
usage.run('free-helper', 'claude-haiku-4-5', 'openrouter', 'vision', 5000, 300, null, 1779549300)
|
||||
})
|
||||
|
||||
const calls = await collectCalls(tmpDir, `${dbPath}#hermes-session=free-helper`)
|
||||
expect(calls).toHaveLength(3)
|
||||
expect(calls[1]).toMatchObject({ costUSD: 0, costIsEstimated: false, billing: 'metered' })
|
||||
expect(calls[2]!.costUSD).toBe(calculateCost('claude-haiku-4-5', 5000, 300, 0, 0, 0))
|
||||
expect(calls[2]!.costUSD).toBeGreaterThan(0)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue