mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-07-10 01:29:41 +00:00
fix(cursor): stable conversation crediting, restored stream coverage, and cache invalidation
Review fixes for the real-token accounting: Conversation input now lands on one composer-anchored record (cursor:composer-input:<id>) timestamped at composerData.createdAt, so the credited day no longer depends on the parse window or cache floors, daily-cache gap fills dedupe instead of multiplying, and each conversation picks exactly one input source (real bubble tokenCounts, the context meter, the agent stream, or visible text) so sources can never stack or double count. A zero totalUsedTokens no longer shadows contextTokensUsed. The agent stream regained what the parseAgentKv removal dropped: tool and system rows count as context, stream-only replies count as output, and sessions with no bubble join are emitted again (DB mtime timestamp, as before). Block-array content is measured by its text, not its JSON envelope. Rows written before their requestId appears buffer forward instead of inheriting the previous conversation, and a system row closes the boundary. Tool names canonicalize to Bash and commands go through extractBashCommands so cross-provider breakdowns merge; the classifier no longer special-cases Shell (which also reclassified Copilot turns). User bubbles consume their own queue entry so assistant replies pair with the right question, and every cursor call is flagged costIsEstimated. The requestId and model joins ride the existing budgeted bubble scan instead of two new unbounded full-table decodes, and the composerData read seeks the key range. SQLITE_BUSY now propagates to the parser's retry path instead of caching a silently degraded parse. Upgrades actually take effect: the session cache gets a cursor parse version, DAILY_CACHE_VERSION bumps to 10 so finalized days re-hydrate under the new accounting, the cursor results cache bumps to v6, and the builtin composer rates participate in the price config hash (rates now cite cursor.com/docs/models). Verified against a real Cursor store: all metered conversations match the on-disk meter exactly, narrow and wide parse windows anchor identically, repeat runs are byte-identical, and agentKv-only sessions reappear.
This commit is contained in:
AgentSeal
parent
c71bf9bb4b
commit
21145ea5a2
8 changed files with 517 additions and 220 deletions
|
|
@ -17,7 +17,7 @@ const URL_PATTERN = /https?:\/\/\S+/i
|
|||
|
||||
export const EDIT_TOOLS = new Set(['Edit', 'Write', 'FileEditTool', 'FileWriteTool', 'NotebookEdit', 'cursor:edit'])
|
||||
const READ_TOOLS = new Set(['Read', 'Grep', 'Glob', 'FileReadTool', 'GrepTool', 'GlobTool'])
|
||||
export const BASH_TOOLS = new Set(['Bash', 'BashTool', 'PowerShellTool', 'Shell'])
|
||||
export const BASH_TOOLS = new Set(['Bash', 'BashTool', 'PowerShellTool'])
|
||||
const TASK_TOOLS = new Set(['TaskCreate', 'TaskUpdate', 'TaskGet', 'TaskList', 'TaskOutput', 'TaskStop', 'TodoWrite'])
|
||||
const SEARCH_TOOLS = new Set(['WebSearch', 'WebFetch', 'ToolSearch'])
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,11 @@ import type { ParsedProviderCall } from './providers/types.js'
|
|||
// real context tokens from composerData.promptTokenBreakdown now drive
|
||||
// input, and agentKv is used only for the tools/bash breakdown. Cached v4
|
||||
// results contain stale agentKv calls and lack the real token figures.
|
||||
const CURSOR_CACHE_VERSION = 5
|
||||
// Version 6: conversation input moved to composer-anchored records
|
||||
// (cursor:composer-input:<id>) with per-conversation source selection, the
|
||||
// agent stream regained tool/system context and stream-only sessions, and
|
||||
// tool names are canonicalized. v5 results mix crediting regimes.
|
||||
const CURSOR_CACHE_VERSION = 6
|
||||
|
||||
type ResultCache = {
|
||||
version?: number
|
||||
|
|
|
|||
|
|
@ -5,18 +5,18 @@ import { homedir } from 'os'
|
|||
import { join } from 'path'
|
||||
import type { DateRange, ProjectSummary } from './types.js'
|
||||
|
||||
// Bumped to 9: providers added since the v8 rollup (Grok, Hermes, ZCode) parse
|
||||
// usage that older binaries skipped, so days cached at v8 omit them and report
|
||||
// $0 for those providers across history. Raising MIN_SUPPORTED_VERSION to 9 too
|
||||
// forces a one-time full re-hydration so newly supported providers backfill
|
||||
// without a manual cache clear.
|
||||
// Bumped to 10: cursor accounting changed (real composer context tokens on
|
||||
// conversation-anchored records, Cursor-published composer pricing), so days
|
||||
// finalized at v9 carry the old double-counted agentKv estimates and
|
||||
// sonnet-proxy composer costs. Raising MIN_SUPPORTED_VERSION forces the
|
||||
// one-time full re-hydration that backfills history under the new accounting.
|
||||
//
|
||||
// v8 added local-model savings to the daily rollup (savingsUSD per day / model /
|
||||
// category / provider). The `savingsConfigHash` field is invalidated separately
|
||||
// when the user changes their `localModelSavings` mapping so historical "saved"
|
||||
// totals stay in sync with the active baseline.
|
||||
export const DAILY_CACHE_VERSION = 9
|
||||
const MIN_SUPPORTED_VERSION = 9
|
||||
// v9: providers added since the v8 rollup (Grok, Hermes, ZCode) parse usage
|
||||
// that older binaries skipped. v8 added local-model savings to the daily
|
||||
// rollup; the `savingsConfigHash` field is invalidated separately when the
|
||||
// user changes their `localModelSavings` mapping.
|
||||
export const DAILY_CACHE_VERSION = 10
|
||||
const MIN_SUPPORTED_VERSION = 10
|
||||
const DAILY_CACHE_FILENAME = 'daily-cache.json'
|
||||
|
||||
export type DailyEntry = {
|
||||
|
|
|
|||
|
|
@ -40,10 +40,11 @@ const WEB_SEARCH_COST = 0.01
|
|||
const ONE_HOUR_CACHE_WRITE_MULTIPLIER_FROM_FIVE_MINUTE_RATE = 1.6
|
||||
|
||||
// Explicit USD/token prices that must override LiteLLM/cache data. Cursor
|
||||
// publishes house-model rates under provider "Cursor" in USD per 1M tokens:
|
||||
// composer-2/2.5: $0.50 input, $2.50 output, $0.20 cache read; composer-1.5:
|
||||
// $3.50/$17.50/$0.35; composer-1: $1.25/$10/$0.125. Cursor publishes no
|
||||
// separate cache-write rate for these, so cache write uses the input rate.
|
||||
// publishes house-model rates in the models table at cursor.com/docs/models
|
||||
// (provider "Cursor", USD per 1M tokens): composer-2/2.5: $0.50 input, $2.50
|
||||
// output, $0.20 cache read; composer-1.5: $3.50/$17.50/$0.35; composer-1:
|
||||
// $1.25/$10/$0.125. Cursor publishes no separate cache-write rate for these,
|
||||
// so cache write uses the input rate.
|
||||
const BUILTIN_PRICE_OVERRIDES: Record<string, SnapshotEntry> = {
|
||||
'composer-2.5': [0.5e-6, 2.5e-6, 0.5e-6, 0.2e-6],
|
||||
'composer-2': [0.5e-6, 2.5e-6, 0.5e-6, 0.2e-6],
|
||||
|
|
@ -501,8 +502,11 @@ export function getLocalModelSavingsConfigHash(): string {
|
|||
}
|
||||
|
||||
export function getPriceOverridesConfigHash(): string {
|
||||
// The builtin overrides participate so editing BUILTIN_PRICE_OVERRIDES in a
|
||||
// release invalidates cached daily costs the same way a user override does.
|
||||
const builtin = `builtin:${JSON.stringify(BUILTIN_PRICE_OVERRIDES)}`
|
||||
const keys = Object.keys(userPriceOverridesConfig).sort()
|
||||
if (keys.length === 0) return ''
|
||||
if (keys.length === 0) return builtin
|
||||
const parts = keys.map(k => {
|
||||
const rates = userPriceOverridesConfig[k]
|
||||
return [
|
||||
|
|
@ -513,7 +517,7 @@ export function getPriceOverridesConfigHash(): string {
|
|||
rates.cacheCreation ?? '',
|
||||
].join('\u0001')
|
||||
})
|
||||
return parts.join('\u0002')
|
||||
return [builtin, ...parts].join('\u0002')
|
||||
}
|
||||
|
||||
// Absolute directory prefixes whose sessions are routed through a
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
import { existsSync, readdirSync, readFileSync } from 'fs'
|
||||
import { existsSync, readdirSync, readFileSync, statSync } from 'fs'
|
||||
import { join } from 'path'
|
||||
import { homedir } from 'os'
|
||||
|
||||
import { calculateCost } from '../models.js'
|
||||
import { extractBashCommands } from '../bash-utils.js'
|
||||
import { readCachedResults, writeCachedResults } from '../cursor-cache.js'
|
||||
import { isSqliteAvailable, getSqliteLoadError, openDatabase, blobToText, type SqliteDatabase } from '../sqlite.js'
|
||||
import { isSqliteAvailable, isSqliteBusyError, getSqliteLoadError, openDatabase, blobToText, type SqliteDatabase } from '../sqlite.js'
|
||||
import type { DateRange } from '../types.js'
|
||||
import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js'
|
||||
|
||||
|
|
@ -48,7 +49,7 @@ type BubbleRow = {
|
|||
output_tokens: number | null
|
||||
model: string | null
|
||||
created_at: string | null
|
||||
conversation_id: string | null
|
||||
request_id: string | null
|
||||
user_text: Uint8Array | string | null
|
||||
text_length: number | null
|
||||
bubble_type: number | null
|
||||
|
|
@ -59,11 +60,18 @@ type BubbleRow = {
|
|||
}
|
||||
|
||||
type AgentKvRow = {
|
||||
key: string
|
||||
role: string | null
|
||||
content: Uint8Array | string | null
|
||||
request_id: string | null
|
||||
content_length: number
|
||||
model: string | null
|
||||
}
|
||||
|
||||
// SQLITE_BUSY must reach parser.ts, whose busy path skips the source without
|
||||
// caching; swallowing it here would stamp a silently degraded parse into the
|
||||
// results cache under an unchanged DB fingerprint (Cursor writes via WAL, so
|
||||
// contention does not change the main file's stat).
|
||||
function rethrowBusy(err: unknown): void {
|
||||
if (isSqliteBusyError(err)) throw err
|
||||
}
|
||||
|
||||
const CHARS_PER_TOKEN = 4
|
||||
|
|
@ -298,7 +306,7 @@ const BUBBLE_QUERY_BASE = `
|
|||
json_extract(value, '$.tokenCount.outputTokens') as output_tokens,
|
||||
json_extract(value, '$.modelInfo.modelName') as model,
|
||||
json_extract(value, '$.createdAt') as created_at,
|
||||
json_extract(value, '$.conversationId') as conversation_id,
|
||||
json_extract(value, '$.requestId') as request_id,
|
||||
CAST(substr(json_extract(value, '$.text'), 1, 500) AS BLOB) as user_text,
|
||||
length(json_extract(value, '$.text')) as text_length,
|
||||
json_extract(value, '$.type') as bubble_type,
|
||||
|
|
@ -309,11 +317,10 @@ const BUBBLE_QUERY_BASE = `
|
|||
|
||||
const AGENTKV_QUERY = `
|
||||
SELECT
|
||||
key,
|
||||
json_extract(value, '$.role') as role,
|
||||
CAST(json_extract(value, '$.content') AS BLOB) as content,
|
||||
json_extract(value, '$.providerOptions.cursor.requestId') as request_id,
|
||||
length(value) as content_length
|
||||
json_extract(value, '$.providerOptions.cursor.modelName') as model
|
||||
FROM cursorDiskKV
|
||||
WHERE key LIKE 'agentKv:blob:%'
|
||||
AND hex(substr(value, 1, 1)) = '7B'
|
||||
|
|
@ -356,7 +363,7 @@ const BUBBLE_QUERY_PAGE = `
|
|||
json_extract(value, '$.tokenCount.outputTokens') as output_tokens,
|
||||
json_extract(value, '$.modelInfo.modelName') as model,
|
||||
json_extract(value, '$.createdAt') as created_at,
|
||||
json_extract(value, '$.conversationId') as conversation_id,
|
||||
json_extract(value, '$.requestId') as request_id,
|
||||
CAST(substr(json_extract(value, '$.text'), 1, 500) AS BLOB) as user_text,
|
||||
length(json_extract(value, '$.text')) as text_length,
|
||||
json_extract(value, '$.type') as bubble_type,
|
||||
|
|
@ -373,7 +380,8 @@ function validateSchema(db: SqliteDatabase): boolean {
|
|||
"SELECT COUNT(*) as cnt FROM cursorDiskKV WHERE key LIKE 'bubbleId:%' LIMIT 1"
|
||||
)
|
||||
return rows.length > 0
|
||||
} catch {
|
||||
} catch (err) {
|
||||
rethrowBusy(err)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -406,7 +414,9 @@ function buildUserMessageMap(db: SqliteDatabase, timeFloor: string): Map<string,
|
|||
map.set(composerId, { messages: [text], pos: 0 })
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
} catch (err) {
|
||||
rethrowBusy(err)
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
|
@ -438,7 +448,8 @@ function scanBubblesPaged(
|
|||
let batch: BubbleRow[]
|
||||
try {
|
||||
batch = db.query<BubbleRow>(BUBBLE_QUERY_PAGE, [beforeRowId, BATCH])
|
||||
} catch {
|
||||
} catch (err) {
|
||||
rethrowBusy(err)
|
||||
break
|
||||
}
|
||||
if (batch.length === 0) break
|
||||
|
|
@ -468,82 +479,146 @@ function scanBubblesPaged(
|
|||
// contextTokensUsed (the in-app context meter). This is not cumulative per-turn,
|
||||
// so local SQLite undercounts admin-console usage; parity requires the opt-in
|
||||
// Cursor Admin API: POST api.cursor.com/teams/filtered-usage-events.
|
||||
// Keyed by composerId so parseBubbles can credit it to the right conversation.
|
||||
const COMPOSER_TOKENS_QUERY = `
|
||||
// The key-range predicate seeks the primary key instead of scanning the table.
|
||||
const COMPOSER_META_QUERY = `
|
||||
SELECT
|
||||
substr(key, 14) as composer_id,
|
||||
substr(key, length('composerData:') + 1) as composer_id,
|
||||
json_extract(value, '$.promptTokenBreakdown.totalUsedTokens') as used,
|
||||
json_extract(value, '$.contextTokensUsed') as ctx
|
||||
json_extract(value, '$.contextTokensUsed') as ctx,
|
||||
json_extract(value, '$.createdAt') as created_at
|
||||
FROM cursorDiskKV
|
||||
WHERE key LIKE 'composerData:%'
|
||||
WHERE key >= 'composerData:' AND key < 'composerData;'
|
||||
`
|
||||
|
||||
function loadComposerInputTokens(db: SqliteDatabase): Map<string, number> {
|
||||
const map = new Map<string, number>()
|
||||
type ComposerMeta = { tokens: number; createdAt: number | null }
|
||||
|
||||
function loadComposerMeta(db: SqliteDatabase): Map<string, ComposerMeta> {
|
||||
const map = new Map<string, ComposerMeta>()
|
||||
try {
|
||||
const rows = db.query<{ composer_id: string; used: number | null; ctx: number | null }>(COMPOSER_TOKENS_QUERY)
|
||||
const rows = db.query<{ composer_id: string; used: number | null; ctx: number | null; created_at: number | null }>(COMPOSER_META_QUERY)
|
||||
for (const r of rows) {
|
||||
const tokens = r.used ?? r.ctx ?? 0
|
||||
if (r.composer_id && tokens > 0) map.set(r.composer_id, tokens)
|
||||
// `||` rather than `??`: a recorded-but-zero breakdown must fall through
|
||||
// to the context meter instead of shadowing it.
|
||||
const tokens = (r.used || r.ctx) ?? 0
|
||||
if (r.composer_id && tokens > 0) map.set(r.composer_id, { tokens, createdAt: r.created_at ?? null })
|
||||
}
|
||||
} catch {
|
||||
} catch (err) {
|
||||
rethrowBusy(err)
|
||||
/* best-effort: callers fall back to the per-bubble text estimate */
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
type AgentTools = { tools: string[]; bash: string[]; userChars: number }
|
||||
type AgentStream = {
|
||||
tools: string[]
|
||||
bash: string[]
|
||||
userChars: number
|
||||
contextChars: number
|
||||
assistantChars: number
|
||||
model: string | null
|
||||
}
|
||||
|
||||
// Cursor logs the agent's tool calls (Read, Grep, Shell, ...) in agentKv blobs
|
||||
// keyed by requestId. Bubbles carry the same requestId plus the composerId, so
|
||||
// joining the two attributes each conversation's tools and Shell commands.
|
||||
const BUBBLE_REQUESTID_QUERY = `
|
||||
SELECT key as bubble_key, json_extract(value, '$.requestId') as request_id
|
||||
FROM cursorDiskKV
|
||||
WHERE key LIKE 'bubbleId:%' AND json_extract(value, '$.requestId') IS NOT NULL
|
||||
`
|
||||
function newAgentStream(): AgentStream {
|
||||
return { tools: [], bash: [], userChars: 0, contextChars: 0, assistantChars: 0, model: null }
|
||||
}
|
||||
|
||||
function loadAgentToolsByComposer(db: SqliteDatabase): Map<string, AgentTools> {
|
||||
const byComposer = new Map<string, AgentTools>()
|
||||
|
||||
const requestToComposer = new Map<string, string>()
|
||||
try {
|
||||
const rows = db.query<{ bubble_key: string; request_id: string | null }>(BUBBLE_REQUESTID_QUERY)
|
||||
for (const r of rows) {
|
||||
const composer = parseComposerIdFromKey(r.bubble_key)
|
||||
if (composer && r.request_id) requestToComposer.set(r.request_id, composer)
|
||||
// agentKv rows store content as a plain string or a block array; count only
|
||||
// the text inside blocks so the JSON envelope and non-text parts are not
|
||||
// billed as prompt characters.
|
||||
function contentTextLength(raw: string): number {
|
||||
const trimmed = raw.trimStart()
|
||||
if (trimmed.startsWith('[') || trimmed.startsWith('{')) {
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed) as unknown
|
||||
const blocks = Array.isArray(parsed) ? parsed : [parsed]
|
||||
let len = 0
|
||||
for (const block of blocks) {
|
||||
if (block == null || typeof block !== 'object') continue
|
||||
const b = block as { text?: unknown; content?: unknown }
|
||||
if (typeof b.text === 'string') len += b.text.length
|
||||
else if (typeof b.content === 'string') len += b.content.length
|
||||
}
|
||||
return len
|
||||
} catch {
|
||||
return raw.length
|
||||
}
|
||||
} catch {
|
||||
return byComposer
|
||||
}
|
||||
return raw.length
|
||||
}
|
||||
|
||||
// Cursor logs the agent's stream (prompt, injected context, tool calls, reply
|
||||
// deltas) in agentKv blobs keyed by requestId. Bubbles carry the same
|
||||
// requestId, so the map built from the scanned bubbles joins each request to
|
||||
// its conversation. Requests with no matching bubble are kept separately:
|
||||
// they are real sessions (background runs, older builds) that would otherwise
|
||||
// vanish from totals.
|
||||
function loadAgentStreams(
|
||||
db: SqliteDatabase,
|
||||
requestToComposer: Map<string, string>,
|
||||
): { byComposer: Map<string, AgentStream>; unjoined: Map<string, AgentStream> } {
|
||||
const byComposer = new Map<string, AgentStream>()
|
||||
const unjoined = new Map<string, AgentStream>()
|
||||
|
||||
let rows: AgentKvRow[]
|
||||
try {
|
||||
rows = db.query<AgentKvRow>(AGENTKV_QUERY)
|
||||
} catch {
|
||||
return byComposer
|
||||
} catch (err) {
|
||||
rethrowBusy(err)
|
||||
return { byComposer, unjoined }
|
||||
}
|
||||
|
||||
// Only the turn-opening (user) agentKv row carries the requestId; the
|
||||
// assistant rows that follow inherit it, so track it positionally.
|
||||
let currentRequestId: string | null = null
|
||||
for (const row of rows) {
|
||||
if (row.request_id) currentRequestId = row.request_id
|
||||
if (!row.content || !currentRequestId) continue
|
||||
const composer = requestToComposer.get(currentRequestId)
|
||||
if (!composer) continue
|
||||
const bucketFor = (requestId: string): AgentStream => {
|
||||
const composer = requestToComposer.get(requestId)
|
||||
const map = composer ? byComposer : unjoined
|
||||
const key = composer ?? requestId
|
||||
const existing = map.get(key)
|
||||
if (existing) return existing
|
||||
const fresh = newAgentStream()
|
||||
map.set(key, fresh)
|
||||
return fresh
|
||||
}
|
||||
|
||||
// A user turn stores its prompt (plus injected context) as a plain string.
|
||||
// Track its length so a turn whose bubble text is empty — non-Composer
|
||||
// sessions such as GPT keep it in the agent stream — can still be
|
||||
// estimated instead of dropped.
|
||||
if (row.role === 'user') {
|
||||
const bucket = byComposer.get(composer) ?? { tools: [], bash: [], userChars: 0 }
|
||||
bucket.userChars += blobToText(row.content).length
|
||||
byComposer.set(composer, bucket)
|
||||
// Only the turn-opening (user) agentKv row carries the requestId; rows that
|
||||
// follow inherit it. Rows written BEFORE their request's id appears (the
|
||||
// system prompt and opening user prompt at a conversation start) buffer
|
||||
// until the next id, and a system row closes the previous request so
|
||||
// interleaved sessions cannot inherit across a conversation boundary.
|
||||
let currentRequestId: string | null = null
|
||||
let pendingUserChars = 0
|
||||
let pendingContextChars = 0
|
||||
for (const row of rows) {
|
||||
if (row.request_id) {
|
||||
currentRequestId = row.request_id
|
||||
if (pendingUserChars > 0 || pendingContextChars > 0) {
|
||||
const bucket = bucketFor(currentRequestId)
|
||||
bucket.userChars += pendingUserChars
|
||||
bucket.contextChars += pendingContextChars
|
||||
pendingUserChars = 0
|
||||
pendingContextChars = 0
|
||||
}
|
||||
}
|
||||
if (row.model && currentRequestId) {
|
||||
const bucket = bucketFor(currentRequestId)
|
||||
if (!bucket.model) bucket.model = row.model
|
||||
}
|
||||
if (!row.content) continue
|
||||
|
||||
if (row.role === 'system') {
|
||||
pendingContextChars += contentTextLength(blobToText(row.content))
|
||||
currentRequestId = null
|
||||
continue
|
||||
}
|
||||
if (row.role !== 'assistant') continue
|
||||
if (row.role === 'user') {
|
||||
const len = contentTextLength(blobToText(row.content))
|
||||
if (currentRequestId) bucketFor(currentRequestId).userChars += len
|
||||
else pendingUserChars += len
|
||||
continue
|
||||
}
|
||||
if (row.role === 'tool') {
|
||||
if (currentRequestId) bucketFor(currentRequestId).contextChars += contentTextLength(blobToText(row.content))
|
||||
continue
|
||||
}
|
||||
if (row.role !== 'assistant' || !currentRequestId) continue
|
||||
|
||||
let content: unknown
|
||||
try {
|
||||
|
|
@ -552,48 +627,50 @@ function loadAgentToolsByComposer(db: SqliteDatabase): Map<string, AgentTools> {
|
|||
continue
|
||||
}
|
||||
if (!Array.isArray(content)) continue
|
||||
const bucket = byComposer.get(composer) ?? { tools: [], bash: [], userChars: 0 }
|
||||
for (const block of content as Array<{ type?: string; toolName?: string; args?: { command?: string } }>) {
|
||||
if (!block || block.type !== 'tool-call' || !block.toolName) continue
|
||||
bucket.tools.push(block.toolName)
|
||||
if (block.toolName === 'Shell') {
|
||||
const command = block.args?.command?.trim()
|
||||
if (command) bucket.bash.push(command)
|
||||
const bucket = bucketFor(currentRequestId)
|
||||
for (const block of content as Array<{ type?: string; text?: unknown; toolName?: unknown; args?: { command?: unknown } }>) {
|
||||
if (block == null || typeof block !== 'object') continue
|
||||
if (typeof block.text === 'string') bucket.assistantChars += block.text.length
|
||||
if (block.type !== 'tool-call' || typeof block.toolName !== 'string' || !block.toolName) continue
|
||||
// Cursor's terminal tool is 'Shell'; emit the canonical 'Bash' so the
|
||||
// cross-provider tool and command breakdowns merge.
|
||||
bucket.tools.push(block.toolName === 'Shell' ? 'Bash' : block.toolName)
|
||||
if (block.toolName === 'Shell' && typeof block.args?.command === 'string') {
|
||||
bucket.bash.push(...extractBashCommands(block.args.command))
|
||||
}
|
||||
}
|
||||
byComposer.set(composer, bucket)
|
||||
}
|
||||
return byComposer
|
||||
return { byComposer, unjoined }
|
||||
}
|
||||
|
||||
// What drives a conversation's input figure, decided once per conversation so
|
||||
// the sources can never stack on each other:
|
||||
// bubbleTokens - some bubble carries a real tokenCount (older builds), so
|
||||
// per-turn counts are authoritative and nothing is estimated.
|
||||
// meter - the composerData context meter exists; one conversation
|
||||
// record carries it.
|
||||
// stream - no meter, but the agent stream holds the prompt/context; one
|
||||
// conversation record carries the estimate.
|
||||
// text - only visible bubble text exists; estimated per bubble.
|
||||
type InputSource = 'bubbleTokens' | 'meter' | 'stream' | 'text'
|
||||
|
||||
type ComposerScan = {
|
||||
hasRealTokens: boolean
|
||||
firstBubbleTs: string | null
|
||||
assistantTextChars: number
|
||||
model: string | null
|
||||
}
|
||||
|
||||
function parseBubbles(
|
||||
db: SqliteDatabase,
|
||||
seenKeys: Set<string>,
|
||||
timeFloor: string,
|
||||
composerInput: Map<string, number>,
|
||||
agentTools: Map<string, AgentTools>,
|
||||
agentKvTimestamp: string,
|
||||
): { calls: ParsedProviderCall[] } {
|
||||
const results: ParsedProviderCall[] = []
|
||||
let skipped = 0
|
||||
// Each conversation's real context is credited once (on its first turn) so a
|
||||
// multi-turn chat does not multiply the snapshot across every bubble.
|
||||
const creditedComposers = new Set<string>()
|
||||
|
||||
// Build a composerId -> model map from assistant bubbles. User bubbles
|
||||
// (type=1) carry no modelInfo, so when we credit real input tokens onto a
|
||||
// user bubble we need the conversation's actual model for pricing.
|
||||
const composerModel = new Map<string, string>()
|
||||
try {
|
||||
const modelRows = db.query<{ bubble_key: string; model: string | null }>(`
|
||||
SELECT key as bubble_key, json_extract(value, '$.modelInfo.modelName') as model
|
||||
FROM cursorDiskKV
|
||||
WHERE key LIKE 'bubbleId:%' AND json_extract(value, '$.modelInfo.modelName') IS NOT NULL
|
||||
`)
|
||||
for (const r of modelRows) {
|
||||
const cid = parseComposerIdFromKey(r.bubble_key)
|
||||
if (cid && r.model && !composerModel.has(cid)) composerModel.set(cid, r.model)
|
||||
}
|
||||
} catch { /* best-effort */ }
|
||||
const composerMeta = loadComposerMeta(db)
|
||||
|
||||
// The bubble timestamp lives inside the JSON value (no index), so the date
|
||||
// filter forces a full JSON decode per row. Multi-GB Cursor DBs (500k+
|
||||
|
|
@ -612,9 +689,9 @@ function parseBubbles(
|
|||
"SELECT COUNT(*) as cnt FROM cursorDiskKV WHERE key LIKE 'bubbleId:%'"
|
||||
)
|
||||
total = countRows[0]?.cnt ?? 0
|
||||
} catch { /* best-effort */ }
|
||||
|
||||
const userMessages = buildUserMessageMap(db, timeFloor)
|
||||
} catch (err) {
|
||||
rethrowBusy(err)
|
||||
}
|
||||
|
||||
let rows: BubbleRow[]
|
||||
try {
|
||||
|
|
@ -631,121 +708,141 @@ function parseBubbles(
|
|||
} else {
|
||||
rows = db.query<BubbleRow>(BUBBLE_QUERY_SINCE, [timeFloor])
|
||||
}
|
||||
} catch {
|
||||
} catch (err) {
|
||||
rethrowBusy(err)
|
||||
return { calls: results }
|
||||
}
|
||||
|
||||
// Pre-pass: per-conversation facts the crediting decisions need, plus the
|
||||
// requestId join for the agent stream — all from the rows already fetched,
|
||||
// so no extra unbudgeted table scans.
|
||||
const scans = new Map<string, ComposerScan>()
|
||||
const requestToComposer = new Map<string, string>()
|
||||
for (const row of rows) {
|
||||
const cid = parseComposerIdFromKey(row.bubble_key)
|
||||
if (!cid) continue
|
||||
if (row.request_id) requestToComposer.set(row.request_id, cid)
|
||||
let scan = scans.get(cid)
|
||||
if (!scan) {
|
||||
scan = { hasRealTokens: false, firstBubbleTs: null, assistantTextChars: 0, model: null }
|
||||
scans.set(cid, scan)
|
||||
}
|
||||
if ((row.input_tokens ?? 0) > 0 || (row.output_tokens ?? 0) > 0) scan.hasRealTokens = true
|
||||
if (!scan.firstBubbleTs && row.created_at) scan.firstBubbleTs = row.created_at
|
||||
if (row.bubble_type !== 1) scan.assistantTextChars += row.text_length ?? 0
|
||||
if (!scan.model && row.model) scan.model = row.model
|
||||
}
|
||||
|
||||
const { byComposer: agentStreams, unjoined } = loadAgentStreams(db, requestToComposer)
|
||||
const userMessages = buildUserMessageMap(db, timeFloor)
|
||||
const lastUserMsg = new Map<string, string>()
|
||||
|
||||
const inputSource = (cid: string): InputSource => {
|
||||
if (scans.get(cid)?.hasRealTokens) return 'bubbleTokens'
|
||||
if (composerMeta.has(cid)) return 'meter'
|
||||
const stream = agentStreams.get(cid)
|
||||
if ((stream?.userChars ?? 0) + (stream?.contextChars ?? 0) > 0) return 'stream'
|
||||
return 'text'
|
||||
}
|
||||
|
||||
const emit = (call: Omit<ParsedProviderCall, 'provider' | 'speed' | 'cacheCreationInputTokens' | 'cacheReadInputTokens' | 'cachedInputTokens' | 'reasoningTokens' | 'webSearchRequests' | 'costIsEstimated'>): void => {
|
||||
results.push({
|
||||
provider: 'cursor',
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
speed: 'standard',
|
||||
// Output is a reply-text estimate and the input meter is the latest
|
||||
// context snapshot, not a per-turn sum, so no cursor figure is exact.
|
||||
costIsEstimated: true,
|
||||
...call,
|
||||
})
|
||||
}
|
||||
|
||||
const toolsAttached = new Set<string>()
|
||||
for (const row of rows) {
|
||||
try {
|
||||
// The JSON `conversationId` field on bubbles is empty in current Cursor
|
||||
// builds. The real composerId lives in the row key
|
||||
// `bubbleId:<composerId>:<bubbleUuid>`. parseComposerIdFromKey returns
|
||||
// null for non-UUID composer segments (Cursor stores tool-call output
|
||||
// under `bubbleId:task-call_xxx\nfc_yyy:<bubbleUuid>` and similar shapes),
|
||||
// which are NOT standalone sessions.
|
||||
const parsedComposerId = parseComposerIdFromKey(row.bubble_key)
|
||||
if (!parsedComposerId) {
|
||||
// The real composerId lives in the row key `bubbleId:<composerId>:<uuid>`
|
||||
// (the JSON conversationId field is empty in current builds).
|
||||
// parseComposerIdFromKey returns null for non-UUID composer segments
|
||||
// (tool-call output rows and similar shapes), which are NOT sessions.
|
||||
const conversationId = parseComposerIdFromKey(row.bubble_key)
|
||||
if (!conversationId) {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
const conversationId = parsedComposerId
|
||||
|
||||
const createdAt = row.created_at ?? ''
|
||||
const createdAt = row.created_at
|
||||
if (!createdAt) continue
|
||||
|
||||
// Pair each user turn with its own prompt (even when the turn itself
|
||||
// emits nothing) so the assistant reply that follows classifies against
|
||||
// the right question.
|
||||
if (row.bubble_type === 1) {
|
||||
lastUserMsg.set(conversationId, takeUserMessage(userMessages, conversationId))
|
||||
}
|
||||
|
||||
let inputTokens = row.input_tokens ?? 0
|
||||
let outputTokens = row.output_tokens ?? 0
|
||||
// The conversation's tools/bash attach to the single call that carries its
|
||||
// real input (its first turn), so they are counted exactly once.
|
||||
let creditedHere = false
|
||||
|
||||
// Current Cursor leaves tokenCount at {0,0}. Use the latest local
|
||||
// context-window snapshot for input, credited once per conversation; it is
|
||||
// not cumulative per-turn, so it undercounts Cursor Admin console totals.
|
||||
// Output is a reply-text estimate, and cache tokens are server-side only
|
||||
// (0 on disk). Admin-console parity requires POST
|
||||
// api.cursor.com/teams/filtered-usage-events.
|
||||
// Fall back to the visible-text estimate only when no breakdown was
|
||||
// recorded (older builds).
|
||||
if (inputTokens === 0 && outputTokens === 0) {
|
||||
const textLen = row.text_length ?? 0
|
||||
if (row.bubble_type === 1) {
|
||||
const real = composerInput.get(conversationId)
|
||||
if (real != null) {
|
||||
if (creditedComposers.has(conversationId)) {
|
||||
inputTokens = 0
|
||||
} else {
|
||||
inputTokens = real
|
||||
creditedComposers.add(conversationId)
|
||||
creditedHere = true
|
||||
}
|
||||
} else if (textLen > 0) {
|
||||
// Conversation-level input (meter or stream) is emitted once after
|
||||
// this loop; per-bubble text only counts when it is the
|
||||
// conversation's best available signal.
|
||||
if (inputSource(conversationId) === 'text' && textLen > 0) {
|
||||
inputTokens = Math.ceil(textLen / CHARS_PER_TOKEN)
|
||||
} else if (!creditedComposers.has(conversationId)) {
|
||||
// Non-Composer sessions (e.g. GPT) record no context meter and keep
|
||||
// the prompt in the agent stream, leaving the bubble text empty.
|
||||
// Estimate from the stream text, credited once per conversation.
|
||||
const agentChars = agentTools.get(conversationId)?.userChars ?? 0
|
||||
if (agentChars > 0) {
|
||||
inputTokens = Math.ceil(agentChars / CHARS_PER_TOKEN)
|
||||
creditedComposers.add(conversationId)
|
||||
creditedHere = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
outputTokens = Math.ceil(textLen / CHARS_PER_TOKEN)
|
||||
}
|
||||
if (inputTokens === 0 && outputTokens === 0) continue
|
||||
}
|
||||
|
||||
// Use the SQLite row key (bubbleId:<unique>) as the dedup key.
|
||||
// Cursor mutates token counts on the row in place when streaming
|
||||
// completes — including tokens in the dedup key (the previous
|
||||
// implementation) caused the same bubble to be counted twice once
|
||||
// its tokens stabilized.
|
||||
const dedupKey = `cursor:bubble:${row.bubble_key}`
|
||||
|
||||
if (seenKeys.has(dedupKey)) continue
|
||||
seenKeys.add(dedupKey)
|
||||
|
||||
// User bubbles (type=1) carry no modelInfo, so when real input tokens
|
||||
// are credited onto them, fall back to the conversation's model (found
|
||||
// on the assistant bubble) for pricing and display.
|
||||
const effectiveModel = row.model ?? composerModel.get(conversationId) ?? null
|
||||
// User bubbles (type=1) carry no modelInfo, so fall back to the
|
||||
// conversation's model seen on its assistant bubbles or agent stream.
|
||||
const effectiveModel = row.model ?? scans.get(conversationId)?.model ?? agentStreams.get(conversationId)?.model ?? null
|
||||
const pricingModel = resolveModel(effectiveModel)
|
||||
const displayModel = modelForDisplay(effectiveModel)
|
||||
|
||||
const costUSD = calculateCost(pricingModel, inputTokens, outputTokens, 0, 0, 0)
|
||||
|
||||
const timestamp = createdAt
|
||||
const userQuestion = takeUserMessage(userMessages, conversationId)
|
||||
const userQuestion = lastUserMsg.get(conversationId) ?? ''
|
||||
const assistantText = blobToText(row.user_text)
|
||||
const userText = (userQuestion + ' ' + assistantText).trim()
|
||||
|
||||
const languages = extractLanguages(blobToText(row.code_blocks))
|
||||
const hasCode = languages.length > 0
|
||||
|
||||
const agentTurn = creditedHere ? agentTools.get(conversationId) : undefined
|
||||
const cursorTools: string[] = [
|
||||
...(hasCode ? ['cursor:edit', ...languages.map(l => `lang:${l}`)] : []),
|
||||
...(agentTurn?.tools ?? []),
|
||||
]
|
||||
const bashCommands = agentTurn?.bash ?? []
|
||||
// Meter/stream conversations carry their agent tools on the synthetic
|
||||
// conversation record below; the rest attach them to their first
|
||||
// emitted call so they are counted exactly once.
|
||||
let agentTurn: AgentStream | undefined
|
||||
const source = inputSource(conversationId)
|
||||
if ((source === 'text' || source === 'bubbleTokens') && !toolsAttached.has(conversationId)) {
|
||||
agentTurn = agentStreams.get(conversationId)
|
||||
if (agentTurn) toolsAttached.add(conversationId)
|
||||
}
|
||||
|
||||
results.push({
|
||||
provider: 'cursor',
|
||||
model: displayModel,
|
||||
emit({
|
||||
model: modelForDisplay(effectiveModel),
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
costUSD,
|
||||
tools: cursorTools,
|
||||
bashCommands,
|
||||
timestamp,
|
||||
speed: 'standard',
|
||||
tools: [
|
||||
...(hasCode ? ['cursor:edit', ...languages.map(l => `lang:${l}`)] : []),
|
||||
...(agentTurn?.tools ?? []),
|
||||
],
|
||||
bashCommands: agentTurn?.bash ?? [],
|
||||
timestamp: createdAt,
|
||||
deduplicationKey: dedupKey,
|
||||
userMessage: userText,
|
||||
sessionId: conversationId,
|
||||
|
|
@ -755,6 +852,75 @@ function parseBubbles(
|
|||
}
|
||||
}
|
||||
|
||||
// One conversation-level input record per metered/stream conversation,
|
||||
// anchored to the conversation's own start (composerData.createdAt) so the
|
||||
// credited day never depends on the parse window or cache state, and keyed
|
||||
// by composerId so re-parses and daily-cache gap fills dedupe instead of
|
||||
// multiplying. The meter is the LATEST context size, not a per-turn sum;
|
||||
// growth after the anchor day is finalized stays uncounted, which keeps the
|
||||
// documented undercount-vs-admin-console tradeoff but never double counts.
|
||||
for (const [cid, scan] of scans) {
|
||||
const source = inputSource(cid)
|
||||
if (source !== 'meter' && source !== 'stream') continue
|
||||
const stream = agentStreams.get(cid)
|
||||
const meta = composerMeta.get(cid)
|
||||
const inputTokens = source === 'meter'
|
||||
? meta?.tokens ?? 0
|
||||
: Math.ceil(((stream?.userChars ?? 0) + (stream?.contextChars ?? 0)) / CHARS_PER_TOKEN)
|
||||
// Reply text normally lives on assistant bubbles; count the stream's
|
||||
// reply deltas only when the bubbles carried none.
|
||||
const outputTokens = scan.assistantTextChars > 0 ? 0 : Math.ceil((stream?.assistantChars ?? 0) / CHARS_PER_TOKEN)
|
||||
if (inputTokens === 0 && outputTokens === 0) continue
|
||||
|
||||
const dedupKey = `cursor:composer-input:${cid}`
|
||||
if (seenKeys.has(dedupKey)) continue
|
||||
seenKeys.add(dedupKey)
|
||||
|
||||
const createdAtMs = meta?.createdAt
|
||||
const timestamp = typeof createdAtMs === 'number' && createdAtMs > 0 ? new Date(createdAtMs).toISOString() : scan.firstBubbleTs
|
||||
if (!timestamp) continue
|
||||
|
||||
const effectiveModel = scan.model ?? stream?.model ?? null
|
||||
emit({
|
||||
model: modelForDisplay(effectiveModel),
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
costUSD: calculateCost(resolveModel(effectiveModel), inputTokens, outputTokens, 0, 0, 0),
|
||||
tools: stream?.tools ?? [],
|
||||
bashCommands: stream?.bash ?? [],
|
||||
timestamp,
|
||||
deduplicationKey: dedupKey,
|
||||
userMessage: '',
|
||||
sessionId: cid,
|
||||
})
|
||||
}
|
||||
|
||||
// Sessions recorded only in the agent stream (no bubble carries their
|
||||
// requestId). agentKv stores no timestamps, so these reuse the DB file's
|
||||
// mtime as a bounded "last write" time, like the pre-composer parser did.
|
||||
for (const [requestId, stream] of unjoined) {
|
||||
const inputTokens = Math.ceil((stream.userChars + stream.contextChars) / CHARS_PER_TOKEN)
|
||||
const outputTokens = Math.ceil(stream.assistantChars / CHARS_PER_TOKEN)
|
||||
if (inputTokens === 0 && outputTokens === 0) continue
|
||||
|
||||
const dedupKey = `cursor:agentKv:${requestId}`
|
||||
if (seenKeys.has(dedupKey)) continue
|
||||
seenKeys.add(dedupKey)
|
||||
|
||||
emit({
|
||||
model: modelForDisplay(stream.model),
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
costUSD: calculateCost(resolveModel(stream.model), inputTokens, outputTokens, 0, 0, 0),
|
||||
tools: stream.tools,
|
||||
bashCommands: stream.bash,
|
||||
timestamp: agentKvTimestamp,
|
||||
deduplicationKey: dedupKey,
|
||||
userMessage: '',
|
||||
sessionId: requestId,
|
||||
})
|
||||
}
|
||||
|
||||
if (skipped > 0) {
|
||||
process.stderr.write(`codeburn: skipped ${skipped} unreadable Cursor entries\n`)
|
||||
}
|
||||
|
|
@ -815,6 +981,7 @@ function createParser(
|
|||
try {
|
||||
db = openDatabase(dbPath)
|
||||
} catch (err) {
|
||||
rethrowBusy(err)
|
||||
process.stderr.write(`codeburn: cannot open Cursor database: ${err instanceof Error ? err.message : err}\n`)
|
||||
return
|
||||
}
|
||||
|
|
@ -827,14 +994,15 @@ function createParser(
|
|||
// seenKeys is not mutated by calls that the workspace filter is
|
||||
// about to drop. Cross-source dedup happens at yield time.
|
||||
const localSeen = new Set<string>()
|
||||
// Real per-conversation input tokens from
|
||||
// composerData.promptTokenBreakdown supersedes the old agentKv
|
||||
// content-char estimate, which double-counted against the bubble
|
||||
// stream. agentKv is now used only for the tools/bash breakdown
|
||||
// via loadAgentToolsByComposer().
|
||||
const composerInput = loadComposerInputTokens(db)
|
||||
const agentTools = loadAgentToolsByComposer(db)
|
||||
const { calls: bubbleCalls } = parseBubbles(db, localSeen, timeFloor, composerInput, agentTools)
|
||||
// agentKv rows carry no timestamps; sessions found only there get
|
||||
// the DB's last-write time.
|
||||
let agentKvTimestamp: string
|
||||
try {
|
||||
agentKvTimestamp = new Date(statSync(dbPath).mtimeMs).toISOString()
|
||||
} catch {
|
||||
agentKvTimestamp = new Date().toISOString()
|
||||
}
|
||||
const { calls: bubbleCalls } = parseBubbles(db, localSeen, timeFloor, agentKvTimestamp)
|
||||
allCalls = bubbleCalls
|
||||
await writeCachedResults(dbPath, allCalls, timeFloor)
|
||||
} finally {
|
||||
|
|
|
|||
|
|
@ -110,6 +110,7 @@ export const DURABLE_PROVIDER_NAMES: ReadonlySet<string> = new Set(['copilot'])
|
|||
const PROVIDER_PARSE_VERSIONS: Record<string, string> = {
|
||||
claude: 'cowork-space-grouping-v1',
|
||||
cline: 'worktree-project-grouping-v1',
|
||||
cursor: 'composer-anchored-crediting-v1',
|
||||
'cursor-agent': 'workspaceless-transcript-v1',
|
||||
copilot: 'otel-durable-v1',
|
||||
hermes: 'reasoning-output-accounting-v1',
|
||||
|
|
|
|||
|
|
@ -316,13 +316,16 @@ describe('user price overrides', () => {
|
|||
expect(mini!.outputCostPerToken).toBe(miniSnapshot!.outputCostPerToken)
|
||||
})
|
||||
|
||||
it('includes price overrides in the daily cache config hash without changing the empty-override hash', () => {
|
||||
it('includes builtin and user price overrides in the daily cache config hash', () => {
|
||||
setLocalModelSavings({ local: 'gpt-4o' })
|
||||
setPriceOverrides({})
|
||||
|
||||
const savingsOnly = getLocalModelSavingsConfigHash()
|
||||
expect(getPriceOverridesConfigHash()).toBe('')
|
||||
expect(getDailyCacheConfigHash()).toBe(savingsOnly)
|
||||
// The builtin overrides always participate, so a release that edits them
|
||||
// invalidates cached daily costs even with no user overrides configured.
|
||||
const builtinOnly = getPriceOverridesConfigHash()
|
||||
expect(builtinOnly).toContain('builtin:')
|
||||
expect(getPriceOverridesConfigHash()).toBe(builtinOnly)
|
||||
const baseline = getDailyCacheConfigHash()
|
||||
|
||||
setPriceOverrides({ 'price-hash-model': { input: 1, output: 2 } })
|
||||
const firstCombined = getDailyCacheConfigHash()
|
||||
|
|
@ -330,8 +333,8 @@ describe('user price overrides', () => {
|
|||
setPriceOverrides({ 'price-hash-model': { input: 3, output: 2 } })
|
||||
const secondCombined = getDailyCacheConfigHash()
|
||||
|
||||
expect(firstCombined).not.toBe(savingsOnly)
|
||||
expect(secondCombined).not.toBe(savingsOnly)
|
||||
expect(firstCombined).not.toBe(baseline)
|
||||
expect(secondCombined).not.toBe(baseline)
|
||||
expect(secondCombined).not.toBe(firstCombined)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { mkdtemp, rm, writeFile } from 'fs/promises'
|
||||
import { mkdtemp, rm } from 'fs/promises'
|
||||
import { tmpdir } from 'os'
|
||||
import { join } from 'path'
|
||||
import { createRequire } from 'node:module'
|
||||
|
|
@ -35,7 +35,6 @@ function buildDb(fn: (db: {
|
|||
close(): void
|
||||
}) => void): string {
|
||||
const dbPath = join(tmpDir, 'state.vscdb')
|
||||
writeFile(dbPath, '')
|
||||
const { DatabaseSync: Database } = requireForTest('node:sqlite')
|
||||
const db = new Database(dbPath)
|
||||
db.exec('CREATE TABLE cursorDiskKV (key TEXT PRIMARY KEY, value BLOB)')
|
||||
|
|
@ -82,6 +81,7 @@ function insertComposerData(db: {
|
|||
composerId: string
|
||||
totalUsedTokens?: number | null
|
||||
contextTokensUsed?: number | null
|
||||
createdAt?: number
|
||||
}): void {
|
||||
const key = `composerData:${opts.composerId}`
|
||||
const breakdown = opts.totalUsedTokens !== undefined
|
||||
|
|
@ -90,6 +90,7 @@ function insertComposerData(db: {
|
|||
const value = JSON.stringify({
|
||||
promptTokenBreakdown: breakdown,
|
||||
contextTokensUsed: opts.contextTokensUsed ?? undefined,
|
||||
createdAt: opts.createdAt ?? undefined,
|
||||
})
|
||||
db.prepare('INSERT INTO cursorDiskKV (key, value) VALUES (?, ?)').run(key, value)
|
||||
}
|
||||
|
|
@ -140,17 +141,16 @@ describe.skipIf(skipReason !== null)('cursor real context tokens (#575)', () =>
|
|||
const provider = createCursorProvider(dbPath)
|
||||
const calls = await collectCalls(provider, dbPath)
|
||||
|
||||
// User bubble gets the real context; assistant gets text estimate.
|
||||
const userCall = calls.find(c => c.inputTokens === 50000)
|
||||
expect(userCall).toBeDefined()
|
||||
expect(userCall!.inputTokens).toBe(50000)
|
||||
const credited = calls.find(c => c.inputTokens === 50000)
|
||||
expect(credited).toBeDefined()
|
||||
expect(credited!.deduplicationKey).toBe(`cursor:composer-input:${composerId}`)
|
||||
expect(credited!.costIsEstimated).toBe(true)
|
||||
})
|
||||
|
||||
it('credits real input tokens once per conversation, not per bubble', async () => {
|
||||
const composerId = 'bbbb1111-2222-3333-4444-555566667777'
|
||||
const dbPath = buildDb((db) => {
|
||||
insertComposerData(db, { composerId, totalUsedTokens: 30000 })
|
||||
// Multiple user bubbles in the same conversation
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b1', type: 1, text: 'turn 1' })
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b2', type: 2, text: 'reply 1', model: 'gpt-5' })
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b3', type: 1, text: 'turn 2' })
|
||||
|
|
@ -160,15 +160,32 @@ describe.skipIf(skipReason !== null)('cursor real context tokens (#575)', () =>
|
|||
const provider = createCursorProvider(dbPath)
|
||||
const calls = await collectCalls(provider, dbPath)
|
||||
|
||||
// Exactly one call should have 30000 input tokens
|
||||
const credited = calls.filter(c => c.inputTokens === 30000)
|
||||
expect(credited.length).toBe(1)
|
||||
// The metered conversation's user-bubble text must not be counted on top.
|
||||
const inputTotal = calls.reduce((s, c) => s + c.inputTokens, 0)
|
||||
expect(inputTotal).toBe(30000)
|
||||
})
|
||||
|
||||
it('anchors the conversation record to composerData.createdAt, independent of the parse window', async () => {
|
||||
const composerId = 'ab121111-2222-3333-4444-555566667777'
|
||||
const startMs = Date.parse('2026-06-01T10:00:00.000Z')
|
||||
const dbPath = buildDb((db) => {
|
||||
insertComposerData(db, { composerId, totalUsedTokens: 20000, createdAt: startMs })
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b1', type: 1, text: 'later turn' })
|
||||
})
|
||||
|
||||
const provider = createCursorProvider(dbPath)
|
||||
const calls = await collectCalls(provider, dbPath)
|
||||
|
||||
const credited = calls.find(c => c.inputTokens === 20000)
|
||||
expect(credited).toBeDefined()
|
||||
expect(credited!.timestamp).toBe(new Date(startMs).toISOString())
|
||||
})
|
||||
|
||||
it('falls back to text estimation when no composerData exists', async () => {
|
||||
const composerId = 'cccc1111-2222-3333-4444-555566667777'
|
||||
const dbPath = buildDb((db) => {
|
||||
// No composerData row
|
||||
insertBubble(db, {
|
||||
composerId, bubbleUuid: 'b1', type: 1, text: 'hello world this is a test',
|
||||
})
|
||||
|
|
@ -179,7 +196,6 @@ describe.skipIf(skipReason !== null)('cursor real context tokens (#575)', () =>
|
|||
|
||||
const userCall = calls.find(c => c.inputTokens > 0)
|
||||
expect(userCall).toBeDefined()
|
||||
// text length 25 / 4 = 7 tokens
|
||||
expect(userCall!.inputTokens).toBe(Math.ceil('hello world this is a test'.length / 4))
|
||||
})
|
||||
|
||||
|
|
@ -198,7 +214,38 @@ describe.skipIf(skipReason !== null)('cursor real context tokens (#575)', () =>
|
|||
expect(credited).toBeDefined()
|
||||
})
|
||||
|
||||
it('attributes aggregated agentKv tools once in a multi-bubble conversation', async () => {
|
||||
it('uses contextTokensUsed when totalUsedTokens is present but zero', async () => {
|
||||
const composerId = 'de001111-2222-3333-4444-555566667777'
|
||||
const dbPath = buildDb((db) => {
|
||||
insertComposerData(db, { composerId, totalUsedTokens: 0, contextTokensUsed: 42000 })
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b1', type: 1, text: 'prompt' })
|
||||
})
|
||||
|
||||
const provider = createCursorProvider(dbPath)
|
||||
const calls = await collectCalls(provider, dbPath)
|
||||
|
||||
const credited = calls.find(c => c.inputTokens === 42000)
|
||||
expect(credited).toBeDefined()
|
||||
})
|
||||
|
||||
it('skips the meter when any bubble carries real tokenCounts', async () => {
|
||||
const composerId = 'ef001111-2222-3333-4444-555566667777'
|
||||
const dbPath = buildDb((db) => {
|
||||
insertComposerData(db, { composerId, totalUsedTokens: 80000 })
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b1', type: 1, text: 'prompt', inputTokens: 6000 })
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b2', type: 2, text: 'reply', model: 'gpt-5', outputTokens: 900 })
|
||||
})
|
||||
|
||||
const provider = createCursorProvider(dbPath)
|
||||
const calls = await collectCalls(provider, dbPath)
|
||||
|
||||
// Real per-bubble counts are authoritative; the snapshot must not stack.
|
||||
expect(calls.find(c => c.inputTokens === 80000)).toBeUndefined()
|
||||
const inputTotal = calls.reduce((s, c) => s + c.inputTokens, 0)
|
||||
expect(inputTotal).toBe(6000)
|
||||
})
|
||||
|
||||
it('attributes aggregated agentKv tools once, with canonical Bash names', async () => {
|
||||
const composerId = 'eeee1111-2222-3333-4444-555566667777'
|
||||
const requestId = 'req-001'
|
||||
const dbPath = buildDb((db) => {
|
||||
|
|
@ -207,7 +254,6 @@ describe.skipIf(skipReason !== null)('cursor real context tokens (#575)', () =>
|
|||
insertBubble(db, { composerId, bubbleUuid: 'b2', type: 2, text: 'doing stuff', model: 'gpt-5' })
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b3', type: 1, text: 'do more stuff', requestId: 'req-002' })
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b4', type: 2, text: 'doing more stuff', model: 'gpt-5' })
|
||||
// agentKv with tool calls
|
||||
insertAgentKv(db, {
|
||||
blobId: 'akv-1', role: 'user',
|
||||
content: [{ type: 'text', text: 'do stuff' }],
|
||||
|
|
@ -228,17 +274,17 @@ describe.skipIf(skipReason !== null)('cursor real context tokens (#575)', () =>
|
|||
const callWithTools = calls.find(c => c.tools.length > 0)
|
||||
expect(callWithTools).toBeDefined()
|
||||
expect(callWithTools!.tools).toContain('Read')
|
||||
expect(callWithTools!.tools).toContain('Shell')
|
||||
expect(callWithTools!.bashCommands).toContain('npm test')
|
||||
expect(callWithTools!.tools).toContain('Bash')
|
||||
expect(callWithTools!.bashCommands).toContain('npm')
|
||||
|
||||
const allTools = calls.flatMap(c => c.tools)
|
||||
const allBashCommands = calls.flatMap(c => c.bashCommands)
|
||||
expect(allTools.filter(t => t === 'Read').length).toBe(1)
|
||||
expect(allTools.filter(t => t === 'Shell').length).toBe(1)
|
||||
expect(allBashCommands.filter(cmd => cmd === 'npm test').length).toBe(1)
|
||||
expect(allTools.filter(t => t === 'Bash').length).toBe(1)
|
||||
expect(allBashCommands.filter(cmd => cmd === 'npm').length).toBe(1)
|
||||
})
|
||||
|
||||
it('uses conversation model for pricing when input is on a user bubble', async () => {
|
||||
it('uses conversation model for pricing the conversation record', async () => {
|
||||
const composerId = 'ffff1111-2222-3333-4444-555566667777'
|
||||
const dbPath = buildDb((db) => {
|
||||
insertComposerData(db, { composerId, totalUsedTokens: 100000 })
|
||||
|
|
@ -254,9 +300,6 @@ describe.skipIf(skipReason !== null)('cursor real context tokens (#575)', () =>
|
|||
|
||||
const creditedCall = calls.find(c => c.inputTokens === 100000)
|
||||
expect(creditedCall).toBeDefined()
|
||||
// Should NOT be cursor-auto (the fallback for user bubbles without model)
|
||||
expect(creditedCall!.model).not.toBe('cursor-auto')
|
||||
// Should be the conversation's actual model
|
||||
expect(creditedCall!.model).toBe('claude-4.5-opus-high-thinking')
|
||||
})
|
||||
|
||||
|
|
@ -265,8 +308,6 @@ describe.skipIf(skipReason !== null)('cursor real context tokens (#575)', () =>
|
|||
const requestId = 'req-gpt-1'
|
||||
const prompt = '<user_info>OS: darwin</user_info> refactor the auth module and add tests'
|
||||
const dbPath = buildDb((db) => {
|
||||
// No composerData meter (non-Composer session, e.g. GPT), and the user
|
||||
// turn's text lives in the agent stream, so the bubble text is empty.
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b1', type: 1, text: '', requestId })
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b2', type: 2, text: 'done', model: 'gpt-5' })
|
||||
insertAgentKv(db, { blobId: 'akv-1', role: 'user', content: prompt, requestId })
|
||||
|
|
@ -280,6 +321,82 @@ describe.skipIf(skipReason !== null)('cursor real context tokens (#575)', () =>
|
|||
expect(credited!.inputTokens).toBe(Math.ceil(prompt.length / 4))
|
||||
})
|
||||
|
||||
it('counts tool and system stream rows as context for meterless sessions', async () => {
|
||||
const composerId = '77770000-1111-2222-3333-444455556666'
|
||||
const requestId = 'req-gpt-2'
|
||||
const prompt = 'summarize the repo'
|
||||
const toolResult = 'x'.repeat(4000)
|
||||
const dbPath = buildDb((db) => {
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b1', type: 1, text: '', requestId })
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b2', type: 2, text: 'done', model: 'gpt-5' })
|
||||
insertAgentKv(db, { blobId: 'akv-1', role: 'user', content: prompt, requestId })
|
||||
insertAgentKv(db, { blobId: 'akv-2', role: 'tool', content: [{ type: 'text', text: toolResult }] })
|
||||
})
|
||||
|
||||
const provider = createCursorProvider(dbPath)
|
||||
const calls = await collectCalls(provider, dbPath)
|
||||
|
||||
const credited = calls.find(c => c.inputTokens > 0)
|
||||
expect(credited).toBeDefined()
|
||||
expect(credited!.inputTokens).toBe(Math.ceil((prompt.length + toolResult.length) / 4))
|
||||
})
|
||||
|
||||
it('does not double count turns that also have bubble text in stream-estimated conversations', async () => {
|
||||
const composerId = '66660000-1111-2222-3333-444455556666'
|
||||
const requestId = 'req-gpt-3'
|
||||
const streamPrompt = 'the full prompt with injected context'
|
||||
const dbPath = buildDb((db) => {
|
||||
// Turn 1 has visible bubble text; turn 2's lives only in the stream.
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b1', type: 1, text: 'visible text', requestId })
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b2', type: 1, text: '' })
|
||||
insertAgentKv(db, { blobId: 'akv-1', role: 'user', content: streamPrompt, requestId })
|
||||
})
|
||||
|
||||
const provider = createCursorProvider(dbPath)
|
||||
const calls = await collectCalls(provider, dbPath)
|
||||
|
||||
const inputTotal = calls.reduce((s, c) => s + c.inputTokens, 0)
|
||||
expect(inputTotal).toBe(Math.ceil(streamPrompt.length / 4))
|
||||
})
|
||||
|
||||
it('emits sessions recorded only in the agent stream', async () => {
|
||||
const requestId = 'req-headless-1'
|
||||
const prompt = 'run the nightly data export'
|
||||
const reply = 'export completed with 3 warnings'
|
||||
const dbPath = buildDb((db) => {
|
||||
insertAgentKv(db, { blobId: 'akv-1', role: 'user', content: prompt, requestId })
|
||||
insertAgentKv(db, { blobId: 'akv-2', role: 'assistant', content: [{ type: 'text', text: reply }] })
|
||||
})
|
||||
|
||||
const provider = createCursorProvider(dbPath)
|
||||
const calls = await collectCalls(provider, dbPath)
|
||||
|
||||
const session = calls.find(c => c.deduplicationKey === `cursor:agentKv:${requestId}`)
|
||||
expect(session).toBeDefined()
|
||||
expect(session!.inputTokens).toBe(Math.ceil(prompt.length / 4))
|
||||
expect(session!.outputTokens).toBe(Math.ceil(reply.length / 4))
|
||||
})
|
||||
|
||||
it('pairs each assistant reply with its own turn\'s user question', async () => {
|
||||
const composerId = '55550000-1111-2222-3333-444455556666'
|
||||
const dbPath = buildDb((db) => {
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b1', type: 1, text: 'first question' })
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b2', type: 2, text: 'first reply', model: 'gpt-5' })
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b3', type: 1, text: 'second question' })
|
||||
insertBubble(db, { composerId, bubbleUuid: 'b4', type: 2, text: 'second reply', model: 'gpt-5' })
|
||||
})
|
||||
|
||||
const provider = createCursorProvider(dbPath)
|
||||
const calls = await collectCalls(provider, dbPath)
|
||||
|
||||
const firstReply = calls.find(c => c.userMessage.includes('first reply'))
|
||||
const secondReply = calls.find(c => c.userMessage.includes('second reply'))
|
||||
expect(firstReply).toBeDefined()
|
||||
expect(secondReply).toBeDefined()
|
||||
expect(firstReply!.userMessage).toContain('first question')
|
||||
expect(secondReply!.userMessage).toContain('second question')
|
||||
})
|
||||
|
||||
it('does not fabricate input when an empty-text turn has no agent stream', async () => {
|
||||
const composerId = '88880000-1111-2222-3333-444455556666'
|
||||
const dbPath = buildDb((db) => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue