mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-07-31 20:05:15 +00:00
refactor(core): tail migrations — crush, zcode, zed, forge, goose (phase 8, sqlite batch 1)
Category B (sqlite) variant of the bridge migration: the sqlite driver and every
SQL query stay CLI-side. Each provider's `readRecords` opens the database, runs
the same queries as before, and hands the resulting rows (blob and all) to a
pure core decoder; `toProviderCall` maps the rich, cost-free decode back onto
ParsedProviderCall, where cost re-enters via the parser.ts pricing pass.
Per provider:
- crush: session row + dominant-model query -> one combined record. Crush stores
cost in dollars, so a row with cost > 0 carries `measuredCostUSD` (costBasis
'measured'); a zero-cost row falls back to token estimation, arm order intact.
- zcode: model_usage + tool_usage row sets -> one composite record. Each turn's
tools still attach to the first non-skipped usage row of that turn only.
- zed: threads rows handed over compressed; zstd decompression, JSON parsing and
per-request/cumulative-remainder accounting are pure. The Node >= 22.15 zstd
capability check stays host-side.
- forge: conversation row handed over with `context` still serialized; JSON
parsing and per-message decode are pure. Bash base-name extraction (and its
strip-ansi dependency) stays CLI-side over the decoder's raw command strings.
- goose: session + assistant tool-message + first-user-message rows, BLOB
columns pre-converted to text host-side, bundled into one composite record.
Validator fixes (original behavior is the authority):
- forge: the draft replaced the pre-migration `mapToolName` switch with an
object-literal lookup. Tool names come straight from conversation JSON, so
names colliding with Object.prototype members ("constructor", "toString",
"__proto__", "hasOwnProperty") resolved to inherited Functions / the prototype
object and were pushed into `tools` as non-strings instead of falling through
to the identity default. Restored the switch and pinned the arm in the fixture.
- zed: the draft routed the "skipped N unreadable Zed threads" notice into
record diagnostics, which the bridge discards, silently dropping a warning the
pre-migration decode printed. Re-emitted host-side from the diagnostics count
and pinned with a stderr assertion.
- Fixture coverage extended for the arms that were regression-blind: forge's
prototype-named tool calls, zed's aggregate stderr line, and goose's
single-turn `toolSequence` omission plus the unparseable-timestamp fallback.
Parity was verified independently of the bridge tests with a git-show harness
that runs the same fixtures through the pre-migration provider files and asserts
field-for-field equality, including the extra arms above.
This commit is contained in:
iamtoruk
parent
104365414f
commit
afaf9ffc8a
39 changed files with 3626 additions and 775 deletions
|
|
@ -2,8 +2,12 @@ import { readFile } from 'fs/promises'
|
|||
import { join, resolve } from 'path'
|
||||
import { homedir, platform } from 'os'
|
||||
|
||||
import { decodeCrush } from '@codeburn/core/providers/crush'
|
||||
import type { CrushDecodedCall, CrushRawRecord } from '@codeburn/core/providers/crush'
|
||||
|
||||
import { isSqliteAvailable, getSqliteLoadError, openDatabase, type SqliteDatabase } from '../sqlite.js'
|
||||
import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js'
|
||||
import { createBridgedProvider } from './bridge.js'
|
||||
import type { Provider, SessionSource, ParsedProviderCall } from './types.js'
|
||||
|
||||
/// Crush stores per-project SQLite databases discovered through a JSON registry.
|
||||
/// We only read both. Schema source: charmbracelet/crush
|
||||
|
|
@ -93,13 +97,6 @@ function validateSchema(db: SqliteDatabase): boolean {
|
|||
}
|
||||
}
|
||||
|
||||
function epochSecondsToIso(epochSeconds: number | null): string {
|
||||
if (epochSeconds === null || !Number.isFinite(epochSeconds)) {
|
||||
return new Date(0).toISOString()
|
||||
}
|
||||
return new Date(epochSeconds * 1000).toISOString()
|
||||
}
|
||||
|
||||
function dominantModel(db: SqliteDatabase, sessionId: string): string {
|
||||
try {
|
||||
const rows = db.query<{ model: string | null }>(
|
||||
|
|
@ -117,80 +114,34 @@ function dominantModel(db: SqliteDatabase, sessionId: string): string {
|
|||
}
|
||||
}
|
||||
|
||||
function createParser(source: SessionSource, seenKeys: Set<string>): SessionParser {
|
||||
// Map one rich, cost-free-or-measured decoder call into the host's
|
||||
// ParsedProviderCall. Crush already stores cost in dollars, so a row with
|
||||
// `measuredCostUSD` maps to `costBasis: 'measured'` (the pricing pass leaves it
|
||||
// untouched); otherwise the row falls back to token-based estimation. Crush
|
||||
// never captures a user message, so it is hardcoded empty here rather than
|
||||
// carried through the rich decode.
|
||||
function toProviderCall(rich: CrushDecodedCall): ParsedProviderCall {
|
||||
const measured = rich.measuredCostUSD !== undefined
|
||||
return {
|
||||
async *parse(): AsyncGenerator<ParsedProviderCall> {
|
||||
if (!isSqliteAvailable()) {
|
||||
process.stderr.write(getSqliteLoadError() + '\n')
|
||||
return
|
||||
}
|
||||
|
||||
// Source paths are encoded as `<dbPath>:<sessionId>`. Split from the
|
||||
// right because dbPath may contain a colon on Windows (drive letter).
|
||||
const segments = source.path.split(':')
|
||||
const sessionId = segments[segments.length - 1]!
|
||||
const dbPath = segments.slice(0, -1).join(':')
|
||||
|
||||
let db: SqliteDatabase
|
||||
try {
|
||||
db = openDatabase(dbPath)
|
||||
} catch (err) {
|
||||
process.stderr.write(
|
||||
`codeburn: cannot open Crush database: ${err instanceof Error ? err.message : err}\n`,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
if (!validateSchema(db)) return
|
||||
|
||||
const rows = db.query<SessionRow>(
|
||||
`SELECT id, prompt_tokens, completion_tokens, cost, created_at, updated_at, message_count
|
||||
FROM sessions
|
||||
WHERE id = ? AND parent_session_id IS NULL`,
|
||||
[sessionId],
|
||||
)
|
||||
if (rows.length === 0) return
|
||||
const session = rows[0]!
|
||||
|
||||
const inputTokens = session.prompt_tokens ?? 0
|
||||
const outputTokens = session.completion_tokens ?? 0
|
||||
const cost = session.cost ?? 0
|
||||
if (inputTokens === 0 && outputTokens === 0 && cost === 0) return
|
||||
|
||||
const dedupKey = `crush:${sessionId}`
|
||||
if (seenKeys.has(dedupKey)) return
|
||||
seenKeys.add(dedupKey)
|
||||
|
||||
const model = dominantModel(db, sessionId)
|
||||
// Crush already records cost in dollars; trust it. Fall back to
|
||||
// host-side pricing-table calculation only when the row is missing a cost.
|
||||
|
||||
yield {
|
||||
provider: 'crush',
|
||||
model,
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
...(cost > 0
|
||||
? { costUSD: cost, costBasis: 'measured' as const }
|
||||
: { costBasis: 'estimated' as const }),
|
||||
tools: [],
|
||||
bashCommands: [],
|
||||
timestamp: epochSecondsToIso(session.updated_at ?? session.created_at),
|
||||
speed: 'standard',
|
||||
deduplicationKey: dedupKey,
|
||||
userMessage: '',
|
||||
sessionId,
|
||||
}
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
},
|
||||
provider: 'crush',
|
||||
model: rich.model,
|
||||
inputTokens: rich.inputTokens,
|
||||
outputTokens: rich.outputTokens,
|
||||
cacheCreationInputTokens: rich.cacheCreationInputTokens,
|
||||
cacheReadInputTokens: rich.cacheReadInputTokens,
|
||||
cachedInputTokens: rich.cachedInputTokens,
|
||||
reasoningTokens: rich.reasoningTokens,
|
||||
webSearchRequests: rich.webSearchRequests,
|
||||
...(measured
|
||||
? { costUSD: rich.measuredCostUSD, costBasis: 'measured' as const }
|
||||
: { costBasis: 'estimated' as const }),
|
||||
tools: rich.tools,
|
||||
bashCommands: [],
|
||||
timestamp: rich.timestamp,
|
||||
speed: rich.speed,
|
||||
deduplicationKey: rich.deduplicationKey,
|
||||
userMessage: '',
|
||||
sessionId: rich.sessionId,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -222,7 +173,7 @@ async function discoverFromDb(dbPath: string, project: string): Promise<SessionS
|
|||
}
|
||||
|
||||
export function createCrushProvider(): Provider {
|
||||
return {
|
||||
return createBridgedProvider<CrushDecodedCall>({
|
||||
name: 'crush',
|
||||
displayName: 'Crush',
|
||||
|
||||
|
|
@ -247,10 +198,53 @@ export function createCrushProvider(): Provider {
|
|||
return sources
|
||||
},
|
||||
|
||||
createSessionParser(source: SessionSource, seenKeys: Set<string>): SessionParser {
|
||||
return createParser(source, seenKeys)
|
||||
// I/O adapter: open the db, run the session-row query and the dominant-model
|
||||
// query (both sqlite-side), and hand the core decoder one combined record.
|
||||
async readRecords(source: SessionSource): Promise<unknown[] | null> {
|
||||
if (!isSqliteAvailable()) {
|
||||
process.stderr.write(getSqliteLoadError() + '\n')
|
||||
return null
|
||||
}
|
||||
|
||||
// Source paths are encoded as `<dbPath>:<sessionId>`. Split from the
|
||||
// right because dbPath may contain a colon on Windows (drive letter).
|
||||
const segments = source.path.split(':')
|
||||
const sessionId = segments[segments.length - 1]!
|
||||
const dbPath = segments.slice(0, -1).join(':')
|
||||
|
||||
let db: SqliteDatabase
|
||||
try {
|
||||
db = openDatabase(dbPath)
|
||||
} catch (err) {
|
||||
process.stderr.write(
|
||||
`codeburn: cannot open Crush database: ${err instanceof Error ? err.message : err}\n`,
|
||||
)
|
||||
return null
|
||||
}
|
||||
|
||||
try {
|
||||
if (!validateSchema(db)) return null
|
||||
|
||||
const rows = db.query<SessionRow>(
|
||||
`SELECT id, prompt_tokens, completion_tokens, cost, created_at, updated_at, message_count
|
||||
FROM sessions
|
||||
WHERE id = ? AND parent_session_id IS NULL`,
|
||||
[sessionId],
|
||||
)
|
||||
if (rows.length === 0) return null
|
||||
const session = rows[0]!
|
||||
|
||||
const model = dominantModel(db, sessionId)
|
||||
const record: CrushRawRecord = { ...session, model }
|
||||
return [record]
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
decode: decodeCrush,
|
||||
toProviderCall,
|
||||
})
|
||||
}
|
||||
|
||||
export const crush = createCrushProvider()
|
||||
|
|
|
|||
|
|
@ -2,18 +2,13 @@ import { existsSync } from 'fs'
|
|||
import { homedir } from 'os'
|
||||
import { join } from 'path'
|
||||
|
||||
import { decodeForge } from '@codeburn/core/providers/forge'
|
||||
import type { ForgeConversationRow, ForgeDecodedCall } from '@codeburn/core/providers/forge'
|
||||
|
||||
import { extractBashCommands } from '../bash-utils.js'
|
||||
import { getSqliteLoadError, isSqliteAvailable, openDatabase, type SqliteDatabase } from '../sqlite.js'
|
||||
import type { ParsedProviderCall, Provider, SessionParser, SessionSource } from './types.js'
|
||||
|
||||
type ConversationRow = {
|
||||
conversation_id: string
|
||||
title: string | null
|
||||
workspace_id: number | string
|
||||
context: string | null
|
||||
created_at: string | null
|
||||
updated_at: string | null
|
||||
}
|
||||
import { createBridgedProvider } from './bridge.js'
|
||||
import type { ParsedProviderCall, Provider, SessionSource } from './types.js'
|
||||
|
||||
type DiscoveryRow = {
|
||||
conversation_id: string
|
||||
|
|
@ -21,18 +16,6 @@ type DiscoveryRow = {
|
|||
workspace_id: string
|
||||
}
|
||||
|
||||
type ContextMessage = {
|
||||
message?: {
|
||||
text?: {
|
||||
role?: unknown
|
||||
content?: unknown
|
||||
model?: unknown
|
||||
tool_calls?: unknown
|
||||
}
|
||||
}
|
||||
usage?: unknown
|
||||
}
|
||||
|
||||
const DEFAULT_DB_PATH = join(homedir(), '.forge', '.forge.db')
|
||||
|
||||
function validateSchema(db: SqliteDatabase): boolean {
|
||||
|
|
@ -44,186 +27,36 @@ function validateSchema(db: SqliteDatabase): boolean {
|
|||
}
|
||||
}
|
||||
|
||||
function sqliteTimestampToIso(value: string | null | undefined): string {
|
||||
if (!value) return new Date(0).toISOString()
|
||||
|
||||
const match = value.match(/^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})(?:\.(\d+))?$/)
|
||||
if (match) {
|
||||
const ms = (match[3] ?? '').padEnd(3, '0').slice(0, 3)
|
||||
const parsed = new Date(`${match[1]}T${match[2]}.${ms}Z`)
|
||||
if (!Number.isNaN(parsed.getTime())) return parsed.toISOString()
|
||||
}
|
||||
|
||||
const parsed = new Date(value)
|
||||
return Number.isNaN(parsed.getTime()) ? new Date(0).toISOString() : parsed.toISOString()
|
||||
}
|
||||
|
||||
function actual(value: unknown): number {
|
||||
if (!value || typeof value !== 'object') return 0
|
||||
const raw = (value as Record<string, unknown>)['actual']
|
||||
return typeof raw === 'number' && Number.isFinite(raw) ? raw : 0
|
||||
}
|
||||
|
||||
function usageActual(usage: unknown, key: string): number {
|
||||
if (!usage || typeof usage !== 'object') return 0
|
||||
return actual((usage as Record<string, unknown>)[key])
|
||||
}
|
||||
|
||||
function mapToolName(name: string): string {
|
||||
switch (name) {
|
||||
case 'shell':
|
||||
case 'bash':
|
||||
return 'Bash'
|
||||
case 'read':
|
||||
case 'Read':
|
||||
return 'Read'
|
||||
case 'write':
|
||||
case 'Write':
|
||||
return 'Write'
|
||||
case 'patch':
|
||||
case 'Edit':
|
||||
case 'edit':
|
||||
return 'Edit'
|
||||
case 'fs_search':
|
||||
case 'grep':
|
||||
return 'Grep'
|
||||
case 'task':
|
||||
case 'dispatch_agent':
|
||||
return 'Agent'
|
||||
default:
|
||||
return name
|
||||
}
|
||||
}
|
||||
|
||||
function pushUnique(values: string[], value: string): void {
|
||||
if (!values.includes(value)) values.push(value)
|
||||
}
|
||||
|
||||
function toolCalls(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.filter(v => v && typeof v === 'object') as Record<string, unknown>[] : []
|
||||
}
|
||||
|
||||
function extractToolsAndCommands(calls: Record<string, unknown>[]): { tools: string[]; bashCommands: string[]; firstCallId?: string } {
|
||||
const tools: string[] = []
|
||||
const bashCommands: string[] = []
|
||||
let firstCallId: string | undefined
|
||||
|
||||
for (const call of calls) {
|
||||
const rawName = call['name']
|
||||
if (typeof rawName !== 'string') continue
|
||||
if (!firstCallId && typeof call['call_id'] === 'string') firstCallId = call['call_id']
|
||||
|
||||
const tool = mapToolName(rawName)
|
||||
pushUnique(tools, tool)
|
||||
|
||||
if (tool === 'Bash') {
|
||||
const args = call['arguments']
|
||||
if (args && typeof args === 'object') {
|
||||
const command = (args as Record<string, unknown>)['command']
|
||||
if (typeof command === 'string') {
|
||||
for (const cmd of extractBashCommands(command)) pushUnique(bashCommands, cmd)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { tools, bashCommands, firstCallId }
|
||||
}
|
||||
|
||||
function splitSourcePath(path: string): { dbPath: string; conversationId: string } | null {
|
||||
const idx = path.lastIndexOf(':')
|
||||
if (idx < 0) return null
|
||||
return { dbPath: path.slice(0, idx), conversationId: path.slice(idx + 1) }
|
||||
}
|
||||
|
||||
function createParser(source: SessionSource, seenKeys: Set<string>): SessionParser {
|
||||
// Map one rich, cost-free decoder call into the host's ParsedProviderCall. Cost
|
||||
// re-enters here: `costBasis: 'estimated'` marks the call so the parser.ts
|
||||
// pricing pass fills `costUSD` from the token buckets. Bash base-name
|
||||
// extraction (and its `strip-ansi` dependency) stays CLI-side: the core
|
||||
// decoder carries the raw command strings; the host reduces them here.
|
||||
function toProviderCall(rich: ForgeDecodedCall): ParsedProviderCall {
|
||||
return {
|
||||
async *parse(): AsyncGenerator<ParsedProviderCall> {
|
||||
if (!isSqliteAvailable()) {
|
||||
process.stderr.write(getSqliteLoadError() + '\n')
|
||||
return
|
||||
}
|
||||
|
||||
const split = splitSourcePath(source.path)
|
||||
if (!split) return
|
||||
|
||||
let db: SqliteDatabase
|
||||
try {
|
||||
db = openDatabase(split.dbPath)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
if (!validateSchema(db)) return
|
||||
const rows = db.query<ConversationRow>(
|
||||
`SELECT conversation_id, title, CAST(workspace_id AS TEXT) AS workspace_id, context, created_at, updated_at
|
||||
FROM conversations
|
||||
WHERE conversation_id = ?`,
|
||||
[split.conversationId],
|
||||
)
|
||||
const row = rows[0]
|
||||
if (!row?.context) return
|
||||
|
||||
let parsed: unknown
|
||||
try {
|
||||
parsed = JSON.parse(row.context)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
const messages = Array.isArray((parsed as { messages?: unknown }).messages)
|
||||
? (parsed as { messages: ContextMessage[] }).messages
|
||||
: []
|
||||
|
||||
let userMessage = ''
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
const text = messages[i]?.message?.text
|
||||
const role = typeof text?.role === 'string' ? text.role.toLowerCase() : ''
|
||||
const content = typeof text?.content === 'string' ? text.content : ''
|
||||
|
||||
if (role === 'user') {
|
||||
userMessage = content.length > 500 ? content.slice(0, 500) : content
|
||||
continue
|
||||
}
|
||||
if (role !== 'assistant') continue
|
||||
|
||||
const promptTokens = usageActual(messages[i]?.usage, 'prompt_tokens')
|
||||
const outputTokens = usageActual(messages[i]?.usage, 'completion_tokens')
|
||||
const cachedInputTokens = usageActual(messages[i]?.usage, 'cached_tokens')
|
||||
const inputTokens = Math.max(0, promptTokens - cachedInputTokens)
|
||||
if (inputTokens === 0 && outputTokens === 0) continue
|
||||
|
||||
const model = typeof text?.model === 'string' ? text.model : 'unknown'
|
||||
const calls = toolCalls(text?.tool_calls)
|
||||
const { tools, bashCommands, firstCallId } = extractToolsAndCommands(calls)
|
||||
const stableId = firstCallId ?? `${model}:${promptTokens}:${outputTokens}:${i}`
|
||||
const deduplicationKey = `forge:${row.conversation_id}:${stableId}`
|
||||
if (seenKeys.has(deduplicationKey)) continue
|
||||
seenKeys.add(deduplicationKey)
|
||||
yield {
|
||||
provider: 'forge',
|
||||
model,
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: cachedInputTokens,
|
||||
cachedInputTokens,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
costBasis: 'estimated',
|
||||
tools,
|
||||
bashCommands,
|
||||
timestamp: sqliteTimestampToIso(row.updated_at ?? row.created_at),
|
||||
speed: 'standard',
|
||||
deduplicationKey,
|
||||
userMessage,
|
||||
sessionId: row.conversation_id,
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
},
|
||||
provider: 'forge',
|
||||
model: rich.model,
|
||||
inputTokens: rich.inputTokens,
|
||||
outputTokens: rich.outputTokens,
|
||||
cacheCreationInputTokens: rich.cacheCreationInputTokens,
|
||||
cacheReadInputTokens: rich.cacheReadInputTokens,
|
||||
cachedInputTokens: rich.cachedInputTokens,
|
||||
reasoningTokens: rich.reasoningTokens,
|
||||
webSearchRequests: rich.webSearchRequests,
|
||||
costBasis: 'estimated',
|
||||
tools: rich.tools,
|
||||
bashCommands: [...new Set(rich.rawBashCommands.flatMap(c => extractBashCommands(c)))],
|
||||
timestamp: rich.timestamp,
|
||||
speed: rich.speed,
|
||||
deduplicationKey: rich.deduplicationKey,
|
||||
userMessage: rich.userMessage,
|
||||
sessionId: rich.sessionId,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -257,7 +90,7 @@ async function discoverFromDb(dbPath: string): Promise<SessionSource[]> {
|
|||
}
|
||||
|
||||
export function createForgeProvider(dbPath = DEFAULT_DB_PATH): Provider {
|
||||
return {
|
||||
return createBridgedProvider<ForgeDecodedCall>({
|
||||
name: 'forge',
|
||||
displayName: 'Forge',
|
||||
|
||||
|
|
@ -274,10 +107,43 @@ export function createForgeProvider(dbPath = DEFAULT_DB_PATH): Provider {
|
|||
return discoverFromDb(dbPath)
|
||||
},
|
||||
|
||||
createSessionParser(source: SessionSource, seenKeys: Set<string>): SessionParser {
|
||||
return createParser(source, seenKeys)
|
||||
// I/O adapter: open the db and run the conversation-row query (sqlite-side).
|
||||
// The `context` JSON blob is handed to the core decoder still serialized;
|
||||
// parsing it is decode logic.
|
||||
async readRecords(source: SessionSource): Promise<unknown[] | null> {
|
||||
if (!isSqliteAvailable()) {
|
||||
process.stderr.write(getSqliteLoadError() + '\n')
|
||||
return null
|
||||
}
|
||||
|
||||
const split = splitSourcePath(source.path)
|
||||
if (!split) return null
|
||||
|
||||
let db: SqliteDatabase
|
||||
try {
|
||||
db = openDatabase(split.dbPath)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
|
||||
try {
|
||||
if (!validateSchema(db)) return null
|
||||
const rows = db.query<ForgeConversationRow>(
|
||||
`SELECT conversation_id, title, CAST(workspace_id AS TEXT) AS workspace_id, context, created_at, updated_at
|
||||
FROM conversations
|
||||
WHERE conversation_id = ?`,
|
||||
[split.conversationId],
|
||||
)
|
||||
if (rows.length === 0) return null
|
||||
return [rows[0]!]
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
decode: decodeForge,
|
||||
toProviderCall,
|
||||
})
|
||||
}
|
||||
|
||||
export const forge = createForgeProvider()
|
||||
|
|
|
|||
|
|
@ -1,13 +1,16 @@
|
|||
import { join } from 'path'
|
||||
import { homedir, platform } from 'os'
|
||||
|
||||
import { decodeGoose, gooseToolNameMap } from '@codeburn/core/providers/goose'
|
||||
import type { GooseDecodedCall, GooseMessageRow, GooseSessionRecords, GooseSessionRow } from '@codeburn/core/providers/goose'
|
||||
|
||||
import { getShortModelName } from '../models.js'
|
||||
import { extractBashCommands } from '../bash-utils.js'
|
||||
import { isSqliteAvailable, getSqliteLoadError, openDatabase, blobToText, type SqliteDatabase } from '../sqlite.js'
|
||||
import type { ToolCall } from '../types.js'
|
||||
import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js'
|
||||
import { createBridgedProvider } from './bridge.js'
|
||||
import type { Provider, SessionSource, ParsedProviderCall } from './types.js'
|
||||
|
||||
type SessionRow = {
|
||||
type RawSessionRow = {
|
||||
id: string
|
||||
name: string
|
||||
working_dir: string | null
|
||||
|
|
@ -19,33 +22,6 @@ type SessionRow = {
|
|||
model_config_json: Uint8Array | string | null
|
||||
}
|
||||
|
||||
type ModelConfig = {
|
||||
model_name?: string
|
||||
reasoning?: boolean
|
||||
}
|
||||
|
||||
type MessageRow = {
|
||||
message_id: string
|
||||
role: string
|
||||
content_json: Uint8Array | string
|
||||
created_timestamp: number
|
||||
}
|
||||
|
||||
type ContentItem = {
|
||||
type: string
|
||||
toolCall?: { value?: { name?: string; arguments?: Record<string, unknown> } }
|
||||
}
|
||||
|
||||
const toolNameMap: Record<string, string> = {
|
||||
developer__shell: 'Bash',
|
||||
developer__text_editor: 'Edit',
|
||||
developer__read_file: 'Read',
|
||||
developer__write_file: 'Write',
|
||||
developer__list_directory: 'LS',
|
||||
developer__search_files: 'Grep',
|
||||
computercontroller__shell: 'Bash',
|
||||
}
|
||||
|
||||
function sanitize(dir: string): string {
|
||||
return dir.replace(/^\//, '').replace(/\//g, '-')
|
||||
}
|
||||
|
|
@ -72,157 +48,37 @@ function validateSchema(db: SqliteDatabase): boolean {
|
|||
}
|
||||
}
|
||||
|
||||
function parseModelConfig(raw: string | null): ModelConfig {
|
||||
if (!raw) return {}
|
||||
try {
|
||||
return JSON.parse(raw) as ModelConfig
|
||||
} catch {
|
||||
return {}
|
||||
}
|
||||
}
|
||||
const SESSION_COLUMNS = 'id, name, working_dir, created_at, updated_at, accumulated_input_tokens, accumulated_output_tokens, provider_name, CAST(model_config_json AS BLOB) AS model_config_json'
|
||||
|
||||
function extractToolsFromMessages(db: SqliteDatabase, sessionId: string): { tools: string[]; bashCommands: string[]; toolSequence: ToolCall[][] } {
|
||||
const tools: string[] = []
|
||||
const bashCommands: string[] = []
|
||||
const seen = new Set<string>()
|
||||
const toolSequence: ToolCall[][] = []
|
||||
|
||||
try {
|
||||
const rows = db.query<{ content_json: Uint8Array | string }>(
|
||||
"SELECT CAST(content_json AS BLOB) AS content_json FROM messages WHERE session_id = ? AND role = 'assistant' AND content_json LIKE '%toolRequest%' ORDER BY created_timestamp ASC",
|
||||
[sessionId],
|
||||
)
|
||||
|
||||
for (const row of rows) {
|
||||
let items: ContentItem[]
|
||||
try {
|
||||
items = JSON.parse(blobToText(row.content_json)) as ContentItem[]
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
const msgCalls: ToolCall[] = []
|
||||
for (const item of items) {
|
||||
if (item.type !== 'toolRequest') continue
|
||||
const rawName = item.toolCall?.value?.name ?? ''
|
||||
if (!rawName) continue
|
||||
const mapped = toolNameMap[rawName] ?? rawName.split('__').pop() ?? rawName
|
||||
if (!seen.has(mapped)) {
|
||||
seen.add(mapped)
|
||||
tools.push(mapped)
|
||||
}
|
||||
const call: ToolCall = { tool: mapped }
|
||||
const args = item.toolCall?.value?.arguments
|
||||
if (args && typeof args === 'object') {
|
||||
const fp = (args as Record<string, unknown>)['file_path']
|
||||
if (typeof fp === 'string') call.file = fp
|
||||
const cmd = (args as Record<string, unknown>)['command']
|
||||
if (typeof cmd === 'string') call.command = cmd
|
||||
}
|
||||
msgCalls.push(call)
|
||||
if (mapped === 'Bash') {
|
||||
const cmd = item.toolCall?.value?.arguments?.command
|
||||
if (typeof cmd === 'string') {
|
||||
for (const c of extractBashCommands(cmd)) {
|
||||
if (!bashCommands.includes(c)) bashCommands.push(c)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (msgCalls.length > 0) toolSequence.push(msgCalls)
|
||||
}
|
||||
} catch { /* best-effort */ }
|
||||
|
||||
return { tools, bashCommands, toolSequence }
|
||||
}
|
||||
|
||||
function getFirstUserMessage(db: SqliteDatabase, sessionId: string): string {
|
||||
try {
|
||||
const rows = db.query<{ content_json: Uint8Array | string }>(
|
||||
"SELECT CAST(content_json AS BLOB) AS content_json FROM messages WHERE session_id = ? AND role = 'user' ORDER BY created_timestamp ASC LIMIT 1",
|
||||
[sessionId],
|
||||
)
|
||||
if (rows.length === 0) return ''
|
||||
const items = JSON.parse(blobToText(rows[0]!.content_json)) as ContentItem[]
|
||||
const text = items.find(i => i.type === 'text') as { text?: string } | undefined
|
||||
return (text?.text ?? '').slice(0, 500)
|
||||
} catch {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
function createParser(source: SessionSource, seenKeys: Set<string>): SessionParser {
|
||||
// Map one rich, cost-free decoder call into the host's ParsedProviderCall. Cost
|
||||
// re-enters here: `costBasis: 'estimated'` marks the call so the parser.ts
|
||||
// pricing pass fills `costUSD` from the token buckets. Bash base-name
|
||||
// extraction (and its `strip-ansi` dependency) stays CLI-side: the core
|
||||
// decoder carries the raw command strings; the host reduces them here.
|
||||
function toProviderCall(rich: GooseDecodedCall): ParsedProviderCall {
|
||||
return {
|
||||
async *parse(): AsyncGenerator<ParsedProviderCall> {
|
||||
if (!isSqliteAvailable()) {
|
||||
process.stderr.write(getSqliteLoadError() + '\n')
|
||||
return
|
||||
}
|
||||
|
||||
const segments = source.path.split(':')
|
||||
const sessionId = segments[segments.length - 1]!
|
||||
const dbPath = segments.slice(0, -1).join(':')
|
||||
|
||||
let db: SqliteDatabase
|
||||
try {
|
||||
db = openDatabase(dbPath)
|
||||
} catch (err) {
|
||||
process.stderr.write(`codeburn: cannot open Goose database: ${err instanceof Error ? err.message : err}\n`)
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
if (!validateSchema(db)) return
|
||||
|
||||
const rows = db.query<SessionRow>(
|
||||
'SELECT id, name, working_dir, created_at, updated_at, accumulated_input_tokens, accumulated_output_tokens, provider_name, CAST(model_config_json AS BLOB) AS model_config_json FROM sessions WHERE id = ?',
|
||||
[sessionId],
|
||||
)
|
||||
if (rows.length === 0) return
|
||||
|
||||
const session = rows[0]!
|
||||
const inputTokens = session.accumulated_input_tokens ?? 0
|
||||
const outputTokens = session.accumulated_output_tokens ?? 0
|
||||
if (inputTokens === 0 && outputTokens === 0) return
|
||||
|
||||
const dedupKey = `goose:${sessionId}`
|
||||
if (seenKeys.has(dedupKey)) return
|
||||
seenKeys.add(dedupKey)
|
||||
|
||||
const config = parseModelConfig(blobToText(session.model_config_json))
|
||||
const model = config.model_name ?? 'unknown'
|
||||
|
||||
const { tools, bashCommands, toolSequence } = extractToolsFromMessages(db, sessionId)
|
||||
const userMessage = getFirstUserMessage(db, sessionId)
|
||||
|
||||
const raw = session.updated_at || session.created_at || ''
|
||||
let ts = new Date(raw)
|
||||
if (isNaN(ts.getTime())) ts = new Date(raw + 'Z')
|
||||
if (isNaN(ts.getTime())) ts = new Date()
|
||||
|
||||
yield {
|
||||
provider: 'goose',
|
||||
model,
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
costBasis: 'estimated',
|
||||
tools,
|
||||
bashCommands,
|
||||
toolSequence: toolSequence.length > 1 ? toolSequence : undefined,
|
||||
timestamp: ts.toISOString(),
|
||||
speed: 'standard',
|
||||
deduplicationKey: dedupKey,
|
||||
userMessage,
|
||||
sessionId,
|
||||
}
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
},
|
||||
provider: 'goose',
|
||||
model: rich.model,
|
||||
inputTokens: rich.inputTokens,
|
||||
outputTokens: rich.outputTokens,
|
||||
cacheCreationInputTokens: rich.cacheCreationInputTokens,
|
||||
cacheReadInputTokens: rich.cacheReadInputTokens,
|
||||
cachedInputTokens: rich.cachedInputTokens,
|
||||
reasoningTokens: rich.reasoningTokens,
|
||||
webSearchRequests: rich.webSearchRequests,
|
||||
costBasis: 'estimated',
|
||||
tools: rich.tools,
|
||||
bashCommands: [...new Set(rich.rawBashCommands.flatMap(c => extractBashCommands(c)))],
|
||||
toolSequence: rich.toolSequence,
|
||||
// The pre-migration decode fell back to `new Date()` (the current time)
|
||||
// when both updated_at/created_at were unparseable. That clock read can't
|
||||
// live in the pure core decoder, so decodeGoose emits '' in that case and
|
||||
// the fallback is applied here instead.
|
||||
timestamp: rich.timestamp || new Date().toISOString(),
|
||||
speed: rich.speed,
|
||||
deduplicationKey: rich.deduplicationKey,
|
||||
userMessage: rich.userMessage,
|
||||
sessionId: rich.sessionId,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -235,8 +91,8 @@ async function discoverFromDb(dbPath: string): Promise<SessionSource[]> {
|
|||
}
|
||||
|
||||
try {
|
||||
const rows = db.query<SessionRow>(
|
||||
'SELECT id, name, working_dir, created_at, updated_at, accumulated_input_tokens, accumulated_output_tokens, provider_name, CAST(model_config_json AS BLOB) AS model_config_json FROM sessions ORDER BY updated_at DESC',
|
||||
const rows = db.query<RawSessionRow>(
|
||||
`SELECT ${SESSION_COLUMNS} FROM sessions ORDER BY updated_at DESC`,
|
||||
)
|
||||
|
||||
return rows
|
||||
|
|
@ -262,7 +118,7 @@ const modelDisplayNames: Record<string, string> = {
|
|||
}
|
||||
|
||||
export function createGooseProvider(): Provider {
|
||||
return {
|
||||
return createBridgedProvider<GooseDecodedCall>({
|
||||
name: 'goose',
|
||||
displayName: 'Goose',
|
||||
|
||||
|
|
@ -271,7 +127,7 @@ export function createGooseProvider(): Provider {
|
|||
},
|
||||
|
||||
toolDisplayName(rawTool: string): string {
|
||||
return toolNameMap[rawTool] ?? rawTool
|
||||
return gooseToolNameMap[rawTool] ?? rawTool
|
||||
},
|
||||
|
||||
async discoverSessions(): Promise<SessionSource[]> {
|
||||
|
|
@ -280,10 +136,81 @@ export function createGooseProvider(): Provider {
|
|||
return discoverFromDb(dbPath)
|
||||
},
|
||||
|
||||
createSessionParser(source: SessionSource, seenKeys: Set<string>): SessionParser {
|
||||
return createParser(source, seenKeys)
|
||||
// I/O adapter: open the db and run the session query, the assistant
|
||||
// tool-message query, and the first-user-message query (all sqlite-side),
|
||||
// converting each BLOB column to text (the same charset-safe conversion
|
||||
// fs-utils performs for file reads). The core decoder gets one composite
|
||||
// record bundling all three; JSON parsing and per-message decode are pure
|
||||
// and happen there.
|
||||
async readRecords(source: SessionSource): Promise<unknown[] | null> {
|
||||
if (!isSqliteAvailable()) {
|
||||
process.stderr.write(getSqliteLoadError() + '\n')
|
||||
return null
|
||||
}
|
||||
|
||||
const segments = source.path.split(':')
|
||||
const sessionId = segments[segments.length - 1]!
|
||||
const dbPath = segments.slice(0, -1).join(':')
|
||||
|
||||
let db: SqliteDatabase
|
||||
try {
|
||||
db = openDatabase(dbPath)
|
||||
} catch (err) {
|
||||
process.stderr.write(`codeburn: cannot open Goose database: ${err instanceof Error ? err.message : err}\n`)
|
||||
return null
|
||||
}
|
||||
|
||||
try {
|
||||
if (!validateSchema(db)) return null
|
||||
|
||||
const rows = db.query<RawSessionRow>(
|
||||
`SELECT ${SESSION_COLUMNS} FROM sessions WHERE id = ?`,
|
||||
[sessionId],
|
||||
)
|
||||
if (rows.length === 0) return null
|
||||
const raw = rows[0]!
|
||||
|
||||
const session: GooseSessionRow = {
|
||||
id: raw.id,
|
||||
workingDir: raw.working_dir,
|
||||
createdAt: raw.created_at,
|
||||
updatedAt: raw.updated_at,
|
||||
accumulatedInputTokens: raw.accumulated_input_tokens,
|
||||
accumulatedOutputTokens: raw.accumulated_output_tokens,
|
||||
modelConfigJson: blobToText(raw.model_config_json) || null,
|
||||
}
|
||||
|
||||
let assistantToolMessages: GooseMessageRow[] = []
|
||||
let firstUserMessage: GooseMessageRow | null = null
|
||||
try {
|
||||
const toolRows = db.query<{ content_json: Uint8Array | string }>(
|
||||
"SELECT CAST(content_json AS BLOB) AS content_json FROM messages WHERE session_id = ? AND role = 'assistant' AND content_json LIKE '%toolRequest%' ORDER BY created_timestamp ASC",
|
||||
[sessionId],
|
||||
)
|
||||
assistantToolMessages = toolRows.map(r => ({ contentJson: blobToText(r.content_json) }))
|
||||
} catch {
|
||||
// best-effort, matches the pre-migration decode
|
||||
}
|
||||
try {
|
||||
const userRows = db.query<{ content_json: Uint8Array | string }>(
|
||||
"SELECT CAST(content_json AS BLOB) AS content_json FROM messages WHERE session_id = ? AND role = 'user' ORDER BY created_timestamp ASC LIMIT 1",
|
||||
[sessionId],
|
||||
)
|
||||
if (userRows.length > 0) firstUserMessage = { contentJson: blobToText(userRows[0]!.content_json) }
|
||||
} catch {
|
||||
// best-effort, matches the pre-migration decode
|
||||
}
|
||||
|
||||
const record: GooseSessionRecords = { sessionId, session, assistantToolMessages, firstUserMessage }
|
||||
return [record]
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
decode: decodeGoose,
|
||||
toProviderCall,
|
||||
})
|
||||
}
|
||||
|
||||
export const goose = createGooseProvider()
|
||||
|
|
|
|||
|
|
@ -1,8 +1,12 @@
|
|||
import { join } from 'path'
|
||||
import { homedir } from 'os'
|
||||
|
||||
import { decodeZcode } from '@codeburn/core/providers/zcode'
|
||||
import type { ZcodeDecodedCall, ZcodeSessionRecords, ZcodeToolRow, ZcodeUsageRow } from '@codeburn/core/providers/zcode'
|
||||
|
||||
import { isSqliteAvailable, getSqliteLoadError, openDatabase, type SqliteDatabase } from '../sqlite.js'
|
||||
import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js'
|
||||
import { createBridgedProvider } from './bridge.js'
|
||||
import type { Provider, SessionSource, ParsedProviderCall } from './types.js'
|
||||
|
||||
/// ZCode (CLI v0.14.x) records usage in a single SQLite database at
|
||||
/// ~/.zcode/cli/db/db.sqlite. We read it because the other on-disk sources are
|
||||
|
|
@ -16,24 +20,6 @@ type SessionRow = {
|
|||
directory: string
|
||||
}
|
||||
|
||||
type UsageRow = {
|
||||
id: string
|
||||
turn_id: string | null
|
||||
model_id: string
|
||||
input_tokens: number
|
||||
output_tokens: number
|
||||
reasoning_tokens: number
|
||||
cache_creation_input_tokens: number
|
||||
cache_read_input_tokens: number
|
||||
started_at: number
|
||||
completed_at: number | null
|
||||
}
|
||||
|
||||
type ToolRow = {
|
||||
turn_id: string | null
|
||||
tool_name: string
|
||||
}
|
||||
|
||||
function getDbPath(override?: string): string {
|
||||
return override ?? join(homedir(), '.zcode', 'cli', 'db', 'db.sqlite')
|
||||
}
|
||||
|
|
@ -42,11 +28,6 @@ function sanitizeProject(path: string): string {
|
|||
return path.replace(/^\//, '').replace(/\//g, '-')
|
||||
}
|
||||
|
||||
function epochMsToIso(ms: number | null): string {
|
||||
if (ms === null || !Number.isFinite(ms) || ms <= 0) return new Date(0).toISOString()
|
||||
return new Date(ms).toISOString()
|
||||
}
|
||||
|
||||
function validateSchema(db: SqliteDatabase): boolean {
|
||||
try {
|
||||
db.query<{ cnt: number }>('SELECT COUNT(*) as cnt FROM model_usage LIMIT 1')
|
||||
|
|
@ -85,121 +66,37 @@ function discover(dbPath: string): SessionSource[] {
|
|||
}
|
||||
}
|
||||
|
||||
function createParser(source: SessionSource, seenKeys: Set<string>): SessionParser {
|
||||
// Map one rich, cost-free decoder call into the host's ParsedProviderCall. Cost
|
||||
// re-enters here: `costBasis: 'estimated'` marks the call so the parser.ts
|
||||
// pricing pass fills `costUSD` from the token buckets. ZCode never captures a
|
||||
// user message, so it is hardcoded empty here rather than carried through the
|
||||
// rich decode.
|
||||
function toProviderCall(rich: ZcodeDecodedCall): ParsedProviderCall {
|
||||
return {
|
||||
async *parse(): AsyncGenerator<ParsedProviderCall> {
|
||||
if (!isSqliteAvailable()) {
|
||||
process.stderr.write(getSqliteLoadError() + '\n')
|
||||
return
|
||||
}
|
||||
|
||||
// Source paths are `<dbPath>:<sessionId>`. Split from the right so a colon
|
||||
// in the path (Windows drive letter) doesn't corrupt the session id.
|
||||
const segments = source.path.split(':')
|
||||
const sessionId = segments[segments.length - 1]!
|
||||
const dbPath = segments.slice(0, -1).join(':')
|
||||
|
||||
let db: SqliteDatabase
|
||||
try {
|
||||
db = openDatabase(dbPath)
|
||||
} catch (err) {
|
||||
process.stderr.write(
|
||||
`codeburn: cannot open ZCode database: ${err instanceof Error ? err.message : err}\n`,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
if (!validateSchema(db)) return
|
||||
|
||||
// model_usage rows don't link to individual tool calls, only to a turn,
|
||||
// so collect each turn's tools and attach them to one request per turn
|
||||
// (below) to avoid double-counting across a turn's multiple requests.
|
||||
const toolRows = db.query<ToolRow>(
|
||||
`SELECT turn_id, tool_name FROM tool_usage
|
||||
WHERE session_id = ? AND turn_id IS NOT NULL
|
||||
ORDER BY started_at ASC`,
|
||||
[sessionId],
|
||||
)
|
||||
const toolsByTurn = new Map<string, string[]>()
|
||||
for (const tool of toolRows) {
|
||||
if (!tool.turn_id) continue
|
||||
const list = toolsByTurn.get(tool.turn_id) ?? []
|
||||
list.push(tool.tool_name)
|
||||
toolsByTurn.set(tool.turn_id, list)
|
||||
}
|
||||
|
||||
const rows = db.query<UsageRow>(
|
||||
`SELECT id, turn_id, model_id, input_tokens, output_tokens, reasoning_tokens,
|
||||
cache_creation_input_tokens, cache_read_input_tokens, started_at, completed_at
|
||||
FROM model_usage WHERE session_id = ?
|
||||
ORDER BY started_at ASC`,
|
||||
[sessionId],
|
||||
)
|
||||
|
||||
const turnsWithToolsEmitted = new Set<string>()
|
||||
|
||||
for (const row of rows) {
|
||||
const cacheRead = row.cache_read_input_tokens ?? 0
|
||||
const cacheCreation = row.cache_creation_input_tokens ?? 0
|
||||
const output = row.output_tokens ?? 0
|
||||
const reasoning = row.reasoning_tokens ?? 0
|
||||
// ZCode folds cached tokens into input_tokens (OpenAI-style). Split
|
||||
// them back out so fresh input bills at the input rate and cached at
|
||||
// the cache-read rate, matching the pricing table's Anthropic-style
|
||||
// semantics.
|
||||
const freshInput = Math.max(0, (row.input_tokens ?? 0) - cacheRead - cacheCreation)
|
||||
|
||||
if (freshInput === 0 && output === 0 && reasoning === 0 && cacheRead === 0 && cacheCreation === 0) {
|
||||
continue
|
||||
}
|
||||
|
||||
const dedupKey = `zcode:${row.id}`
|
||||
if (seenKeys.has(dedupKey)) continue
|
||||
seenKeys.add(dedupKey)
|
||||
|
||||
let tools: string[] = []
|
||||
if (row.turn_id && !turnsWithToolsEmitted.has(row.turn_id)) {
|
||||
const turnTools = toolsByTurn.get(row.turn_id)
|
||||
if (turnTools && turnTools.length > 0) {
|
||||
tools = turnTools
|
||||
turnsWithToolsEmitted.add(row.turn_id)
|
||||
}
|
||||
}
|
||||
|
||||
const model = row.model_id
|
||||
|
||||
yield {
|
||||
provider: 'zcode',
|
||||
model,
|
||||
inputTokens: freshInput,
|
||||
outputTokens: output,
|
||||
cacheCreationInputTokens: cacheCreation,
|
||||
cacheReadInputTokens: cacheRead,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: reasoning,
|
||||
webSearchRequests: 0,
|
||||
costBasis: 'estimated',
|
||||
tools,
|
||||
bashCommands: [],
|
||||
timestamp: epochMsToIso(row.completed_at ?? row.started_at),
|
||||
speed: 'standard',
|
||||
deduplicationKey: dedupKey,
|
||||
turnId: row.turn_id ?? undefined,
|
||||
userMessage: '',
|
||||
sessionId,
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
},
|
||||
provider: 'zcode',
|
||||
model: rich.model,
|
||||
inputTokens: rich.inputTokens,
|
||||
outputTokens: rich.outputTokens,
|
||||
cacheCreationInputTokens: rich.cacheCreationInputTokens,
|
||||
cacheReadInputTokens: rich.cacheReadInputTokens,
|
||||
cachedInputTokens: rich.cachedInputTokens,
|
||||
reasoningTokens: rich.reasoningTokens,
|
||||
webSearchRequests: rich.webSearchRequests,
|
||||
costBasis: 'estimated',
|
||||
tools: rich.tools,
|
||||
bashCommands: [],
|
||||
timestamp: rich.timestamp,
|
||||
speed: rich.speed,
|
||||
deduplicationKey: rich.deduplicationKey,
|
||||
turnId: rich.turnId,
|
||||
userMessage: '',
|
||||
sessionId: rich.sessionId,
|
||||
}
|
||||
}
|
||||
|
||||
export function createZcodeProvider(dbPathOverride?: string): Provider {
|
||||
const dbPath = getDbPath(dbPathOverride)
|
||||
return {
|
||||
return createBridgedProvider<ZcodeDecodedCall>({
|
||||
name: 'zcode',
|
||||
displayName: 'ZCode',
|
||||
|
||||
|
|
@ -216,10 +113,63 @@ export function createZcodeProvider(dbPathOverride?: string): Provider {
|
|||
return discover(dbPath)
|
||||
},
|
||||
|
||||
createSessionParser(source: SessionSource, seenKeys: Set<string>): SessionParser {
|
||||
return createParser(source, seenKeys)
|
||||
// I/O adapter: open the db, run the model_usage and tool_usage queries for
|
||||
// this session (both sqlite-side), and hand the core decoder one combined
|
||||
// record bundling both row sets.
|
||||
async readRecords(source: SessionSource): Promise<unknown[] | null> {
|
||||
if (!isSqliteAvailable()) {
|
||||
process.stderr.write(getSqliteLoadError() + '\n')
|
||||
return null
|
||||
}
|
||||
|
||||
// Source paths are `<dbPath>:<sessionId>`. Split from the right so a colon
|
||||
// in the path (Windows drive letter) doesn't corrupt the session id.
|
||||
const segments = source.path.split(':')
|
||||
const sessionId = segments[segments.length - 1]!
|
||||
const readDbPath = segments.slice(0, -1).join(':')
|
||||
|
||||
let db: SqliteDatabase
|
||||
try {
|
||||
db = openDatabase(readDbPath)
|
||||
} catch (err) {
|
||||
process.stderr.write(
|
||||
`codeburn: cannot open ZCode database: ${err instanceof Error ? err.message : err}\n`,
|
||||
)
|
||||
return null
|
||||
}
|
||||
|
||||
try {
|
||||
if (!validateSchema(db)) return null
|
||||
|
||||
// model_usage rows don't link to individual tool calls, only to a turn,
|
||||
// so collect each turn's tools and attach them to one request per turn
|
||||
// (in the decoder) to avoid double-counting across a turn's multiple
|
||||
// requests.
|
||||
const toolRows = db.query<ZcodeToolRow>(
|
||||
`SELECT turn_id, tool_name FROM tool_usage
|
||||
WHERE session_id = ? AND turn_id IS NOT NULL
|
||||
ORDER BY started_at ASC`,
|
||||
[sessionId],
|
||||
)
|
||||
|
||||
const usageRows = db.query<ZcodeUsageRow>(
|
||||
`SELECT id, turn_id, model_id, input_tokens, output_tokens, reasoning_tokens,
|
||||
cache_creation_input_tokens, cache_read_input_tokens, started_at, completed_at
|
||||
FROM model_usage WHERE session_id = ?
|
||||
ORDER BY started_at ASC`,
|
||||
[sessionId],
|
||||
)
|
||||
|
||||
const record: ZcodeSessionRecords = { sessionId, usageRows, toolRows }
|
||||
return [record]
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
decode: decodeZcode,
|
||||
toProviderCall,
|
||||
})
|
||||
}
|
||||
|
||||
export const zcode = createZcodeProvider()
|
||||
|
|
|
|||
|
|
@ -3,16 +3,24 @@ import { join } from 'path'
|
|||
import { homedir } from 'os'
|
||||
import zlib from 'zlib'
|
||||
|
||||
import { decodeZed } from '@codeburn/core/providers/zed'
|
||||
import type { ZedDecodedCall, ZedThreadRow } from '@codeburn/core/providers/zed'
|
||||
|
||||
import { getSqliteLoadError, isSqliteAvailable, openDatabase, type SqliteDatabase } from '../sqlite.js'
|
||||
import type { ParsedProviderCall, Provider, SessionParser, SessionSource } from './types.js'
|
||||
import { createBridgedProvider } from './bridge.js'
|
||||
import type { ParsedProviderCall, Provider, SessionSource } from './types.js'
|
||||
|
||||
// Zed's built-in agent stores one row per thread in a single SQLite database;
|
||||
// the `data` blob is zstd-compressed JSON carrying `request_token_usage`
|
||||
// (per-request Anthropic-shaped token counts) and the thread's model.
|
||||
// Format documented in issue #480.
|
||||
// (per-request Anthropic-shaped token counts) and the thread's model. The
|
||||
// sqlite driver and SQL query stay CLI-side; decompression, JSON parsing, and
|
||||
// per-request token accounting moved to @codeburn/core/providers/zed. Format
|
||||
// documented in issue #480.
|
||||
|
||||
// zstd landed in node:zlib in 22.15 / 23.8; the package floor is 22.13, so the
|
||||
// provider degrades with a notice instead of assuming the export exists.
|
||||
// provider degrades with a notice instead of assuming the export exists. This
|
||||
// Node-version capability check is host-side by nature and stays here — the
|
||||
// core decoder is only ever called once this has confirmed support exists.
|
||||
const zstdDecompress = (zlib as { zstdDecompressSync?: (buf: Buffer) => Buffer }).zstdDecompressSync
|
||||
|
||||
function getZedThreadsDbPath(): string {
|
||||
|
|
@ -31,176 +39,34 @@ const THREADS_QUERY = `
|
|||
ORDER BY updated_at ASC
|
||||
`
|
||||
|
||||
type ThreadRow = {
|
||||
id: string
|
||||
summary: string | null
|
||||
updated_at: string | null
|
||||
data_type: string | null
|
||||
data: Uint8Array | null
|
||||
}
|
||||
|
||||
type TokenUsage = {
|
||||
input_tokens?: number
|
||||
output_tokens?: number
|
||||
cache_creation_input_tokens?: number
|
||||
cache_read_input_tokens?: number
|
||||
}
|
||||
|
||||
type ThreadJson = {
|
||||
model?: { provider?: string; model?: string }
|
||||
request_token_usage?: Record<string, TokenUsage>
|
||||
cumulative_token_usage?: TokenUsage
|
||||
}
|
||||
|
||||
function num(value: number | undefined): number {
|
||||
return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : 0
|
||||
}
|
||||
|
||||
function usageIsEmpty(usage: TokenUsage): boolean {
|
||||
return (
|
||||
num(usage.input_tokens) === 0 &&
|
||||
num(usage.output_tokens) === 0 &&
|
||||
num(usage.cache_creation_input_tokens) === 0 &&
|
||||
num(usage.cache_read_input_tokens) === 0
|
||||
)
|
||||
}
|
||||
|
||||
function buildCall(opts: {
|
||||
threadId: string
|
||||
requestKey: string
|
||||
usage: TokenUsage
|
||||
model: string
|
||||
timestamp: string
|
||||
userMessage: string
|
||||
}): ParsedProviderCall {
|
||||
const input = num(opts.usage.input_tokens)
|
||||
const output = num(opts.usage.output_tokens)
|
||||
const cacheWrite = num(opts.usage.cache_creation_input_tokens)
|
||||
const cacheRead = num(opts.usage.cache_read_input_tokens)
|
||||
// Map one rich, cost-free decoder call into the host's ParsedProviderCall. Cost
|
||||
// re-enters here: `costBasis: 'estimated'` marks the call so the parser.ts
|
||||
// pricing pass fills `costUSD` from the token buckets. Zed never captures tool
|
||||
// calls, so `bashCommands` is always empty (no extractBashCommands needed).
|
||||
function toProviderCall(rich: ZedDecodedCall): ParsedProviderCall {
|
||||
return {
|
||||
provider: 'zed',
|
||||
model: opts.model,
|
||||
inputTokens: input,
|
||||
outputTokens: output,
|
||||
cacheCreationInputTokens: cacheWrite,
|
||||
cacheReadInputTokens: cacheRead,
|
||||
cachedInputTokens: cacheRead,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
model: rich.model,
|
||||
inputTokens: rich.inputTokens,
|
||||
outputTokens: rich.outputTokens,
|
||||
cacheCreationInputTokens: rich.cacheCreationInputTokens,
|
||||
cacheReadInputTokens: rich.cacheReadInputTokens,
|
||||
cachedInputTokens: rich.cachedInputTokens,
|
||||
reasoningTokens: rich.reasoningTokens,
|
||||
webSearchRequests: rich.webSearchRequests,
|
||||
costBasis: 'estimated',
|
||||
tools: [],
|
||||
tools: rich.tools,
|
||||
bashCommands: [],
|
||||
timestamp: opts.timestamp,
|
||||
speed: 'standard',
|
||||
deduplicationKey: `zed:${opts.threadId}:${opts.requestKey}`,
|
||||
userMessage: opts.userMessage,
|
||||
sessionId: opts.threadId,
|
||||
}
|
||||
}
|
||||
|
||||
function parseThreads(db: SqliteDatabase, seenKeys: Set<string>): ParsedProviderCall[] {
|
||||
const calls: ParsedProviderCall[] = []
|
||||
let skipped = 0
|
||||
|
||||
let rows: ThreadRow[]
|
||||
try {
|
||||
rows = db.query<ThreadRow>(THREADS_QUERY)
|
||||
} catch {
|
||||
return calls
|
||||
}
|
||||
|
||||
for (const row of rows) {
|
||||
try {
|
||||
// Zed's DataType enum is "zstd" (current save path) or "json" (legacy
|
||||
// uncompressed rows); anything else is unknown.
|
||||
if (!row.id || !row.data || (row.data_type !== 'zstd' && row.data_type !== 'json')) {
|
||||
if (row.data != null) skipped++
|
||||
continue
|
||||
}
|
||||
const parsedAt = new Date(row.updated_at ?? '')
|
||||
if (Number.isNaN(parsedAt.getTime())) continue
|
||||
const timestamp = parsedAt.toISOString()
|
||||
|
||||
const jsonText = row.data_type === 'zstd'
|
||||
? zstdDecompress!(Buffer.from(row.data)).toString('utf-8')
|
||||
: Buffer.from(row.data).toString('utf-8')
|
||||
const thread = JSON.parse(jsonText) as ThreadJson
|
||||
const model = thread.model?.model || 'unknown'
|
||||
const userMessage = row.summary ?? ''
|
||||
|
||||
const requests = Object.entries(thread.request_token_usage ?? {}).filter(([, usage]) => usage != null && !usageIsEmpty(usage))
|
||||
// The per-request map is keyed by user message and does not cover every
|
||||
// request (verified on a real thread: cumulative was ~3x the map sum),
|
||||
// so a remainder entry tops the thread up to the exact cumulative
|
||||
// counter. Threads with an empty map degrade to one cumulative call.
|
||||
const entries: Array<[string, TokenUsage]> = [...requests]
|
||||
const cumulative = thread.cumulative_token_usage
|
||||
if (cumulative && !usageIsEmpty(cumulative)) {
|
||||
let sumIn = 0, sumOut = 0, sumWrite = 0, sumRead = 0
|
||||
for (const [, usage] of requests) {
|
||||
sumIn += num(usage.input_tokens)
|
||||
sumOut += num(usage.output_tokens)
|
||||
sumWrite += num(usage.cache_creation_input_tokens)
|
||||
sumRead += num(usage.cache_read_input_tokens)
|
||||
}
|
||||
const remainder: TokenUsage = {
|
||||
input_tokens: Math.max(0, num(cumulative.input_tokens) - sumIn),
|
||||
output_tokens: Math.max(0, num(cumulative.output_tokens) - sumOut),
|
||||
cache_creation_input_tokens: Math.max(0, num(cumulative.cache_creation_input_tokens) - sumWrite),
|
||||
cache_read_input_tokens: Math.max(0, num(cumulative.cache_read_input_tokens) - sumRead),
|
||||
}
|
||||
if (!usageIsEmpty(remainder)) entries.push(['cumulative-remainder', remainder])
|
||||
}
|
||||
|
||||
for (const [requestKey, usage] of entries) {
|
||||
const call = buildCall({ threadId: row.id, requestKey, usage, model, timestamp, userMessage })
|
||||
if (seenKeys.has(call.deduplicationKey)) continue
|
||||
seenKeys.add(call.deduplicationKey)
|
||||
calls.push(call)
|
||||
}
|
||||
} catch {
|
||||
skipped++
|
||||
}
|
||||
}
|
||||
|
||||
if (skipped > 0) {
|
||||
process.stderr.write(`codeburn: skipped ${skipped} unreadable Zed threads\n`)
|
||||
}
|
||||
return calls
|
||||
}
|
||||
|
||||
function createParser(source: SessionSource, seenKeys: Set<string>): SessionParser {
|
||||
return {
|
||||
async *parse(): AsyncGenerator<ParsedProviderCall> {
|
||||
if (!isSqliteAvailable()) {
|
||||
process.stderr.write(getSqliteLoadError() + '\n')
|
||||
return
|
||||
}
|
||||
if (!zstdDecompress) {
|
||||
process.stderr.write('codeburn: Zed threads need Node >= 22.15 (zstd support); skipping Zed usage.\n')
|
||||
return
|
||||
}
|
||||
|
||||
let db: SqliteDatabase
|
||||
try {
|
||||
db = openDatabase(source.path)
|
||||
} catch (err) {
|
||||
process.stderr.write(`codeburn: cannot open Zed database: ${err instanceof Error ? err.message : err}\n`)
|
||||
return
|
||||
}
|
||||
try {
|
||||
for (const call of parseThreads(db, seenKeys)) {
|
||||
yield call
|
||||
}
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
},
|
||||
timestamp: rich.timestamp,
|
||||
speed: rich.speed,
|
||||
deduplicationKey: rich.deduplicationKey,
|
||||
userMessage: rich.userMessage,
|
||||
sessionId: rich.sessionId,
|
||||
}
|
||||
}
|
||||
|
||||
export function createZedProvider(dbPathOverride?: string): Provider {
|
||||
return {
|
||||
return createBridgedProvider<ZedDecodedCall>({
|
||||
name: 'zed',
|
||||
displayName: 'Zed',
|
||||
|
||||
|
|
@ -219,10 +85,49 @@ export function createZedProvider(dbPathOverride?: string): Provider {
|
|||
return [{ path: dbPath, project: 'zed', provider: 'zed' }]
|
||||
},
|
||||
|
||||
createSessionParser(source: SessionSource, seenKeys: Set<string>): SessionParser {
|
||||
return createParser(source, seenKeys)
|
||||
// I/O adapter: open the db and run the threads query (sqlite-side). Rows
|
||||
// (blob and all) are handed straight to the core decoder, which does the
|
||||
// zstd decompression and JSON parsing now that Node-version support for it
|
||||
// is confirmed.
|
||||
async readRecords(source: SessionSource): Promise<unknown[] | null> {
|
||||
if (!isSqliteAvailable()) {
|
||||
process.stderr.write(getSqliteLoadError() + '\n')
|
||||
return null
|
||||
}
|
||||
if (!zstdDecompress) {
|
||||
process.stderr.write('codeburn: Zed threads need Node >= 22.15 (zstd support); skipping Zed usage.\n')
|
||||
return null
|
||||
}
|
||||
|
||||
let db: SqliteDatabase
|
||||
try {
|
||||
db = openDatabase(source.path)
|
||||
} catch (err) {
|
||||
process.stderr.write(`codeburn: cannot open Zed database: ${err instanceof Error ? err.message : err}\n`)
|
||||
return null
|
||||
}
|
||||
try {
|
||||
return db.query<ZedThreadRow>(THREADS_QUERY)
|
||||
} catch {
|
||||
return []
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
// The core decoder reports unreadable/unknown-shape threads as record
|
||||
// diagnostics, which the bridge discards. The pre-migration decode printed
|
||||
// one aggregate stderr line per scan for exactly those rows, so that notice
|
||||
// is re-emitted here rather than silently dropped.
|
||||
decode(input) {
|
||||
const { calls, diagnostics } = decodeZed(input)
|
||||
if (diagnostics.length > 0) {
|
||||
process.stderr.write(`codeburn: skipped ${diagnostics.length} unreadable Zed threads\n`)
|
||||
}
|
||||
return { calls }
|
||||
},
|
||||
toProviderCall,
|
||||
})
|
||||
}
|
||||
|
||||
export const zed = createZedProvider()
|
||||
|
|
|
|||
212
packages/cli/tests/providers/crush-bridge.test.ts
Normal file
212
packages/cli/tests/providers/crush-bridge.test.ts
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
import { mkdtemp, mkdir, writeFile, rm } from 'fs/promises'
|
||||
import { mkdirSync } from 'fs'
|
||||
import { join } from 'path'
|
||||
import { tmpdir } from 'os'
|
||||
import { createRequire } from 'node:module'
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
|
||||
import { createCrushProvider } from '../../src/providers/crush.js'
|
||||
import { priceProviderCall } from '../../src/pricing-pass.js'
|
||||
import { isSqliteAvailable } from '../../src/sqlite.js'
|
||||
import type { ParsedProviderCall } from '../../src/providers/types.js'
|
||||
|
||||
// Byte-identical parity gate for the crush bridge migration (phase 8, Category
|
||||
// B / sqlite). Crush is not present in the frozen corpus, so a committed
|
||||
// fixture golden is THE parity gate: the bridged provider (sqlite driver + SQL
|
||||
// queries CLI-side, pure row->call decode delegated to
|
||||
// @codeburn/core/providers/crush) must reproduce exactly what the
|
||||
// pre-migration in-CLI decode produced for this fixture DB. The GOLDEN below
|
||||
// was captured from the legacy provider before the migration.
|
||||
|
||||
const requireForTest = createRequire(import.meta.url)
|
||||
|
||||
// CREATE TABLE statements taken verbatim from charmbracelet/crush@v0.66.1
|
||||
// internal/db/migrations/20250424200609_initial.sql (same fixture shape as
|
||||
// tests/providers/crush.test.ts).
|
||||
function createCrushDb(dir: string): string {
|
||||
mkdirSync(dir, { recursive: true })
|
||||
const dbPath = join(dir, 'crush.db')
|
||||
const { DatabaseSync: Database } = requireForTest('node:sqlite')
|
||||
const db = new Database(dbPath)
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
parent_session_id TEXT,
|
||||
title TEXT NOT NULL,
|
||||
message_count INTEGER NOT NULL DEFAULT 0 CHECK (message_count >= 0),
|
||||
prompt_tokens INTEGER NOT NULL DEFAULT 0 CHECK (prompt_tokens >= 0),
|
||||
completion_tokens INTEGER NOT NULL DEFAULT 0 CHECK (completion_tokens >= 0),
|
||||
cost REAL NOT NULL DEFAULT 0.0 CHECK (cost >= 0.0),
|
||||
updated_at INTEGER NOT NULL,
|
||||
created_at INTEGER NOT NULL,
|
||||
summary_message_id TEXT,
|
||||
todos TEXT
|
||||
)
|
||||
`)
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id TEXT PRIMARY KEY,
|
||||
session_id TEXT NOT NULL,
|
||||
role TEXT NOT NULL,
|
||||
parts TEXT NOT NULL DEFAULT '[]',
|
||||
model TEXT,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL,
|
||||
finished_at INTEGER,
|
||||
provider TEXT,
|
||||
is_summary_message INTEGER DEFAULT 0 NOT NULL,
|
||||
FOREIGN KEY (session_id) REFERENCES sessions (id) ON DELETE CASCADE
|
||||
)
|
||||
`)
|
||||
db.close()
|
||||
return dbPath
|
||||
}
|
||||
|
||||
function seed(dbPath: string): void {
|
||||
const { DatabaseSync: Database } = requireForTest('node:sqlite')
|
||||
const db = new Database(dbPath)
|
||||
try {
|
||||
db.prepare(`
|
||||
INSERT INTO sessions (id, parent_session_id, title, message_count, prompt_tokens, completion_tokens, cost, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run('sess-alpha', null, 'test session', 3, 1500, 420, 0.0921, 1_700_000_100, 1_700_000_555)
|
||||
for (const [id, model, ts] of [['m1', 'claude-sonnet-4-6', 1_700_000_100], ['m2', 'claude-sonnet-4-6', 1_700_000_200], ['m3', 'gpt-5', 1_700_000_300]] as const) {
|
||||
db.prepare(`INSERT INTO messages (id, session_id, role, parts, model, created_at, updated_at) VALUES (?, ?, ?, '[]', ?, ?, ?)`)
|
||||
.run(id, 'sess-alpha', 'assistant', model, ts, ts)
|
||||
}
|
||||
// No cost -> estimated fallback.
|
||||
db.prepare(`
|
||||
INSERT INTO sessions (id, parent_session_id, title, message_count, prompt_tokens, completion_tokens, cost, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`).run('sess-beta', null, 'test session 2', 1, 300, 80, 0, 1_700_000_600, 1_700_000_600)
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
}
|
||||
|
||||
// Captured from the pre-migration crush decode (dominant model wins among
|
||||
// tied assistant messages, measured cost from the DB, estimated fallback when
|
||||
// cost is 0). `costUSD` in the GOLDEN below is the raw decoder output
|
||||
// (pre-pricing-pass); the second test proves the pricing pass adds costUSD to
|
||||
// the estimated row and leaves the measured row untouched.
|
||||
const GOLDEN: ParsedProviderCall[] = [
|
||||
{
|
||||
provider: 'crush',
|
||||
model: 'claude-sonnet-4-6',
|
||||
inputTokens: 1500,
|
||||
outputTokens: 420,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
costUSD: 0.0921,
|
||||
costBasis: 'measured',
|
||||
tools: [],
|
||||
bashCommands: [],
|
||||
timestamp: '2023-11-14T22:22:35.000Z',
|
||||
speed: 'standard',
|
||||
deduplicationKey: 'crush:sess-alpha',
|
||||
userMessage: '',
|
||||
sessionId: 'sess-alpha',
|
||||
},
|
||||
{
|
||||
provider: 'crush',
|
||||
model: 'unknown',
|
||||
inputTokens: 300,
|
||||
outputTokens: 80,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
costBasis: 'estimated',
|
||||
tools: [],
|
||||
bashCommands: [],
|
||||
timestamp: '2023-11-14T22:23:20.000Z',
|
||||
speed: 'standard',
|
||||
deduplicationKey: 'crush:sess-beta',
|
||||
userMessage: '',
|
||||
sessionId: 'sess-beta',
|
||||
},
|
||||
]
|
||||
|
||||
let tmpRoot: string
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpRoot = await mkdtemp(join(tmpdir(), 'crush-bridge-'))
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tmpRoot, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
async function collect(): Promise<ParsedProviderCall[]> {
|
||||
const projectDir = join(tmpRoot, 'project-alpha')
|
||||
const dbPath = createCrushDb(join(projectDir, '.crush'))
|
||||
seed(dbPath)
|
||||
|
||||
const globalData = join(tmpRoot, 'crush-global')
|
||||
await mkdir(globalData, { recursive: true })
|
||||
await writeFile(join(globalData, 'projects.json'), JSON.stringify({
|
||||
'proj-a': { path: projectDir, data_dir: '.crush' },
|
||||
}))
|
||||
process.env['CRUSH_GLOBAL_DATA'] = globalData
|
||||
|
||||
const provider = createCrushProvider()
|
||||
const sources = await provider.discoverSessions()
|
||||
sources.sort((a, b) => a.path.localeCompare(b.path))
|
||||
const seen = new Set<string>()
|
||||
const calls: ParsedProviderCall[] = []
|
||||
for (const source of sources) {
|
||||
for await (const call of provider.createSessionParser(source, seen).parse()) {
|
||||
calls.push(call)
|
||||
}
|
||||
}
|
||||
return calls
|
||||
}
|
||||
|
||||
describe.skipIf(!isSqliteAvailable())('crush bridge — fixture parity', () => {
|
||||
it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => {
|
||||
expect(await collect()).toEqual(GOLDEN)
|
||||
})
|
||||
|
||||
it('the priced output survives the pricing pass with only costUSD (for the estimated row) filled in', async () => {
|
||||
const raw = await collect()
|
||||
const priced = raw.map(priceProviderCall)
|
||||
priced.forEach((call, i) => {
|
||||
expect(typeof call.costUSD).toBe('number')
|
||||
expect(Number.isFinite(call.costUSD)).toBe(true)
|
||||
const { costUSD, ...rest } = call
|
||||
const { costUSD: rawCostUSD, ...rawRest } = raw[i]!
|
||||
expect(rest).toEqual(rawRest)
|
||||
// The measured row's costUSD is untouched; the estimated row's is filled in.
|
||||
if (raw[i]!.costBasis === 'measured') expect(costUSD).toBe(rawCostUSD)
|
||||
})
|
||||
})
|
||||
|
||||
it('discovery, I/O, and dedup stay CLI-side; the shared seenKeys set dedups', async () => {
|
||||
const projectDir = join(tmpRoot, 'project-alpha')
|
||||
const dbPath = createCrushDb(join(projectDir, '.crush'))
|
||||
seed(dbPath)
|
||||
const globalData = join(tmpRoot, 'crush-global')
|
||||
await mkdir(globalData, { recursive: true })
|
||||
await writeFile(join(globalData, 'projects.json'), JSON.stringify({ 'proj-a': { path: projectDir, data_dir: '.crush' } }))
|
||||
process.env['CRUSH_GLOBAL_DATA'] = globalData
|
||||
|
||||
const provider = createCrushProvider()
|
||||
const sources = await provider.discoverSessions()
|
||||
const seen = new Set<string>()
|
||||
const first: ParsedProviderCall[] = []
|
||||
for (const source of sources) {
|
||||
for await (const call of provider.createSessionParser(source, seen).parse()) first.push(call)
|
||||
}
|
||||
const second: ParsedProviderCall[] = []
|
||||
for (const source of sources) {
|
||||
for await (const call of provider.createSessionParser(source, seen).parse()) second.push(call)
|
||||
}
|
||||
expect(first.length).toBe(2)
|
||||
expect(second).toEqual([])
|
||||
})
|
||||
})
|
||||
219
packages/cli/tests/providers/forge-bridge.test.ts
Normal file
219
packages/cli/tests/providers/forge-bridge.test.ts
Normal file
|
|
@ -0,0 +1,219 @@
|
|||
import { mkdtemp, rm } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
import { tmpdir } from 'os'
|
||||
import { createRequire } from 'node:module'
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
|
||||
import { createForgeProvider } from '../../src/providers/forge.js'
|
||||
import { priceProviderCall } from '../../src/pricing-pass.js'
|
||||
import { isSqliteAvailable } from '../../src/sqlite.js'
|
||||
import type { ParsedProviderCall } from '../../src/providers/types.js'
|
||||
|
||||
// Byte-identical parity gate for the forge bridge migration (phase 8, Category
|
||||
// B / sqlite). Forge is not present in the frozen corpus, so a committed
|
||||
// fixture golden is THE parity gate. The GOLDEN below was captured from the
|
||||
// legacy provider before the migration.
|
||||
|
||||
const requireForTest = createRequire(import.meta.url)
|
||||
|
||||
type TestDb = {
|
||||
exec(sql: string): void
|
||||
prepare(sql: string): { run(...params: unknown[]): void }
|
||||
close(): void
|
||||
}
|
||||
|
||||
function createForgeDb(dir: string): string {
|
||||
const dbPath = join(dir, 'forge.db')
|
||||
const { DatabaseSync: Database } = requireForTest('node:sqlite')
|
||||
const db = new Database(dbPath)
|
||||
db.exec(`CREATE TABLE conversations(
|
||||
conversation_id TEXT PRIMARY KEY NOT NULL, title TEXT, workspace_id BIGINT NOT NULL,
|
||||
context TEXT, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP, metrics TEXT
|
||||
)`)
|
||||
db.close()
|
||||
return dbPath
|
||||
}
|
||||
|
||||
function withTestDb(dbPath: string, fn: (db: TestDb) => void): void {
|
||||
const { DatabaseSync: Database } = requireForTest('node:sqlite')
|
||||
const db = new Database(dbPath)
|
||||
try {
|
||||
fn(db)
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
}
|
||||
|
||||
const CONTEXT = {
|
||||
messages: [
|
||||
{ message: { text: { role: 'User', content: 'implement forge bridge' } } },
|
||||
{
|
||||
message: {
|
||||
text: {
|
||||
role: 'Assistant', content: '', model: 'claude-opus-4-6',
|
||||
tool_calls: [
|
||||
{ name: 'shell', call_id: 'call-1', arguments: { command: 'git status && npm test' } },
|
||||
{ name: 'Read', call_id: 'call-2', arguments: { file_path: '/tmp/a' } },
|
||||
],
|
||||
},
|
||||
},
|
||||
usage: { prompt_tokens: { actual: 1200 }, completion_tokens: { actual: 300 }, cached_tokens: { actual: 200 } },
|
||||
},
|
||||
{ message: { text: { role: 'User', content: 'now write tests' } } },
|
||||
{
|
||||
message: { text: { role: 'Assistant', model: 'claude-sonnet-4-6', tool_calls: [{ name: 'unknown_tool', call_id: 'call-3', arguments: {} }] } },
|
||||
usage: { prompt_tokens: { actual: 400 }, completion_tokens: { actual: 90 } },
|
||||
},
|
||||
// Zero-token assistant message: must be skipped.
|
||||
{ message: { text: { role: 'Assistant', model: 'claude-sonnet-4-6' } }, usage: { prompt_tokens: { actual: 0 }, completion_tokens: { actual: 0 } } },
|
||||
// Tool names that collide with Object.prototype members. Tool names come
|
||||
// straight from the conversation JSON, so mapping them through a plain
|
||||
// object-literal lookup would resolve the INHERITED member (a Function, or
|
||||
// Object.prototype itself for `__proto__`) instead of falling through to
|
||||
// the identity default. The pre-migration decode used a `switch` and
|
||||
// returned each name verbatim; this row pins that.
|
||||
{
|
||||
message: {
|
||||
text: {
|
||||
role: 'Assistant', model: 'claude-haiku-4-5',
|
||||
tool_calls: [
|
||||
{ name: 'constructor', call_id: 'call-4', arguments: {} },
|
||||
{ name: 'toString', call_id: 'call-5', arguments: {} },
|
||||
{ name: '__proto__', call_id: 'call-6', arguments: {} },
|
||||
{ name: 'hasOwnProperty', call_id: 'call-7', arguments: {} },
|
||||
],
|
||||
},
|
||||
},
|
||||
usage: { prompt_tokens: { actual: 50 }, completion_tokens: { actual: 5 } },
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
function seed(dbPath: string): void {
|
||||
withTestDb(dbPath, db => {
|
||||
db.prepare(`INSERT INTO conversations (conversation_id, title, workspace_id, context, created_at, updated_at, metrics) VALUES (?, ?, ?, ?, ?, ?, ?)`)
|
||||
.run('conv-1', 'Forge Project', 123, JSON.stringify(CONTEXT), '2026-05-06 15:00:00', '2026-05-06 15:20:41.379094', null)
|
||||
})
|
||||
}
|
||||
|
||||
const GOLDEN: ParsedProviderCall[] = [
|
||||
{
|
||||
provider: 'forge',
|
||||
model: 'claude-opus-4-6',
|
||||
inputTokens: 1000,
|
||||
outputTokens: 300,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 200,
|
||||
cachedInputTokens: 200,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
costBasis: 'estimated',
|
||||
tools: ['Bash', 'Read'],
|
||||
bashCommands: ['git', 'npm'],
|
||||
timestamp: '2026-05-06T15:20:41.379Z',
|
||||
speed: 'standard',
|
||||
deduplicationKey: 'forge:conv-1:call-1',
|
||||
userMessage: 'implement forge bridge',
|
||||
sessionId: 'conv-1',
|
||||
},
|
||||
{
|
||||
provider: 'forge',
|
||||
model: 'claude-sonnet-4-6',
|
||||
inputTokens: 400,
|
||||
outputTokens: 90,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
costBasis: 'estimated',
|
||||
tools: ['unknown_tool'],
|
||||
bashCommands: [],
|
||||
timestamp: '2026-05-06T15:20:41.379Z',
|
||||
speed: 'standard',
|
||||
deduplicationKey: 'forge:conv-1:call-3',
|
||||
userMessage: 'now write tests',
|
||||
sessionId: 'conv-1',
|
||||
},
|
||||
{
|
||||
provider: 'forge',
|
||||
model: 'claude-haiku-4-5',
|
||||
inputTokens: 50,
|
||||
outputTokens: 5,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
costBasis: 'estimated',
|
||||
tools: ['constructor', 'toString', '__proto__', 'hasOwnProperty'],
|
||||
bashCommands: [],
|
||||
timestamp: '2026-05-06T15:20:41.379Z',
|
||||
speed: 'standard',
|
||||
deduplicationKey: 'forge:conv-1:call-4',
|
||||
userMessage: 'now write tests',
|
||||
sessionId: 'conv-1',
|
||||
},
|
||||
]
|
||||
|
||||
let tmpRoot: string
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpRoot = await mkdtemp(join(tmpdir(), 'forge-bridge-'))
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tmpRoot, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
async function collect(dbPath: string, seen = new Set<string>()): Promise<ParsedProviderCall[]> {
|
||||
const provider = createForgeProvider(dbPath)
|
||||
const sources = await provider.discoverSessions()
|
||||
const calls: ParsedProviderCall[] = []
|
||||
for (const source of sources) {
|
||||
for await (const call of provider.createSessionParser(source, seen).parse()) {
|
||||
calls.push(call)
|
||||
}
|
||||
}
|
||||
return calls
|
||||
}
|
||||
|
||||
describe.skipIf(!isSqliteAvailable())('forge bridge — fixture parity', () => {
|
||||
it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => {
|
||||
const dbPath = createForgeDb(tmpRoot)
|
||||
seed(dbPath)
|
||||
expect(await collect(dbPath)).toEqual(GOLDEN)
|
||||
})
|
||||
|
||||
it('the priced output survives the pricing pass with only costUSD added', async () => {
|
||||
const dbPath = createForgeDb(tmpRoot)
|
||||
seed(dbPath)
|
||||
const raw = await collect(dbPath)
|
||||
const priced = raw.map(priceProviderCall)
|
||||
priced.forEach((call, i) => {
|
||||
expect(typeof call.costUSD).toBe('number')
|
||||
expect(Number.isFinite(call.costUSD)).toBe(true)
|
||||
const { costUSD, ...rest } = call
|
||||
expect(rest).toEqual(raw[i])
|
||||
})
|
||||
})
|
||||
|
||||
it('discovery, I/O, and dedup stay CLI-side; the shared seenKeys set dedups', async () => {
|
||||
const dbPath = createForgeDb(tmpRoot)
|
||||
seed(dbPath)
|
||||
const seen = new Set<string>()
|
||||
const first = await collect(dbPath, seen)
|
||||
const second = await collect(dbPath, seen)
|
||||
expect(first.length).toBe(3)
|
||||
expect(second).toEqual([])
|
||||
})
|
||||
|
||||
it('every emitted tool name is a string (no inherited prototype member leaks in)', async () => {
|
||||
const dbPath = createForgeDb(tmpRoot)
|
||||
seed(dbPath)
|
||||
for (const call of await collect(dbPath)) {
|
||||
for (const tool of call.tools) expect(typeof tool).toBe('string')
|
||||
}
|
||||
})
|
||||
})
|
||||
178
packages/cli/tests/providers/goose-bridge.test.ts
Normal file
178
packages/cli/tests/providers/goose-bridge.test.ts
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
import { mkdtemp, rm } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
import { tmpdir } from 'os'
|
||||
import { createRequire } from 'node:module'
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
|
||||
import { createGooseProvider } from '../../src/providers/goose.js'
|
||||
import { priceProviderCall } from '../../src/pricing-pass.js'
|
||||
import { isSqliteAvailable } from '../../src/sqlite.js'
|
||||
import type { ParsedProviderCall, SessionSource } from '../../src/providers/types.js'
|
||||
|
||||
// Byte-identical parity gate for the goose bridge migration (phase 8, Category
|
||||
// B / sqlite). Goose is not present in the frozen corpus and had no pre-
|
||||
// existing fixture test, so this fixture-building pattern is modeled on the
|
||||
// sibling sqlite providers (crush/zed/forge). The GOLDEN below was captured
|
||||
// from the legacy in-CLI decode before the migration (createGooseProvider has
|
||||
// no db-path override, so `createSessionParser` is exercised directly against
|
||||
// a manually-built source, exactly as the capture script did).
|
||||
|
||||
const requireForTest = createRequire(import.meta.url)
|
||||
|
||||
function createGooseDb(dir: string): string {
|
||||
const dbPath = join(dir, 'sessions.db')
|
||||
const { DatabaseSync: Database } = requireForTest('node:sqlite')
|
||||
const db = new Database(dbPath)
|
||||
db.exec(`CREATE TABLE sessions (
|
||||
id TEXT PRIMARY KEY, name TEXT, working_dir TEXT, created_at TEXT, updated_at TEXT,
|
||||
accumulated_input_tokens INTEGER, accumulated_output_tokens INTEGER, provider_name TEXT, model_config_json BLOB
|
||||
)`)
|
||||
db.exec(`CREATE TABLE messages (
|
||||
message_id TEXT PRIMARY KEY, session_id TEXT NOT NULL, role TEXT NOT NULL, content_json BLOB NOT NULL, created_timestamp INTEGER NOT NULL
|
||||
)`)
|
||||
db.close()
|
||||
return dbPath
|
||||
}
|
||||
|
||||
function seed(dbPath: string, opts: { badTimestamps?: boolean; singleTurn?: boolean } = {}): void {
|
||||
const { DatabaseSync: Database } = requireForTest('node:sqlite')
|
||||
const db = new Database(dbPath)
|
||||
try {
|
||||
const modelConfig = JSON.stringify({ model_name: 'gpt-5.5', reasoning: true })
|
||||
db.prepare(`INSERT INTO sessions (id, name, working_dir, created_at, updated_at, accumulated_input_tokens, accumulated_output_tokens, provider_name, model_config_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
.run('sess-1', 'goose session', '/Users/me/project',
|
||||
opts.badTimestamps ? 'not-a-date' : '2026-06-01T10:00:00Z',
|
||||
opts.badTimestamps ? 'also-not-a-date' : '2026-06-01T10:05:30Z',
|
||||
1500, 400, 'openai', Buffer.from(modelConfig))
|
||||
|
||||
db.prepare(`INSERT INTO messages (message_id, session_id, role, content_json, created_timestamp) VALUES (?, ?, ?, ?, ?)`)
|
||||
.run('msg-1', 'sess-1', 'user', Buffer.from(JSON.stringify([{ type: 'text', text: 'please refactor the shell wrapper and run tests' }])), 1_780_000_000)
|
||||
|
||||
const assistantContent1 = [
|
||||
{ type: 'text', text: 'working on it' },
|
||||
{ type: 'toolRequest', toolCall: { value: { name: 'developer__shell', arguments: { command: 'npm test && npm run lint' } } } },
|
||||
{ type: 'toolRequest', toolCall: { value: { name: 'developer__read_file', arguments: { file_path: 'src/index.ts' } } } },
|
||||
]
|
||||
db.prepare(`INSERT INTO messages (message_id, session_id, role, content_json, created_timestamp) VALUES (?, ?, ?, ?, ?)`)
|
||||
.run('msg-2', 'sess-1', 'assistant', Buffer.from(JSON.stringify(assistantContent1)), 1_780_000_010)
|
||||
|
||||
if (!opts.singleTurn) {
|
||||
const assistantContent2 = [
|
||||
{ type: 'toolRequest', toolCall: { value: { name: 'some_native_tool', arguments: {} } } },
|
||||
]
|
||||
db.prepare(`INSERT INTO messages (message_id, session_id, role, content_json, created_timestamp) VALUES (?, ?, ?, ?, ?)`)
|
||||
.run('msg-3', 'sess-1', 'assistant', Buffer.from(JSON.stringify(assistantContent2)), 1_780_000_020)
|
||||
}
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
}
|
||||
|
||||
const GOLDEN: ParsedProviderCall[] = [
|
||||
{
|
||||
provider: 'goose',
|
||||
model: 'gpt-5.5',
|
||||
inputTokens: 1500,
|
||||
outputTokens: 400,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
costBasis: 'estimated',
|
||||
tools: ['Bash', 'Read', 'some_native_tool'],
|
||||
bashCommands: ['npm'],
|
||||
toolSequence: [
|
||||
[
|
||||
{ tool: 'Bash', command: 'npm test && npm run lint' },
|
||||
{ tool: 'Read', file: 'src/index.ts' },
|
||||
],
|
||||
[
|
||||
{ tool: 'some_native_tool' },
|
||||
],
|
||||
],
|
||||
timestamp: '2026-06-01T10:05:30.000Z',
|
||||
speed: 'standard',
|
||||
deduplicationKey: 'goose:sess-1',
|
||||
userMessage: 'please refactor the shell wrapper and run tests',
|
||||
sessionId: 'sess-1',
|
||||
},
|
||||
]
|
||||
|
||||
let tmpRoot: string
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpRoot = await mkdtemp(join(tmpdir(), 'goose-bridge-'))
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tmpRoot, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
async function collect(dbPath: string, seen = new Set<string>()): Promise<ParsedProviderCall[]> {
|
||||
const provider = createGooseProvider()
|
||||
const source: SessionSource = { path: `${dbPath}:sess-1`, project: 'goose-bridge', provider: 'goose' }
|
||||
const calls: ParsedProviderCall[] = []
|
||||
for await (const call of provider.createSessionParser(source, seen).parse()) {
|
||||
calls.push(call)
|
||||
}
|
||||
return calls
|
||||
}
|
||||
|
||||
describe.skipIf(!isSqliteAvailable())('goose bridge — fixture parity', () => {
|
||||
it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => {
|
||||
const dbPath = createGooseDb(tmpRoot)
|
||||
seed(dbPath)
|
||||
expect(await collect(dbPath)).toEqual(GOLDEN)
|
||||
})
|
||||
|
||||
it('the priced output survives the pricing pass with only costUSD added', async () => {
|
||||
const dbPath = createGooseDb(tmpRoot)
|
||||
seed(dbPath)
|
||||
const raw = await collect(dbPath)
|
||||
const priced = raw.map(priceProviderCall)
|
||||
priced.forEach((call, i) => {
|
||||
expect(typeof call.costUSD).toBe('number')
|
||||
expect(Number.isFinite(call.costUSD)).toBe(true)
|
||||
const { costUSD, ...rest } = call
|
||||
expect(rest).toEqual(raw[i])
|
||||
})
|
||||
})
|
||||
|
||||
it('discovery, I/O, and dedup stay CLI-side; the shared seenKeys set dedups', async () => {
|
||||
const dbPath = createGooseDb(tmpRoot)
|
||||
seed(dbPath)
|
||||
const seen = new Set<string>()
|
||||
const first = await collect(dbPath, seen)
|
||||
const second = await collect(dbPath, seen)
|
||||
expect(first.length).toBe(1)
|
||||
expect(second).toEqual([])
|
||||
})
|
||||
|
||||
it('omits toolSequence when only one turn carries tool calls', async () => {
|
||||
const dbPath = createGooseDb(tmpRoot)
|
||||
seed(dbPath, { singleTurn: true })
|
||||
const [call] = await collect(dbPath)
|
||||
expect(call).toEqual({
|
||||
...GOLDEN[0]!,
|
||||
tools: ['Bash', 'Read'],
|
||||
toolSequence: undefined,
|
||||
})
|
||||
})
|
||||
|
||||
it('falls back to the current time when neither timestamp column parses', async () => {
|
||||
const dbPath = createGooseDb(tmpRoot)
|
||||
seed(dbPath, { badTimestamps: true })
|
||||
const before = Date.now()
|
||||
const [call] = await collect(dbPath)
|
||||
const after = Date.now()
|
||||
// Every other field is identical to the golden; only the timestamp differs,
|
||||
// and it must land on "now" exactly as the pre-migration `new Date()` did.
|
||||
const { timestamp, ...rest } = call!
|
||||
const { timestamp: _goldenTimestamp, ...goldenRest } = GOLDEN[0]!
|
||||
expect(rest).toEqual(goldenRest)
|
||||
expect(Date.parse(timestamp)).toBeGreaterThanOrEqual(before - 1000)
|
||||
expect(Date.parse(timestamp)).toBeLessThanOrEqual(after + 1000)
|
||||
})
|
||||
})
|
||||
172
packages/cli/tests/providers/zcode-bridge.test.ts
Normal file
172
packages/cli/tests/providers/zcode-bridge.test.ts
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
import { mkdtemp, rm } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
import { tmpdir } from 'os'
|
||||
import { createRequire } from 'node:module'
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
|
||||
import { createZcodeProvider } from '../../src/providers/zcode.js'
|
||||
import { priceProviderCall } from '../../src/pricing-pass.js'
|
||||
import { isSqliteAvailable } from '../../src/sqlite.js'
|
||||
import type { ParsedProviderCall } from '../../src/providers/types.js'
|
||||
|
||||
// Byte-identical parity gate for the zcode bridge migration (phase 8, Category
|
||||
// B / sqlite). ZCode is not present in the frozen corpus, so a committed
|
||||
// fixture golden is THE parity gate. The GOLDEN below was captured from the
|
||||
// legacy provider before the migration.
|
||||
|
||||
const requireForTest = createRequire(import.meta.url)
|
||||
|
||||
function createZcodeDb(dir: string): string {
|
||||
const dbPath = join(dir, 'db.sqlite')
|
||||
const { DatabaseSync: Database } = requireForTest('node:sqlite')
|
||||
const db = new Database(dbPath)
|
||||
db.exec(`CREATE TABLE session (id TEXT PRIMARY KEY, directory TEXT NOT NULL)`)
|
||||
db.exec(`CREATE TABLE model_usage (
|
||||
id TEXT PRIMARY KEY, session_id TEXT NOT NULL, turn_id TEXT, model_id TEXT NOT NULL,
|
||||
input_tokens INTEGER NOT NULL DEFAULT 0, output_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
reasoning_tokens INTEGER NOT NULL DEFAULT 0, cache_creation_input_tokens INTEGER NOT NULL DEFAULT 0,
|
||||
cache_read_input_tokens INTEGER NOT NULL DEFAULT 0, started_at INTEGER NOT NULL, completed_at INTEGER)`)
|
||||
db.exec(`CREATE TABLE tool_usage (id TEXT PRIMARY KEY, session_id TEXT NOT NULL, turn_id TEXT, tool_name TEXT NOT NULL, started_at INTEGER NOT NULL)`)
|
||||
db.close()
|
||||
return dbPath
|
||||
}
|
||||
|
||||
function seed(dbPath: string): void {
|
||||
const { DatabaseSync: Database } = requireForTest('node:sqlite')
|
||||
const db = new Database(dbPath)
|
||||
try {
|
||||
db.prepare('INSERT INTO session (id, directory) VALUES (?, ?)').run('sess-1', '/Users/me/proj')
|
||||
db.prepare(`INSERT INTO model_usage (id, session_id, turn_id, model_id, input_tokens, output_tokens, reasoning_tokens, cache_creation_input_tokens, cache_read_input_tokens, started_at, completed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
.run('mu-1', 'sess-1', 'turn-1', 'GLM-5.2', 9125, 27, 12, 0, 8064, 1781981181862, 1781981202412)
|
||||
db.prepare(`INSERT INTO model_usage (id, session_id, turn_id, model_id, input_tokens, output_tokens, reasoning_tokens, cache_creation_input_tokens, cache_read_input_tokens, started_at, completed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
.run('mu-2', 'sess-1', 'turn-1', 'GLM-5.2', 200, 40, 0, 0, 0, 1781981210000, 1781981220000)
|
||||
// Zero-token row: must be skipped entirely.
|
||||
db.prepare(`INSERT INTO model_usage (id, session_id, turn_id, model_id, input_tokens, output_tokens, reasoning_tokens, cache_creation_input_tokens, cache_read_input_tokens, started_at, completed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
.run('mu-zero', 'sess-1', 'turn-2', 'GLM-5.2', 0, 0, 0, 0, 0, 1781981230000, 1781981231000)
|
||||
// No turn_id -> no tools attached.
|
||||
db.prepare(`INSERT INTO model_usage (id, session_id, turn_id, model_id, input_tokens, output_tokens, reasoning_tokens, cache_creation_input_tokens, cache_read_input_tokens, started_at, completed_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
.run('mu-3', 'sess-1', null, 'GLM-5.2', 500, 60, 0, 100, 0, 1781981240000, null)
|
||||
|
||||
db.prepare('INSERT INTO tool_usage (id, session_id, turn_id, tool_name, started_at) VALUES (?, ?, ?, ?, ?)').run('tu-1', 'sess-1', 'turn-1', 'Bash', 1781981185000)
|
||||
db.prepare('INSERT INTO tool_usage (id, session_id, turn_id, tool_name, started_at) VALUES (?, ?, ?, ?, ?)').run('tu-2', 'sess-1', 'turn-1', 'Read', 1781981190000)
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
}
|
||||
|
||||
const GOLDEN: ParsedProviderCall[] = [
|
||||
{
|
||||
provider: 'zcode',
|
||||
model: 'GLM-5.2',
|
||||
inputTokens: 1061,
|
||||
outputTokens: 27,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 8064,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 12,
|
||||
webSearchRequests: 0,
|
||||
costBasis: 'estimated',
|
||||
tools: ['Bash', 'Read'],
|
||||
bashCommands: [],
|
||||
timestamp: '2026-06-20T18:46:42.412Z',
|
||||
speed: 'standard',
|
||||
deduplicationKey: 'zcode:mu-1',
|
||||
turnId: 'turn-1',
|
||||
userMessage: '',
|
||||
sessionId: 'sess-1',
|
||||
},
|
||||
{
|
||||
provider: 'zcode',
|
||||
model: 'GLM-5.2',
|
||||
inputTokens: 200,
|
||||
outputTokens: 40,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
costBasis: 'estimated',
|
||||
tools: [],
|
||||
bashCommands: [],
|
||||
timestamp: '2026-06-20T18:47:00.000Z',
|
||||
speed: 'standard',
|
||||
deduplicationKey: 'zcode:mu-2',
|
||||
turnId: 'turn-1',
|
||||
userMessage: '',
|
||||
sessionId: 'sess-1',
|
||||
},
|
||||
{
|
||||
provider: 'zcode',
|
||||
model: 'GLM-5.2',
|
||||
inputTokens: 400,
|
||||
outputTokens: 60,
|
||||
cacheCreationInputTokens: 100,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
costBasis: 'estimated',
|
||||
tools: [],
|
||||
bashCommands: [],
|
||||
timestamp: '2026-06-20T18:47:20.000Z',
|
||||
speed: 'standard',
|
||||
deduplicationKey: 'zcode:mu-3',
|
||||
userMessage: '',
|
||||
sessionId: 'sess-1',
|
||||
},
|
||||
]
|
||||
|
||||
let tmpRoot: string
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpRoot = await mkdtemp(join(tmpdir(), 'zcode-bridge-'))
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tmpRoot, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
async function collect(dbPath: string, seen = new Set<string>()): Promise<ParsedProviderCall[]> {
|
||||
const provider = createZcodeProvider(dbPath)
|
||||
const sources = await provider.discoverSessions()
|
||||
const calls: ParsedProviderCall[] = []
|
||||
for (const source of sources) {
|
||||
for await (const call of provider.createSessionParser(source, seen).parse()) {
|
||||
calls.push(call)
|
||||
}
|
||||
}
|
||||
return calls
|
||||
}
|
||||
|
||||
describe.skipIf(!isSqliteAvailable())('zcode bridge — fixture parity', () => {
|
||||
it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => {
|
||||
const dbPath = createZcodeDb(tmpRoot)
|
||||
seed(dbPath)
|
||||
expect(await collect(dbPath)).toEqual(GOLDEN)
|
||||
})
|
||||
|
||||
it('the priced output survives the pricing pass with only costUSD added', async () => {
|
||||
const dbPath = createZcodeDb(tmpRoot)
|
||||
seed(dbPath)
|
||||
const raw = await collect(dbPath)
|
||||
const priced = raw.map(priceProviderCall)
|
||||
priced.forEach((call, i) => {
|
||||
expect(typeof call.costUSD).toBe('number')
|
||||
expect(Number.isFinite(call.costUSD)).toBe(true)
|
||||
expect(call.costUSD).toBeGreaterThan(0)
|
||||
const { costUSD, ...rest } = call
|
||||
expect(rest).toEqual(raw[i])
|
||||
})
|
||||
})
|
||||
|
||||
it('discovery, I/O, and dedup stay CLI-side; the shared seenKeys set dedups', async () => {
|
||||
const dbPath = createZcodeDb(tmpRoot)
|
||||
seed(dbPath)
|
||||
const seen = new Set<string>()
|
||||
const first = await collect(dbPath, seen)
|
||||
const second = await collect(dbPath, seen)
|
||||
expect(first.length).toBe(3)
|
||||
expect(second).toEqual([])
|
||||
})
|
||||
})
|
||||
231
packages/cli/tests/providers/zed-bridge.test.ts
Normal file
231
packages/cli/tests/providers/zed-bridge.test.ts
Normal file
|
|
@ -0,0 +1,231 @@
|
|||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { mkdtemp, rm } from 'fs/promises'
|
||||
import { tmpdir } from 'os'
|
||||
import { join } from 'path'
|
||||
import { createRequire } from 'node:module'
|
||||
import zlib from 'zlib'
|
||||
|
||||
import { createZedProvider } from '../../src/providers/zed.js'
|
||||
import { isSqliteAvailable } from '../../src/sqlite.js'
|
||||
import { priceProviderCall } from '../../src/pricing-pass.js'
|
||||
import type { ParsedProviderCall } from '../../src/providers/types.js'
|
||||
|
||||
// Byte-identical parity gate for the zed bridge migration (phase 8, Category B
|
||||
// / sqlite). Zed is not present in the frozen corpus, so a committed fixture
|
||||
// golden is THE parity gate. The GOLDEN below was captured from the legacy
|
||||
// provider before the migration.
|
||||
|
||||
const requireForTest = createRequire(import.meta.url)
|
||||
const zstd = (zlib as { zstdCompressSync?: (buf: Buffer) => Buffer }).zstdCompressSync
|
||||
|
||||
const skipReason = !isSqliteAvailable()
|
||||
? 'node:sqlite not available — needs Node 22+; skipping'
|
||||
: !zstd
|
||||
? 'zlib zstd not available — needs Node 22.15+; skipping'
|
||||
: null
|
||||
|
||||
let tmpDir: string
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = await mkdtemp(join(tmpdir(), 'zed-bridge-'))
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tmpDir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
function buildDb(fn: (db: {
|
||||
exec(sql: string): void
|
||||
prepare(sql: string): { run(...params: unknown[]): void }
|
||||
close(): void
|
||||
}) => void): string {
|
||||
const dbPath = join(tmpDir, 'threads.db')
|
||||
const { DatabaseSync: Database } = requireForTest('node:sqlite')
|
||||
const db = new Database(dbPath)
|
||||
db.exec(`CREATE TABLE threads (
|
||||
id TEXT PRIMARY KEY,
|
||||
summary TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
data_type TEXT NOT NULL,
|
||||
data BLOB NOT NULL,
|
||||
parent_id TEXT, folder_paths TEXT, folder_paths_order TEXT, created_at TEXT
|
||||
)`)
|
||||
fn(db)
|
||||
db.close()
|
||||
return dbPath
|
||||
}
|
||||
|
||||
function insertThread(db: {
|
||||
prepare(sql: string): { run(...params: unknown[]): void }
|
||||
}, opts: {
|
||||
id: string
|
||||
summary?: string
|
||||
updatedAt?: string
|
||||
dataType?: string
|
||||
thread?: unknown
|
||||
rawData?: Buffer
|
||||
}): void {
|
||||
const data = opts.rawData ?? zstd!(Buffer.from(JSON.stringify(opts.thread ?? {})))
|
||||
db.prepare('INSERT INTO threads (id, summary, updated_at, data_type, data) VALUES (?, ?, ?, ?, ?)').run(
|
||||
opts.id,
|
||||
opts.summary ?? 'a thread',
|
||||
opts.updatedAt ?? '2026-06-20T10:00:00Z',
|
||||
opts.dataType ?? 'zstd',
|
||||
data,
|
||||
)
|
||||
}
|
||||
|
||||
function seedDb(): string {
|
||||
return buildDb((db) => {
|
||||
insertThread(db, {
|
||||
id: 'thread-1',
|
||||
summary: 'refactor the parser',
|
||||
updatedAt: '2026-06-21T09:30:00Z',
|
||||
thread: {
|
||||
model: { provider: 'anthropic', model: 'claude-opus-4-8' },
|
||||
request_token_usage: {
|
||||
'req-1': { input_tokens: 1200, output_tokens: 300, cache_creation_input_tokens: 5000, cache_read_input_tokens: 90000 },
|
||||
'req-2': { input_tokens: 800, output_tokens: 150, cache_creation_input_tokens: 0, cache_read_input_tokens: 95000 },
|
||||
},
|
||||
cumulative_token_usage: { input_tokens: 2000, output_tokens: 450 },
|
||||
},
|
||||
})
|
||||
insertThread(db, { id: 'bad-type', dataType: 'protobuf', rawData: Buffer.from('{}') })
|
||||
insertThread(db, { id: 'bad-blob', rawData: Buffer.from('not zstd at all') })
|
||||
insertThread(db, {
|
||||
id: 'legacy',
|
||||
dataType: 'json',
|
||||
updatedAt: '2026-06-22T11:00:00Z',
|
||||
rawData: Buffer.from(JSON.stringify({
|
||||
model: { model: 'claude-sonnet-4-6' },
|
||||
request_token_usage: { 'req-1': { input_tokens: 40, output_tokens: 8 } },
|
||||
})),
|
||||
})
|
||||
insertThread(db, {
|
||||
id: 'zero-usage',
|
||||
thread: {
|
||||
model: { model: 'claude-opus-4-8' },
|
||||
request_token_usage: { 'req-1': { input_tokens: 0, output_tokens: 0 } },
|
||||
cumulative_token_usage: { input_tokens: 0, output_tokens: 0 },
|
||||
},
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// Captured from the pre-migration zed decode: two per-request calls for
|
||||
// thread-1 (cumulative exactly covered by the map, no remainder), one call for
|
||||
// the legacy uncompressed row, and the bad-type/bad-blob/zero-usage rows
|
||||
// skipped without dropping the healthy threads.
|
||||
const GOLDEN: ParsedProviderCall[] = [
|
||||
{
|
||||
provider: 'zed',
|
||||
model: 'claude-opus-4-8',
|
||||
inputTokens: 1200,
|
||||
outputTokens: 300,
|
||||
cacheCreationInputTokens: 5000,
|
||||
cacheReadInputTokens: 90000,
|
||||
cachedInputTokens: 90000,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
costBasis: 'estimated',
|
||||
tools: [],
|
||||
bashCommands: [],
|
||||
timestamp: '2026-06-21T09:30:00.000Z',
|
||||
speed: 'standard',
|
||||
deduplicationKey: 'zed:thread-1:req-1',
|
||||
userMessage: 'refactor the parser',
|
||||
sessionId: 'thread-1',
|
||||
},
|
||||
{
|
||||
provider: 'zed',
|
||||
model: 'claude-opus-4-8',
|
||||
inputTokens: 800,
|
||||
outputTokens: 150,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 95000,
|
||||
cachedInputTokens: 95000,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
costBasis: 'estimated',
|
||||
tools: [],
|
||||
bashCommands: [],
|
||||
timestamp: '2026-06-21T09:30:00.000Z',
|
||||
speed: 'standard',
|
||||
deduplicationKey: 'zed:thread-1:req-2',
|
||||
userMessage: 'refactor the parser',
|
||||
sessionId: 'thread-1',
|
||||
},
|
||||
{
|
||||
provider: 'zed',
|
||||
model: 'claude-sonnet-4-6',
|
||||
inputTokens: 40,
|
||||
outputTokens: 8,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
costBasis: 'estimated',
|
||||
tools: [],
|
||||
bashCommands: [],
|
||||
timestamp: '2026-06-22T11:00:00.000Z',
|
||||
speed: 'standard',
|
||||
deduplicationKey: 'zed:legacy:req-1',
|
||||
userMessage: 'a thread',
|
||||
sessionId: 'legacy',
|
||||
},
|
||||
]
|
||||
|
||||
async function collectCalls(dbPath: string, seenKeys = new Set<string>()): Promise<ParsedProviderCall[]> {
|
||||
const provider = createZedProvider(dbPath)
|
||||
const sources = await provider.discoverSessions()
|
||||
const calls: ParsedProviderCall[] = []
|
||||
for (const source of sources) {
|
||||
for await (const call of provider.createSessionParser(source, seenKeys).parse()) {
|
||||
calls.push(call)
|
||||
}
|
||||
}
|
||||
return calls
|
||||
}
|
||||
|
||||
describe.skipIf(skipReason !== null)('zed bridge — fixture parity', () => {
|
||||
it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => {
|
||||
const dbPath = seedDb()
|
||||
expect(await collectCalls(dbPath)).toEqual(GOLDEN)
|
||||
})
|
||||
|
||||
it('the priced output survives the pricing pass with only costUSD added', async () => {
|
||||
const dbPath = seedDb()
|
||||
const raw = await collectCalls(dbPath)
|
||||
const priced = raw.map(priceProviderCall)
|
||||
priced.forEach((call, i) => {
|
||||
expect(typeof call.costUSD).toBe('number')
|
||||
expect(Number.isFinite(call.costUSD)).toBe(true)
|
||||
const { costUSD, ...rest } = call
|
||||
expect(rest).toEqual(raw[i])
|
||||
})
|
||||
})
|
||||
|
||||
it('discovery, I/O, and dedup stay CLI-side; the shared seenKeys set dedups', async () => {
|
||||
const dbPath = seedDb()
|
||||
const seen = new Set<string>()
|
||||
const first = await collectCalls(dbPath, seen)
|
||||
const second = await collectCalls(dbPath, seen)
|
||||
expect(first.length).toBe(3)
|
||||
expect(second).toEqual([])
|
||||
})
|
||||
|
||||
it('still prints the pre-migration aggregate stderr line for unreadable threads', async () => {
|
||||
const dbPath = seedDb()
|
||||
const written: string[] = []
|
||||
const original = process.stderr.write.bind(process.stderr)
|
||||
;(process.stderr as { write: unknown }).write = (chunk: unknown) => { written.push(String(chunk)); return true }
|
||||
try {
|
||||
await collectCalls(dbPath)
|
||||
} finally {
|
||||
;(process.stderr as { write: unknown }).write = original
|
||||
}
|
||||
// bad-type (unknown data_type) + bad-blob (zstd decompress throws) = 2.
|
||||
expect(written).toEqual(['codeburn: skipped 2 unreadable Zed threads\n'])
|
||||
})
|
||||
})
|
||||
|
|
@ -98,6 +98,26 @@
|
|||
"./providers/pi": {
|
||||
"types": "./dist/providers/pi/index.d.ts",
|
||||
"import": "./dist/providers/pi/index.js"
|
||||
},
|
||||
"./providers/crush": {
|
||||
"types": "./dist/providers/crush/index.d.ts",
|
||||
"import": "./dist/providers/crush/index.js"
|
||||
},
|
||||
"./providers/zcode": {
|
||||
"types": "./dist/providers/zcode/index.d.ts",
|
||||
"import": "./dist/providers/zcode/index.js"
|
||||
},
|
||||
"./providers/zed": {
|
||||
"types": "./dist/providers/zed/index.d.ts",
|
||||
"import": "./dist/providers/zed/index.js"
|
||||
},
|
||||
"./providers/forge": {
|
||||
"types": "./dist/providers/forge/index.d.ts",
|
||||
"import": "./dist/providers/forge/index.js"
|
||||
},
|
||||
"./providers/goose": {
|
||||
"types": "./dist/providers/goose/index.d.ts",
|
||||
"import": "./dist/providers/goose/index.js"
|
||||
}
|
||||
},
|
||||
"files": [
|
||||
|
|
|
|||
73
packages/core/src/providers/crush/decode.ts
Normal file
73
packages/core/src/providers/crush/decode.ts
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
// @codeburn/core Crush decoder: pure decode over host-supplied session records.
|
||||
// The host runs the sqlite queries (session row + dominant model) and hands
|
||||
// this decoder one combined record per session; no fs/env/sqlite/pricing here.
|
||||
// Crush already stores cost in dollars, so a row with cost > 0 carries
|
||||
// `measuredCostUSD` instead of estimated token pricing (Phase 0 pattern B).
|
||||
|
||||
import type { DecodeContext } from '../../contracts.js'
|
||||
import type { RecordDiagnostic } from '../../diagnostics.js'
|
||||
import type { CrushDecodedCall, CrushRawRecord } from './types.js'
|
||||
|
||||
function epochSecondsToIso(epochSeconds: number | null): string {
|
||||
if (epochSeconds === null || !Number.isFinite(epochSeconds)) {
|
||||
return new Date(0).toISOString()
|
||||
}
|
||||
return new Date(epochSeconds * 1000).toISOString()
|
||||
}
|
||||
|
||||
export type CrushDecodeInput = {
|
||||
records: unknown[]
|
||||
context: DecodeContext
|
||||
// Optional live dedup set the host mutates in place. Threaded exactly like
|
||||
// codex/qwen; Crush never persists resume state.
|
||||
seenKeys?: Set<string>
|
||||
}
|
||||
|
||||
export type CrushDecodeResult = {
|
||||
calls: CrushDecodedCall[]
|
||||
diagnostics: RecordDiagnostic[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode Crush session records (one per source) into rich, cost-free-or-measured
|
||||
* calls. A session with zero tokens and zero cost is skipped; dedup is keyed on
|
||||
* `crush:<sessionId>` against the live `seenKeys` set (host-owned).
|
||||
*/
|
||||
export function decodeCrush({ records, seenKeys: liveSeen }: CrushDecodeInput): CrushDecodeResult {
|
||||
const seen = liveSeen ?? new Set<string>()
|
||||
const calls: CrushDecodedCall[] = []
|
||||
|
||||
for (const raw of records) {
|
||||
const session = raw as CrushRawRecord
|
||||
|
||||
const inputTokens = session.prompt_tokens ?? 0
|
||||
const outputTokens = session.completion_tokens ?? 0
|
||||
const cost = session.cost ?? 0
|
||||
if (inputTokens === 0 && outputTokens === 0 && cost === 0) continue
|
||||
|
||||
const dedupKey = `crush:${session.id}`
|
||||
if (seen.has(dedupKey)) continue
|
||||
seen.add(dedupKey)
|
||||
|
||||
calls.push({
|
||||
provider: 'crush',
|
||||
model: session.model,
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
...(cost > 0 ? { measuredCostUSD: cost } : {}),
|
||||
tools: [],
|
||||
rawBashCommands: [],
|
||||
timestamp: epochSecondsToIso(session.updated_at ?? session.created_at),
|
||||
speed: 'standard',
|
||||
deduplicationKey: dedupKey,
|
||||
sessionId: session.id,
|
||||
})
|
||||
}
|
||||
|
||||
return { calls, diagnostics: [] }
|
||||
}
|
||||
26
packages/core/src/providers/crush/index.ts
Normal file
26
packages/core/src/providers/crush/index.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
// @codeburn/core Crush provider.
|
||||
//
|
||||
// Two layers:
|
||||
// - Rich pure decode (`decodeCrush`): host-facing, NOT part of the stable
|
||||
// minimized surface. Pure over host-supplied session records (the sqlite
|
||||
// driver and SQL queries stay CLI-side, per the Category B recipe).
|
||||
// - Minimizing transform (`toObservations`): maps the rich decode into the
|
||||
// strict observation envelope; the content-smuggling guarantees bind here.
|
||||
|
||||
export {
|
||||
decodeCrush,
|
||||
type CrushDecodeInput,
|
||||
type CrushDecodeResult,
|
||||
} from './decode.js'
|
||||
|
||||
export {
|
||||
toObservations,
|
||||
type RichCrushSessionDecode,
|
||||
type CrushToObservationsContext,
|
||||
} from './observations.js'
|
||||
|
||||
export type {
|
||||
CrushDecodedCall,
|
||||
CrushRawRecord,
|
||||
CrushSessionRow,
|
||||
} from './types.js'
|
||||
83
packages/core/src/providers/crush/observations.ts
Normal file
83
packages/core/src/providers/crush/observations.ts
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
// Minimizing transform: rich Crush decode -> the strict observation envelope.
|
||||
// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool
|
||||
// names cross into the output. Crush never captures a user message or file
|
||||
// paths, so there is nothing else to fingerprint or drop here.
|
||||
|
||||
import { projectRef, sessionRef } from '../../fingerprint.js'
|
||||
import type { RecordDiagnostic } from '../../diagnostics.js'
|
||||
import type { CallObservation, SessionObservation } from '../../observations.js'
|
||||
import type { CrushDecodedCall } from './types.js'
|
||||
|
||||
/** One Crush session's rich decode, as the host holds it before minimization. */
|
||||
export interface RichCrushSessionDecode {
|
||||
sessionId: string
|
||||
/** Absolute project path; fingerprinted, never emitted raw. */
|
||||
projectPath: string
|
||||
/** Rich calls in decode order (Crush emits one per session). */
|
||||
calls: CrushDecodedCall[]
|
||||
}
|
||||
|
||||
export interface CrushToObservationsContext {
|
||||
/** HMAC key that scopes every fingerprint. */
|
||||
privacyKey: string
|
||||
/** Provider id stamped onto sessions/calls and folded into sessionRef. */
|
||||
provider?: string
|
||||
}
|
||||
|
||||
const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/
|
||||
|
||||
function toCallObservation(call: CrushDecodedCall, turnIndex: number): CallObservation {
|
||||
const measured = call.measuredCostUSD !== undefined
|
||||
return {
|
||||
provider: call.provider,
|
||||
model: call.model,
|
||||
tokens: {
|
||||
input: call.inputTokens,
|
||||
output: call.outputTokens,
|
||||
reasoning: call.reasoningTokens,
|
||||
cacheRead: call.cacheReadInputTokens,
|
||||
cacheCreate: call.cacheCreationInputTokens,
|
||||
},
|
||||
webSearchRequests: call.webSearchRequests,
|
||||
speed: call.speed,
|
||||
costBasis: measured ? 'measured' : 'estimated',
|
||||
...(measured ? { measuredCostUSD: call.measuredCostUSD } : {}),
|
||||
timestamp: call.timestamp,
|
||||
dedupKey: call.deduplicationKey,
|
||||
toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)),
|
||||
turnIndex,
|
||||
}
|
||||
}
|
||||
|
||||
function toSessionObservation(decode: RichCrushSessionDecode, ctx: CrushToObservationsContext): SessionObservation {
|
||||
const provider = ctx.provider ?? 'crush'
|
||||
const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i))
|
||||
|
||||
const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort()
|
||||
const startedAt = timestamps[0] ?? ''
|
||||
const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : ''
|
||||
|
||||
const session: SessionObservation = {
|
||||
sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId),
|
||||
projectRef: projectRef(ctx.privacyKey, decode.projectPath),
|
||||
providerId: provider,
|
||||
startedAt,
|
||||
...(endedAt ? { endedAt } : {}),
|
||||
calls,
|
||||
turnCount: calls.length,
|
||||
}
|
||||
return session
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a rich Crush decode (one or many sessions) into the minimized observation
|
||||
* layer. Returns the `sessions` array plus any per-record `diagnostics`.
|
||||
*/
|
||||
export function toObservations(
|
||||
decode: RichCrushSessionDecode | RichCrushSessionDecode[],
|
||||
ctx: CrushToObservationsContext,
|
||||
): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } {
|
||||
const decodes = Array.isArray(decode) ? decode : [decode]
|
||||
const sessions = decodes.map(d => toSessionObservation(d, ctx))
|
||||
return { sessions, diagnostics: [] }
|
||||
}
|
||||
45
packages/core/src/providers/crush/types.ts
Normal file
45
packages/core/src/providers/crush/types.ts
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
// Raw record + rich-decode types for the Crush provider.
|
||||
//
|
||||
// Crush stores per-project SQLite databases; the sqlite driver and SQL queries
|
||||
// stay CLI-side (Category B). The host runs both the session-row query and the
|
||||
// dominant-model query (a GROUP BY over messages) and hands the decoder one
|
||||
// combined record per session — this decoder performs no further DB access.
|
||||
|
||||
/** The `sessions` row Crush records, moved verbatim from the CLI. */
|
||||
export type CrushSessionRow = {
|
||||
id: string
|
||||
prompt_tokens: number | null
|
||||
completion_tokens: number | null
|
||||
cost: number | null
|
||||
created_at: number | null
|
||||
updated_at: number | null
|
||||
message_count: number | null
|
||||
}
|
||||
|
||||
/**
|
||||
* The record the host hands to the core decoder: the session row plus the
|
||||
* dominant model resolved by the host's second query (unchanged from the
|
||||
* pre-migration decode, which ran `dominantModel()` inline).
|
||||
*/
|
||||
export type CrushRawRecord = CrushSessionRow & { model: string }
|
||||
|
||||
/** The rich decode of one Crush session, pre-pricing. */
|
||||
export type CrushDecodedCall = {
|
||||
provider: 'crush'
|
||||
model: string
|
||||
inputTokens: number
|
||||
outputTokens: number
|
||||
cacheCreationInputTokens: number
|
||||
cacheReadInputTokens: number
|
||||
cachedInputTokens: number
|
||||
reasoningTokens: number
|
||||
webSearchRequests: number
|
||||
/** Crush already records cost in dollars; present only when the row's cost > 0. */
|
||||
measuredCostUSD?: number
|
||||
tools: string[]
|
||||
rawBashCommands: string[]
|
||||
timestamp: string
|
||||
speed: 'standard'
|
||||
deduplicationKey: string
|
||||
sessionId: string
|
||||
}
|
||||
188
packages/core/src/providers/forge/decode.ts
Normal file
188
packages/core/src/providers/forge/decode.ts
Normal file
|
|
@ -0,0 +1,188 @@
|
|||
// @codeburn/core Forge decoder: pure decode over a host-supplied conversation
|
||||
// row (the sqlite driver + query stay CLI-side; the `context` JSON blob is
|
||||
// still a serialized string when it reaches this decoder). No fs/env/sqlite/
|
||||
// pricing/strip-ansi here — bash base-name extraction stays host-side, this
|
||||
// decoder emits raw command strings only.
|
||||
|
||||
import type { DecodeContext } from '../../contracts.js'
|
||||
import type { RecordDiagnostic } from '../../diagnostics.js'
|
||||
import type { ForgeConversationRow, ForgeContextMessage, ForgeDecodedCall } from './types.js'
|
||||
|
||||
function sqliteTimestampToIso(value: string | null | undefined): string {
|
||||
if (!value) return new Date(0).toISOString()
|
||||
|
||||
const match = value.match(/^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})(?:\.(\d+))?$/)
|
||||
if (match) {
|
||||
const ms = (match[3] ?? '').padEnd(3, '0').slice(0, 3)
|
||||
const parsed = new Date(`${match[1]}T${match[2]}.${ms}Z`)
|
||||
if (!Number.isNaN(parsed.getTime())) return parsed.toISOString()
|
||||
}
|
||||
|
||||
const parsed = new Date(value)
|
||||
return Number.isNaN(parsed.getTime()) ? new Date(0).toISOString() : parsed.toISOString()
|
||||
}
|
||||
|
||||
function actual(value: unknown): number {
|
||||
if (!value || typeof value !== 'object') return 0
|
||||
const raw = (value as Record<string, unknown>)['actual']
|
||||
return typeof raw === 'number' && Number.isFinite(raw) ? raw : 0
|
||||
}
|
||||
|
||||
function usageActual(usage: unknown, key: string): number {
|
||||
if (!usage || typeof usage !== 'object') return 0
|
||||
return actual((usage as Record<string, unknown>)[key])
|
||||
}
|
||||
|
||||
// Kept as a `switch` (not a lookup object) and not exported: tool names come
|
||||
// straight from the conversation JSON, so an object-literal map would resolve
|
||||
// inherited `Object.prototype` members ("constructor", "toString", "__proto__")
|
||||
// to non-string values instead of falling through to the identity default. The
|
||||
// pre-migration CLI's `toolDisplayName` was an identity function anyway (raw
|
||||
// tool ids never surfaced; `tools` already carries the mapped canonical name
|
||||
// from this decode), so this has no consumer outside the extraction below.
|
||||
function mapToolName(name: string): string {
|
||||
switch (name) {
|
||||
case 'shell':
|
||||
case 'bash':
|
||||
return 'Bash'
|
||||
case 'read':
|
||||
case 'Read':
|
||||
return 'Read'
|
||||
case 'write':
|
||||
case 'Write':
|
||||
return 'Write'
|
||||
case 'patch':
|
||||
case 'Edit':
|
||||
case 'edit':
|
||||
return 'Edit'
|
||||
case 'fs_search':
|
||||
case 'grep':
|
||||
return 'Grep'
|
||||
case 'task':
|
||||
case 'dispatch_agent':
|
||||
return 'Agent'
|
||||
default:
|
||||
return name
|
||||
}
|
||||
}
|
||||
|
||||
function pushUnique(values: string[], value: string): void {
|
||||
if (!values.includes(value)) values.push(value)
|
||||
}
|
||||
|
||||
function toolCallsOf(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.filter(v => v && typeof v === 'object') as Record<string, unknown>[] : []
|
||||
}
|
||||
|
||||
function extractToolsAndCommands(calls: Record<string, unknown>[]): { tools: string[]; rawBashCommands: string[]; firstCallId?: string } {
|
||||
const tools: string[] = []
|
||||
const rawBashCommands: string[] = []
|
||||
let firstCallId: string | undefined
|
||||
|
||||
for (const call of calls) {
|
||||
const rawName = call['name']
|
||||
if (typeof rawName !== 'string') continue
|
||||
if (!firstCallId && typeof call['call_id'] === 'string') firstCallId = call['call_id']
|
||||
|
||||
const tool = mapToolName(rawName)
|
||||
pushUnique(tools, tool)
|
||||
|
||||
if (tool === 'Bash') {
|
||||
const args = call['arguments']
|
||||
if (args && typeof args === 'object') {
|
||||
const command = (args as Record<string, unknown>)['command']
|
||||
if (typeof command === 'string') {
|
||||
rawBashCommands.push(command)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { tools, rawBashCommands, firstCallId }
|
||||
}
|
||||
|
||||
export type ForgeDecodeInput = {
|
||||
records: unknown[]
|
||||
context: DecodeContext
|
||||
seenKeys?: Set<string>
|
||||
}
|
||||
|
||||
export type ForgeDecodeResult = {
|
||||
calls: ForgeDecodedCall[]
|
||||
diagnostics: RecordDiagnostic[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode one Forge `conversations` row into rich, cost-free calls, one per
|
||||
* assistant message that carries token usage. The nearest previous user
|
||||
* message in the same conversation is attributed as `userMessage`. Dedup keys
|
||||
* on the tool call's `call_id` when present, else a positional fallback,
|
||||
* against the live `seenKeys` set (host-owned).
|
||||
*/
|
||||
export function decodeForge({ records, seenKeys: liveSeen }: ForgeDecodeInput): ForgeDecodeResult {
|
||||
const seen = liveSeen ?? new Set<string>()
|
||||
const calls: ForgeDecodedCall[] = []
|
||||
|
||||
for (const raw of records) {
|
||||
const row = raw as ForgeConversationRow
|
||||
if (!row.context) continue
|
||||
|
||||
let parsed: unknown
|
||||
try {
|
||||
parsed = JSON.parse(row.context)
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
const messages = Array.isArray((parsed as { messages?: unknown }).messages)
|
||||
? (parsed as { messages: ForgeContextMessage[] }).messages
|
||||
: []
|
||||
|
||||
let userMessage = ''
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
const text = messages[i]?.message?.text
|
||||
const role = typeof text?.role === 'string' ? text.role.toLowerCase() : ''
|
||||
const content = typeof text?.content === 'string' ? text.content : ''
|
||||
|
||||
if (role === 'user') {
|
||||
userMessage = content.length > 500 ? content.slice(0, 500) : content
|
||||
continue
|
||||
}
|
||||
if (role !== 'assistant') continue
|
||||
|
||||
const promptTokens = usageActual(messages[i]?.usage, 'prompt_tokens')
|
||||
const outputTokens = usageActual(messages[i]?.usage, 'completion_tokens')
|
||||
const cachedInputTokens = usageActual(messages[i]?.usage, 'cached_tokens')
|
||||
const inputTokens = Math.max(0, promptTokens - cachedInputTokens)
|
||||
if (inputTokens === 0 && outputTokens === 0) continue
|
||||
|
||||
const model = typeof text?.model === 'string' ? text.model : 'unknown'
|
||||
const toolCalls = toolCallsOf(text?.tool_calls)
|
||||
const { tools, rawBashCommands, firstCallId } = extractToolsAndCommands(toolCalls)
|
||||
const stableId = firstCallId ?? `${model}:${promptTokens}:${outputTokens}:${i}`
|
||||
const deduplicationKey = `forge:${row.conversation_id}:${stableId}`
|
||||
if (seen.has(deduplicationKey)) continue
|
||||
seen.add(deduplicationKey)
|
||||
|
||||
calls.push({
|
||||
provider: 'forge',
|
||||
model,
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: cachedInputTokens,
|
||||
cachedInputTokens,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
tools,
|
||||
rawBashCommands,
|
||||
timestamp: sqliteTimestampToIso(row.updated_at ?? row.created_at),
|
||||
speed: 'standard',
|
||||
deduplicationKey,
|
||||
userMessage,
|
||||
sessionId: row.conversation_id,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return { calls, diagnostics: [] }
|
||||
}
|
||||
27
packages/core/src/providers/forge/index.ts
Normal file
27
packages/core/src/providers/forge/index.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
// @codeburn/core Forge provider.
|
||||
//
|
||||
// Two layers:
|
||||
// - Rich pure decode (`decodeForge`): host-facing, NOT part of the stable
|
||||
// minimized surface. Pure over the host-supplied `conversations` row (the
|
||||
// sqlite driver and SQL query stay CLI-side, Category B); parses the
|
||||
// `context` JSON blob itself.
|
||||
// - Minimizing transform (`toObservations`): maps the rich decode into the
|
||||
// strict observation envelope; the content-smuggling guarantees bind here.
|
||||
|
||||
export {
|
||||
decodeForge,
|
||||
type ForgeDecodeInput,
|
||||
type ForgeDecodeResult,
|
||||
} from './decode.js'
|
||||
|
||||
export {
|
||||
toObservations,
|
||||
type RichForgeSessionDecode,
|
||||
type ForgeToObservationsContext,
|
||||
} from './observations.js'
|
||||
|
||||
export type {
|
||||
ForgeConversationRow,
|
||||
ForgeContextMessage,
|
||||
ForgeDecodedCall,
|
||||
} from './types.js'
|
||||
83
packages/core/src/providers/forge/observations.ts
Normal file
83
packages/core/src/providers/forge/observations.ts
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
// Minimizing transform: rich Forge decode -> the strict observation envelope.
|
||||
// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool
|
||||
// names cross into the output — never the user message or a bash command line.
|
||||
// Forge never captures file targets for its tool calls, so there is nothing to
|
||||
// fingerprint into resourceReads/resourceEdits.
|
||||
|
||||
import { projectRef, sessionRef } from '../../fingerprint.js'
|
||||
import type { RecordDiagnostic } from '../../diagnostics.js'
|
||||
import type { CallObservation, SessionObservation } from '../../observations.js'
|
||||
import type { ForgeDecodedCall } from './types.js'
|
||||
|
||||
/** One Forge conversation's rich decode, as the host holds it before minimization. */
|
||||
export interface RichForgeSessionDecode {
|
||||
sessionId: string
|
||||
/** Absolute project path; fingerprinted, never emitted raw. */
|
||||
projectPath: string
|
||||
/** Rich calls in decode order (one per assistant message with usage). */
|
||||
calls: ForgeDecodedCall[]
|
||||
}
|
||||
|
||||
export interface ForgeToObservationsContext {
|
||||
/** HMAC key that scopes every fingerprint. */
|
||||
privacyKey: string
|
||||
/** Provider id stamped onto sessions/calls and folded into sessionRef. */
|
||||
provider?: string
|
||||
}
|
||||
|
||||
const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/
|
||||
|
||||
function toCallObservation(call: ForgeDecodedCall, turnIndex: number): CallObservation {
|
||||
return {
|
||||
provider: call.provider,
|
||||
model: call.model,
|
||||
tokens: {
|
||||
input: call.inputTokens,
|
||||
output: call.outputTokens,
|
||||
reasoning: call.reasoningTokens,
|
||||
cacheRead: call.cacheReadInputTokens,
|
||||
cacheCreate: call.cacheCreationInputTokens,
|
||||
},
|
||||
webSearchRequests: call.webSearchRequests,
|
||||
speed: call.speed,
|
||||
costBasis: 'estimated',
|
||||
timestamp: call.timestamp,
|
||||
dedupKey: call.deduplicationKey,
|
||||
toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)),
|
||||
turnIndex,
|
||||
}
|
||||
}
|
||||
|
||||
function toSessionObservation(decode: RichForgeSessionDecode, ctx: ForgeToObservationsContext): SessionObservation {
|
||||
const provider = ctx.provider ?? 'forge'
|
||||
const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i))
|
||||
|
||||
const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort()
|
||||
const startedAt = timestamps[0] ?? ''
|
||||
const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : ''
|
||||
|
||||
const session: SessionObservation = {
|
||||
sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId),
|
||||
projectRef: projectRef(ctx.privacyKey, decode.projectPath),
|
||||
providerId: provider,
|
||||
startedAt,
|
||||
...(endedAt ? { endedAt } : {}),
|
||||
calls,
|
||||
turnCount: calls.length,
|
||||
}
|
||||
return session
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a rich Forge decode (one or many conversations) into the minimized
|
||||
* observation layer. Returns the `sessions` array plus any per-record
|
||||
* `diagnostics`.
|
||||
*/
|
||||
export function toObservations(
|
||||
decode: RichForgeSessionDecode | RichForgeSessionDecode[],
|
||||
ctx: ForgeToObservationsContext,
|
||||
): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } {
|
||||
const decodes = Array.isArray(decode) ? decode : [decode]
|
||||
const sessions = decodes.map(d => toSessionObservation(d, ctx))
|
||||
return { sessions, diagnostics: [] }
|
||||
}
|
||||
50
packages/core/src/providers/forge/types.ts
Normal file
50
packages/core/src/providers/forge/types.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
// Raw record + rich-decode types for the Forge provider.
|
||||
//
|
||||
// Forge stores one row per conversation in a single SQLite database; the
|
||||
// `context` column is a JSON blob carrying the full message list (including
|
||||
// per-message token usage and tool calls). The sqlite driver and SQL query stay
|
||||
// CLI-side (Category B); the host hands this decoder the raw conversation row
|
||||
// (with the still-serialized `context` string) — JSON parsing and per-message
|
||||
// decode are pure and happen here.
|
||||
|
||||
/** The `conversations` row Forge records, moved verbatim from the CLI. */
|
||||
export type ForgeConversationRow = {
|
||||
conversation_id: string
|
||||
title: string | null
|
||||
workspace_id: number | string
|
||||
context: string | null
|
||||
created_at: string | null
|
||||
updated_at: string | null
|
||||
}
|
||||
|
||||
export type ForgeContextMessage = {
|
||||
message?: {
|
||||
text?: {
|
||||
role?: unknown
|
||||
content?: unknown
|
||||
model?: unknown
|
||||
tool_calls?: unknown
|
||||
}
|
||||
}
|
||||
usage?: unknown
|
||||
}
|
||||
|
||||
/** The rich decode of one Forge assistant message, pre-pricing. */
|
||||
export type ForgeDecodedCall = {
|
||||
provider: 'forge'
|
||||
model: string
|
||||
inputTokens: number
|
||||
outputTokens: number
|
||||
cacheCreationInputTokens: number
|
||||
cacheReadInputTokens: number
|
||||
cachedInputTokens: number
|
||||
reasoningTokens: number
|
||||
webSearchRequests: number
|
||||
tools: string[]
|
||||
rawBashCommands: string[]
|
||||
timestamp: string
|
||||
speed: 'standard'
|
||||
deduplicationKey: string
|
||||
userMessage: string
|
||||
sessionId: string
|
||||
}
|
||||
171
packages/core/src/providers/goose/decode.ts
Normal file
171
packages/core/src/providers/goose/decode.ts
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
// @codeburn/core Goose decoder: pure decode over a host-supplied composite
|
||||
// session record (session row + assistant tool-message rows + first user
|
||||
// message row, all BLOB columns already converted to text host-side). No
|
||||
// fs/env/sqlite/pricing/clock here — the pre-migration decode fell back to
|
||||
// `new Date()` when both timestamp columns were unparseable; that clock read
|
||||
// stays host-side (the CLI's `toProviderCall` applies it), so this decoder
|
||||
// emits an empty timestamp in that case instead.
|
||||
|
||||
import type { DecodeContext } from '../../contracts.js'
|
||||
import type { RecordDiagnostic } from '../../diagnostics.js'
|
||||
import type {
|
||||
GooseContentItem,
|
||||
GooseDecodedCall,
|
||||
GooseMessageRow,
|
||||
GooseModelConfig,
|
||||
GooseSessionRecords,
|
||||
GooseToolCall,
|
||||
} from './types.js'
|
||||
|
||||
export const gooseToolNameMap: Record<string, string> = {
|
||||
developer__shell: 'Bash',
|
||||
developer__text_editor: 'Edit',
|
||||
developer__read_file: 'Read',
|
||||
developer__write_file: 'Write',
|
||||
developer__list_directory: 'LS',
|
||||
developer__search_files: 'Grep',
|
||||
computercontroller__shell: 'Bash',
|
||||
}
|
||||
|
||||
function parseModelConfig(raw: string | null): GooseModelConfig {
|
||||
if (!raw) return {}
|
||||
try {
|
||||
return JSON.parse(raw) as GooseModelConfig
|
||||
} catch {
|
||||
return {}
|
||||
}
|
||||
}
|
||||
|
||||
function extractToolsFromMessages(messages: GooseMessageRow[]): { tools: string[]; rawBashCommands: string[]; toolSequence: GooseToolCall[][] } {
|
||||
const tools: string[] = []
|
||||
const rawBashCommands: string[] = []
|
||||
const seen = new Set<string>()
|
||||
const toolSequence: GooseToolCall[][] = []
|
||||
|
||||
for (const row of messages) {
|
||||
let items: GooseContentItem[]
|
||||
try {
|
||||
items = JSON.parse(row.contentJson) as GooseContentItem[]
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
const msgCalls: GooseToolCall[] = []
|
||||
for (const item of items) {
|
||||
if (item.type !== 'toolRequest') continue
|
||||
const rawName = item.toolCall?.value?.name ?? ''
|
||||
if (!rawName) continue
|
||||
const mapped = gooseToolNameMap[rawName] ?? rawName.split('__').pop() ?? rawName
|
||||
if (!seen.has(mapped)) {
|
||||
seen.add(mapped)
|
||||
tools.push(mapped)
|
||||
}
|
||||
const call: GooseToolCall = { tool: mapped }
|
||||
const args = item.toolCall?.value?.arguments
|
||||
if (args && typeof args === 'object') {
|
||||
const fp = (args as Record<string, unknown>)['file_path']
|
||||
if (typeof fp === 'string') call.file = fp
|
||||
const cmd = (args as Record<string, unknown>)['command']
|
||||
if (typeof cmd === 'string') call.command = cmd
|
||||
}
|
||||
msgCalls.push(call)
|
||||
if (mapped === 'Bash') {
|
||||
const cmd = args && typeof args === 'object' ? (args as Record<string, unknown>)['command'] : undefined
|
||||
if (typeof cmd === 'string') rawBashCommands.push(cmd)
|
||||
}
|
||||
}
|
||||
if (msgCalls.length > 0) toolSequence.push(msgCalls)
|
||||
}
|
||||
|
||||
return { tools, rawBashCommands, toolSequence }
|
||||
}
|
||||
|
||||
function getFirstUserMessage(row: GooseMessageRow | null): string {
|
||||
if (!row) return ''
|
||||
try {
|
||||
const items = JSON.parse(row.contentJson) as GooseContentItem[]
|
||||
const text = items.find(i => i.type === 'text') as { text?: string } | undefined
|
||||
return (text?.text ?? '').slice(0, 500)
|
||||
} catch {
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve updated_at/created_at into an ISO timestamp, exactly as the
|
||||
// pre-migration decode did, EXCEPT the final `new Date()` (current time)
|
||||
// fallback: that clock read is not deterministic and must stay host-side, so
|
||||
// this returns '' when neither column parses.
|
||||
function resolveTimestamp(updatedAt: string | null, createdAt: string | null): string {
|
||||
const raw = updatedAt || createdAt || ''
|
||||
let ts = new Date(raw)
|
||||
if (isNaN(ts.getTime())) ts = new Date(raw + 'Z')
|
||||
if (isNaN(ts.getTime())) return ''
|
||||
return ts.toISOString()
|
||||
}
|
||||
|
||||
function isGooseSessionRecords(value: unknown): value is GooseSessionRecords {
|
||||
return value !== null && typeof value === 'object' && 'session' in (value as object) && 'assistantToolMessages' in (value as object)
|
||||
}
|
||||
|
||||
export type GooseDecodeInput = {
|
||||
records: unknown[]
|
||||
context: DecodeContext
|
||||
seenKeys?: Set<string>
|
||||
}
|
||||
|
||||
export type GooseDecodeResult = {
|
||||
calls: GooseDecodedCall[]
|
||||
diagnostics: RecordDiagnostic[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode one Goose session's composite record into a single rich, cost-free
|
||||
* call. A session with zero accumulated tokens is skipped; dedup is keyed on
|
||||
* `goose:<sessionId>` against the live `seenKeys` set (host-owned).
|
||||
* `toolSequence` is included only when it has MORE THAN ONE turn (matches the
|
||||
* pre-migration `toolSequence.length > 1 ? toolSequence : undefined` exactly).
|
||||
*/
|
||||
export function decodeGoose({ records, seenKeys: liveSeen }: GooseDecodeInput): GooseDecodeResult {
|
||||
const seen = liveSeen ?? new Set<string>()
|
||||
const calls: GooseDecodedCall[] = []
|
||||
|
||||
const session = records.find(isGooseSessionRecords)
|
||||
if (!session) return { calls, diagnostics: [] }
|
||||
|
||||
const { sessionId, session: sessionRow, assistantToolMessages, firstUserMessage } = session
|
||||
|
||||
const inputTokens = sessionRow.accumulatedInputTokens ?? 0
|
||||
const outputTokens = sessionRow.accumulatedOutputTokens ?? 0
|
||||
if (inputTokens === 0 && outputTokens === 0) return { calls, diagnostics: [] }
|
||||
|
||||
const dedupKey = `goose:${sessionId}`
|
||||
if (seen.has(dedupKey)) return { calls, diagnostics: [] }
|
||||
seen.add(dedupKey)
|
||||
|
||||
const config = parseModelConfig(sessionRow.modelConfigJson)
|
||||
const model = config.model_name ?? 'unknown'
|
||||
|
||||
const { tools, rawBashCommands, toolSequence } = extractToolsFromMessages(assistantToolMessages)
|
||||
const userMessage = getFirstUserMessage(firstUserMessage)
|
||||
|
||||
calls.push({
|
||||
provider: 'goose',
|
||||
model,
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
tools,
|
||||
rawBashCommands,
|
||||
toolSequence: toolSequence.length > 1 ? toolSequence : undefined,
|
||||
timestamp: resolveTimestamp(sessionRow.updatedAt, sessionRow.createdAt),
|
||||
speed: 'standard',
|
||||
deduplicationKey: dedupKey,
|
||||
userMessage,
|
||||
sessionId,
|
||||
})
|
||||
|
||||
return { calls, diagnostics: [] }
|
||||
}
|
||||
31
packages/core/src/providers/goose/index.ts
Normal file
31
packages/core/src/providers/goose/index.ts
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
// @codeburn/core Goose provider.
|
||||
//
|
||||
// Two layers:
|
||||
// - Rich pure decode (`decodeGoose`): host-facing, NOT part of the stable
|
||||
// minimized surface. Pure over the host-supplied session/message row bundle
|
||||
// (the sqlite driver and SQL queries stay CLI-side, Category B).
|
||||
// - Minimizing transform (`toObservations`): maps the rich decode into the
|
||||
// strict observation envelope; the content-smuggling guarantees bind here.
|
||||
|
||||
export {
|
||||
decodeGoose,
|
||||
gooseToolNameMap,
|
||||
type GooseDecodeInput,
|
||||
type GooseDecodeResult,
|
||||
} from './decode.js'
|
||||
|
||||
export {
|
||||
toObservations,
|
||||
type RichGooseSessionDecode,
|
||||
type GooseToObservationsContext,
|
||||
} from './observations.js'
|
||||
|
||||
export type {
|
||||
GooseContentItem,
|
||||
GooseDecodedCall,
|
||||
GooseMessageRow,
|
||||
GooseModelConfig,
|
||||
GooseSessionRecords,
|
||||
GooseSessionRow,
|
||||
GooseToolCall,
|
||||
} from './types.js'
|
||||
83
packages/core/src/providers/goose/observations.ts
Normal file
83
packages/core/src/providers/goose/observations.ts
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
// Minimizing transform: rich Goose decode -> the strict observation envelope.
|
||||
// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool
|
||||
// names cross into the output — never the user message, a bash command line, or
|
||||
// a raw file path. Fingerprint file paths through `extractResourceRefs`.
|
||||
|
||||
import { projectRef, sessionRef } from '../../fingerprint.js'
|
||||
import type { RecordDiagnostic } from '../../diagnostics.js'
|
||||
import type { CallObservation, SessionObservation } from '../../observations.js'
|
||||
import { extractResourceRefs } from '../resource-refs.js'
|
||||
import type { GooseDecodedCall } from './types.js'
|
||||
|
||||
/** One Goose session's rich decode, as the host holds it before minimization. */
|
||||
export interface RichGooseSessionDecode {
|
||||
sessionId: string
|
||||
/** Absolute project path (the session's working_dir); fingerprinted, never emitted raw. */
|
||||
projectPath: string
|
||||
/** Rich calls in decode order (Goose emits one per session). */
|
||||
calls: GooseDecodedCall[]
|
||||
}
|
||||
|
||||
export interface GooseToObservationsContext {
|
||||
/** HMAC key that scopes every fingerprint. */
|
||||
privacyKey: string
|
||||
/** Provider id stamped onto sessions/calls and folded into sessionRef. */
|
||||
provider?: string
|
||||
}
|
||||
|
||||
const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/
|
||||
|
||||
function toCallObservation(call: GooseDecodedCall, turnIndex: number, privacyKey: string): CallObservation {
|
||||
return {
|
||||
provider: call.provider,
|
||||
model: call.model,
|
||||
tokens: {
|
||||
input: call.inputTokens,
|
||||
output: call.outputTokens,
|
||||
reasoning: call.reasoningTokens,
|
||||
cacheRead: call.cacheReadInputTokens,
|
||||
cacheCreate: call.cacheCreationInputTokens,
|
||||
},
|
||||
webSearchRequests: call.webSearchRequests,
|
||||
speed: call.speed,
|
||||
costBasis: 'estimated',
|
||||
timestamp: call.timestamp,
|
||||
dedupKey: call.deduplicationKey,
|
||||
toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)),
|
||||
turnIndex,
|
||||
...extractResourceRefs(privacyKey, call.toolSequence),
|
||||
}
|
||||
}
|
||||
|
||||
function toSessionObservation(decode: RichGooseSessionDecode, ctx: GooseToObservationsContext): SessionObservation {
|
||||
const provider = ctx.provider ?? 'goose'
|
||||
const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i, ctx.privacyKey))
|
||||
|
||||
const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort()
|
||||
const startedAt = timestamps[0] ?? ''
|
||||
const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : ''
|
||||
|
||||
const session: SessionObservation = {
|
||||
sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId),
|
||||
projectRef: projectRef(ctx.privacyKey, decode.projectPath),
|
||||
providerId: provider,
|
||||
startedAt,
|
||||
...(endedAt ? { endedAt } : {}),
|
||||
calls,
|
||||
turnCount: calls.length,
|
||||
}
|
||||
return session
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a rich Goose decode (one or many sessions) into the minimized observation
|
||||
* layer. Returns the `sessions` array plus any per-record `diagnostics`.
|
||||
*/
|
||||
export function toObservations(
|
||||
decode: RichGooseSessionDecode | RichGooseSessionDecode[],
|
||||
ctx: GooseToObservationsContext,
|
||||
): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } {
|
||||
const decodes = Array.isArray(decode) ? decode : [decode]
|
||||
const sessions = decodes.map(d => toSessionObservation(d, ctx))
|
||||
return { sessions, diagnostics: [] }
|
||||
}
|
||||
78
packages/core/src/providers/goose/types.ts
Normal file
78
packages/core/src/providers/goose/types.ts
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
// Raw record + rich-decode types for the Goose provider.
|
||||
//
|
||||
// Goose stores one row per session plus a `messages` table with a per-message
|
||||
// `content_json` BLOB column. The sqlite driver and SQL queries stay CLI-side
|
||||
// (Category B); the host runs the session query, the assistant tool-message
|
||||
// query, and the first-user-message query, converts each BLOB column to text
|
||||
// (`blobToText`, the same charset-safe conversion `fs-utils` performs for file
|
||||
// reads), and hands this decoder one composite record bundling all three —
|
||||
// JSON parsing and per-message decode are pure and happen here.
|
||||
|
||||
/** The `sessions` row Goose records, pre-resolved to text columns by the host. */
|
||||
export type GooseSessionRow = {
|
||||
id: string
|
||||
workingDir: string | null
|
||||
createdAt: string | null
|
||||
updatedAt: string | null
|
||||
accumulatedInputTokens: number | null
|
||||
accumulatedOutputTokens: number | null
|
||||
/** `model_config_json`, already converted from BLOB to text host-side. */
|
||||
modelConfigJson: string | null
|
||||
}
|
||||
|
||||
export type GooseModelConfig = {
|
||||
model_name?: string
|
||||
reasoning?: boolean
|
||||
}
|
||||
|
||||
/** One `messages` row, `content_json` already converted from BLOB to text host-side. */
|
||||
export type GooseMessageRow = {
|
||||
contentJson: string
|
||||
}
|
||||
|
||||
export type GooseContentItem = {
|
||||
type: string
|
||||
text?: string
|
||||
toolCall?: { value?: { name?: string; arguments?: Record<string, unknown> } }
|
||||
}
|
||||
|
||||
/** One tool invocation as captured from a Goose transcript. */
|
||||
export type GooseToolCall = {
|
||||
tool: string
|
||||
file?: string
|
||||
command?: string
|
||||
}
|
||||
|
||||
/** The composite record the host hands to the core decoder for one session. */
|
||||
export type GooseSessionRecords = {
|
||||
sessionId: string
|
||||
session: GooseSessionRow
|
||||
/** Assistant messages whose content_json contains a toolRequest, ordered by created_timestamp ASC. */
|
||||
assistantToolMessages: GooseMessageRow[]
|
||||
/** The first user message (by created_timestamp ASC), if any. */
|
||||
firstUserMessage: GooseMessageRow | null
|
||||
}
|
||||
|
||||
/** The rich decode of one Goose session, pre-pricing. */
|
||||
export type GooseDecodedCall = {
|
||||
provider: 'goose'
|
||||
model: string
|
||||
inputTokens: number
|
||||
outputTokens: number
|
||||
cacheCreationInputTokens: number
|
||||
cacheReadInputTokens: number
|
||||
cachedInputTokens: number
|
||||
reasoningTokens: number
|
||||
webSearchRequests: number
|
||||
tools: string[]
|
||||
rawBashCommands: string[]
|
||||
toolSequence?: GooseToolCall[][]
|
||||
/** Resolved from updated_at/created_at; empty when neither parses (the host
|
||||
* falls back to the current time, matching the pre-migration decode — that
|
||||
* clock read must stay host-side, so this decode never touches it). */
|
||||
timestamp: string
|
||||
speed: 'standard'
|
||||
deduplicationKey: string
|
||||
userMessage: string
|
||||
sessionId: string
|
||||
}
|
||||
103
packages/core/src/providers/zcode/decode.ts
Normal file
103
packages/core/src/providers/zcode/decode.ts
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
// @codeburn/core ZCode decoder: pure decode over a host-supplied composite
|
||||
// session record (model_usage rows + tool_usage rows). No fs/env/sqlite/pricing
|
||||
// here — the host runs both queries and hands the rows straight through.
|
||||
|
||||
import type { DecodeContext } from '../../contracts.js'
|
||||
import type { RecordDiagnostic } from '../../diagnostics.js'
|
||||
import type { ZcodeDecodedCall, ZcodeSessionRecords, ZcodeToolRow, ZcodeUsageRow } from './types.js'
|
||||
|
||||
function epochMsToIso(ms: number | null): string {
|
||||
if (ms === null || !Number.isFinite(ms) || ms <= 0) return new Date(0).toISOString()
|
||||
return new Date(ms).toISOString()
|
||||
}
|
||||
|
||||
function isZcodeSessionRecords(value: unknown): value is ZcodeSessionRecords {
|
||||
return value !== null && typeof value === 'object' && 'usageRows' in (value as object) && 'toolRows' in (value as object)
|
||||
}
|
||||
|
||||
export type ZcodeDecodeInput = {
|
||||
records: unknown[]
|
||||
context: DecodeContext
|
||||
seenKeys?: Set<string>
|
||||
}
|
||||
|
||||
export type ZcodeDecodeResult = {
|
||||
calls: ZcodeDecodedCall[]
|
||||
diagnostics: RecordDiagnostic[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode one ZCode session's composite record (model_usage rows joined with
|
||||
* tool_usage rows by turn) into rich, cost-free calls. model_usage rows don't
|
||||
* link to individual tool calls, only to a turn, so each turn's tools attach to
|
||||
* the FIRST usage row of that turn only, to avoid double-counting across a
|
||||
* turn's multiple requests — matching the pre-migration decode exactly.
|
||||
*/
|
||||
export function decodeZcode({ records, seenKeys: liveSeen }: ZcodeDecodeInput): ZcodeDecodeResult {
|
||||
const seen = liveSeen ?? new Set<string>()
|
||||
const calls: ZcodeDecodedCall[] = []
|
||||
|
||||
const session = records.find(isZcodeSessionRecords)
|
||||
if (!session) return { calls, diagnostics: [] }
|
||||
|
||||
const { sessionId, usageRows, toolRows }: { sessionId: string; usageRows: ZcodeUsageRow[]; toolRows: ZcodeToolRow[] } = session
|
||||
|
||||
const toolsByTurn = new Map<string, string[]>()
|
||||
for (const tool of toolRows) {
|
||||
if (!tool.turn_id) continue
|
||||
const list = toolsByTurn.get(tool.turn_id) ?? []
|
||||
list.push(tool.tool_name)
|
||||
toolsByTurn.set(tool.turn_id, list)
|
||||
}
|
||||
|
||||
const turnsWithToolsEmitted = new Set<string>()
|
||||
|
||||
for (const row of usageRows) {
|
||||
const cacheRead = row.cache_read_input_tokens ?? 0
|
||||
const cacheCreation = row.cache_creation_input_tokens ?? 0
|
||||
const output = row.output_tokens ?? 0
|
||||
const reasoning = row.reasoning_tokens ?? 0
|
||||
// ZCode folds cached tokens into input_tokens (OpenAI-style). Split them
|
||||
// back out so fresh input bills at the input rate and cached at the
|
||||
// cache-read rate, matching the pricing table's Anthropic-style semantics.
|
||||
const freshInput = Math.max(0, (row.input_tokens ?? 0) - cacheRead - cacheCreation)
|
||||
|
||||
if (freshInput === 0 && output === 0 && reasoning === 0 && cacheRead === 0 && cacheCreation === 0) {
|
||||
continue
|
||||
}
|
||||
|
||||
const dedupKey = `zcode:${row.id}`
|
||||
if (seen.has(dedupKey)) continue
|
||||
seen.add(dedupKey)
|
||||
|
||||
let tools: string[] = []
|
||||
if (row.turn_id && !turnsWithToolsEmitted.has(row.turn_id)) {
|
||||
const turnTools = toolsByTurn.get(row.turn_id)
|
||||
if (turnTools && turnTools.length > 0) {
|
||||
tools = turnTools
|
||||
turnsWithToolsEmitted.add(row.turn_id)
|
||||
}
|
||||
}
|
||||
|
||||
calls.push({
|
||||
provider: 'zcode',
|
||||
model: row.model_id,
|
||||
inputTokens: freshInput,
|
||||
outputTokens: output,
|
||||
cacheCreationInputTokens: cacheCreation,
|
||||
cacheReadInputTokens: cacheRead,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: reasoning,
|
||||
webSearchRequests: 0,
|
||||
tools,
|
||||
rawBashCommands: [],
|
||||
timestamp: epochMsToIso(row.completed_at ?? row.started_at),
|
||||
speed: 'standard',
|
||||
deduplicationKey: dedupKey,
|
||||
turnId: row.turn_id ?? undefined,
|
||||
sessionId,
|
||||
})
|
||||
}
|
||||
|
||||
return { calls, diagnostics: [] }
|
||||
}
|
||||
27
packages/core/src/providers/zcode/index.ts
Normal file
27
packages/core/src/providers/zcode/index.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
// @codeburn/core ZCode provider.
|
||||
//
|
||||
// Two layers:
|
||||
// - Rich pure decode (`decodeZcode`): host-facing, NOT part of the stable
|
||||
// minimized surface. Pure over the host-supplied model_usage/tool_usage row
|
||||
// bundle (the sqlite driver and SQL queries stay CLI-side, Category B).
|
||||
// - Minimizing transform (`toObservations`): maps the rich decode into the
|
||||
// strict observation envelope; the content-smuggling guarantees bind here.
|
||||
|
||||
export {
|
||||
decodeZcode,
|
||||
type ZcodeDecodeInput,
|
||||
type ZcodeDecodeResult,
|
||||
} from './decode.js'
|
||||
|
||||
export {
|
||||
toObservations,
|
||||
type RichZcodeSessionDecode,
|
||||
type ZcodeToObservationsContext,
|
||||
} from './observations.js'
|
||||
|
||||
export type {
|
||||
ZcodeDecodedCall,
|
||||
ZcodeSessionRecords,
|
||||
ZcodeToolRow,
|
||||
ZcodeUsageRow,
|
||||
} from './types.js'
|
||||
81
packages/core/src/providers/zcode/observations.ts
Normal file
81
packages/core/src/providers/zcode/observations.ts
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
// Minimizing transform: rich ZCode decode -> the strict observation envelope.
|
||||
// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool
|
||||
// names cross into the output. ZCode never captures a user message or file
|
||||
// paths, so there is nothing else to fingerprint or drop here.
|
||||
|
||||
import { projectRef, sessionRef } from '../../fingerprint.js'
|
||||
import type { RecordDiagnostic } from '../../diagnostics.js'
|
||||
import type { CallObservation, SessionObservation } from '../../observations.js'
|
||||
import type { ZcodeDecodedCall } from './types.js'
|
||||
|
||||
/** One ZCode session's rich decode, as the host holds it before minimization. */
|
||||
export interface RichZcodeSessionDecode {
|
||||
sessionId: string
|
||||
/** Absolute project path; fingerprinted, never emitted raw. */
|
||||
projectPath: string
|
||||
/** Rich calls in decode order (one per model_usage row). */
|
||||
calls: ZcodeDecodedCall[]
|
||||
}
|
||||
|
||||
export interface ZcodeToObservationsContext {
|
||||
/** HMAC key that scopes every fingerprint. */
|
||||
privacyKey: string
|
||||
/** Provider id stamped onto sessions/calls and folded into sessionRef. */
|
||||
provider?: string
|
||||
}
|
||||
|
||||
const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/
|
||||
|
||||
function toCallObservation(call: ZcodeDecodedCall, turnIndex: number): CallObservation {
|
||||
return {
|
||||
provider: call.provider,
|
||||
model: call.model,
|
||||
tokens: {
|
||||
input: call.inputTokens,
|
||||
output: call.outputTokens,
|
||||
reasoning: call.reasoningTokens,
|
||||
cacheRead: call.cacheReadInputTokens,
|
||||
cacheCreate: call.cacheCreationInputTokens,
|
||||
},
|
||||
webSearchRequests: call.webSearchRequests,
|
||||
speed: call.speed,
|
||||
costBasis: 'estimated',
|
||||
timestamp: call.timestamp,
|
||||
dedupKey: call.deduplicationKey,
|
||||
toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)),
|
||||
turnIndex,
|
||||
}
|
||||
}
|
||||
|
||||
function toSessionObservation(decode: RichZcodeSessionDecode, ctx: ZcodeToObservationsContext): SessionObservation {
|
||||
const provider = ctx.provider ?? 'zcode'
|
||||
const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i))
|
||||
|
||||
const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort()
|
||||
const startedAt = timestamps[0] ?? ''
|
||||
const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : ''
|
||||
|
||||
const session: SessionObservation = {
|
||||
sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId),
|
||||
projectRef: projectRef(ctx.privacyKey, decode.projectPath),
|
||||
providerId: provider,
|
||||
startedAt,
|
||||
...(endedAt ? { endedAt } : {}),
|
||||
calls,
|
||||
turnCount: calls.length,
|
||||
}
|
||||
return session
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a rich ZCode decode (one or many sessions) into the minimized observation
|
||||
* layer. Returns the `sessions` array plus any per-record `diagnostics`.
|
||||
*/
|
||||
export function toObservations(
|
||||
decode: RichZcodeSessionDecode | RichZcodeSessionDecode[],
|
||||
ctx: ZcodeToObservationsContext,
|
||||
): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } {
|
||||
const decodes = Array.isArray(decode) ? decode : [decode]
|
||||
const sessions = decodes.map(d => toSessionObservation(d, ctx))
|
||||
return { sessions, diagnostics: [] }
|
||||
}
|
||||
53
packages/core/src/providers/zcode/types.ts
Normal file
53
packages/core/src/providers/zcode/types.ts
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
// Raw record + rich-decode types for the ZCode provider.
|
||||
//
|
||||
// ZCode (CLI v0.14.x) records usage in a single SQLite database. The sqlite
|
||||
// driver and SQL queries stay CLI-side (Category B); the host runs both the
|
||||
// model_usage query and the tool_usage query for one session and hands the
|
||||
// decoder a single composite record bundling both row sets.
|
||||
|
||||
/** One `model_usage` row, moved verbatim from the CLI. */
|
||||
export type ZcodeUsageRow = {
|
||||
id: string
|
||||
turn_id: string | null
|
||||
model_id: string
|
||||
input_tokens: number
|
||||
output_tokens: number
|
||||
reasoning_tokens: number
|
||||
cache_creation_input_tokens: number
|
||||
cache_read_input_tokens: number
|
||||
started_at: number
|
||||
completed_at: number | null
|
||||
}
|
||||
|
||||
/** One `tool_usage` row, moved verbatim from the CLI. */
|
||||
export type ZcodeToolRow = {
|
||||
turn_id: string | null
|
||||
tool_name: string
|
||||
}
|
||||
|
||||
/** The composite record the host hands to the core decoder for one session. */
|
||||
export type ZcodeSessionRecords = {
|
||||
sessionId: string
|
||||
usageRows: ZcodeUsageRow[]
|
||||
toolRows: ZcodeToolRow[]
|
||||
}
|
||||
|
||||
/** The rich decode of one ZCode model_usage row, pre-pricing. */
|
||||
export type ZcodeDecodedCall = {
|
||||
provider: 'zcode'
|
||||
model: string
|
||||
inputTokens: number
|
||||
outputTokens: number
|
||||
cacheCreationInputTokens: number
|
||||
cacheReadInputTokens: number
|
||||
cachedInputTokens: number
|
||||
reasoningTokens: number
|
||||
webSearchRequests: number
|
||||
tools: string[]
|
||||
rawBashCommands: string[]
|
||||
timestamp: string
|
||||
speed: 'standard'
|
||||
deduplicationKey: string
|
||||
turnId?: string
|
||||
sessionId: string
|
||||
}
|
||||
143
packages/core/src/providers/zed/decode.ts
Normal file
143
packages/core/src/providers/zed/decode.ts
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
// @codeburn/core Zed decoder: pure decode over host-supplied `threads` rows.
|
||||
// The host runs the sqlite query and the Node-version zstd-availability gate;
|
||||
// this decoder receives the raw rows (blob and all) and does the decompression,
|
||||
// JSON parsing, and per-request token accounting. No fs/env/sqlite/pricing.
|
||||
//
|
||||
// zstd landed in node:zlib in 22.15 / 23.8. The host only calls this decoder
|
||||
// after confirming `zstdDecompressSync` exists (see the CLI's `readRecords`), so
|
||||
// calling it here is safe; the decoder does not re-check Node version support.
|
||||
|
||||
import zlib from 'node:zlib'
|
||||
|
||||
import type { DecodeContext } from '../../contracts.js'
|
||||
import type { RecordDiagnostic } from '../../diagnostics.js'
|
||||
import type { ZedDecodedCall, ZedThreadJson, ZedThreadRow, ZedTokenUsage } from './types.js'
|
||||
|
||||
const zstdDecompressSync = (zlib as { zstdDecompressSync?: (buf: Buffer) => Buffer }).zstdDecompressSync
|
||||
|
||||
function num(value: number | undefined): number {
|
||||
return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : 0
|
||||
}
|
||||
|
||||
function usageIsEmpty(usage: ZedTokenUsage): boolean {
|
||||
return (
|
||||
num(usage.input_tokens) === 0 &&
|
||||
num(usage.output_tokens) === 0 &&
|
||||
num(usage.cache_creation_input_tokens) === 0 &&
|
||||
num(usage.cache_read_input_tokens) === 0
|
||||
)
|
||||
}
|
||||
|
||||
function buildCall(opts: {
|
||||
threadId: string
|
||||
requestKey: string
|
||||
usage: ZedTokenUsage
|
||||
model: string
|
||||
timestamp: string
|
||||
userMessage: string
|
||||
}): ZedDecodedCall {
|
||||
const input = num(opts.usage.input_tokens)
|
||||
const output = num(opts.usage.output_tokens)
|
||||
const cacheWrite = num(opts.usage.cache_creation_input_tokens)
|
||||
const cacheRead = num(opts.usage.cache_read_input_tokens)
|
||||
return {
|
||||
provider: 'zed',
|
||||
model: opts.model,
|
||||
inputTokens: input,
|
||||
outputTokens: output,
|
||||
cacheCreationInputTokens: cacheWrite,
|
||||
cacheReadInputTokens: cacheRead,
|
||||
cachedInputTokens: cacheRead,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
tools: [],
|
||||
rawBashCommands: [],
|
||||
timestamp: opts.timestamp,
|
||||
speed: 'standard',
|
||||
deduplicationKey: `zed:${opts.threadId}:${opts.requestKey}`,
|
||||
userMessage: opts.userMessage,
|
||||
sessionId: opts.threadId,
|
||||
}
|
||||
}
|
||||
|
||||
export type ZedDecodeInput = {
|
||||
records: unknown[]
|
||||
context: DecodeContext
|
||||
seenKeys?: Set<string>
|
||||
}
|
||||
|
||||
export type ZedDecodeResult = {
|
||||
calls: ZedDecodedCall[]
|
||||
diagnostics: RecordDiagnostic[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode Zed `threads` rows into rich, cost-free calls. The per-request map is
|
||||
* keyed by user message and does not cover every request (verified on a real
|
||||
* thread: cumulative was ~3x the map sum), so a remainder entry tops the thread
|
||||
* up to the exact cumulative counter. Threads with an empty map degrade to one
|
||||
* cumulative call. Dedup is keyed on `zed:<threadId>:<requestKey>` against the
|
||||
* live `seenKeys` set (host-owned).
|
||||
*/
|
||||
export function decodeZed({ records, seenKeys: liveSeen }: ZedDecodeInput): ZedDecodeResult {
|
||||
const seen = liveSeen ?? new Set<string>()
|
||||
const calls: ZedDecodedCall[] = []
|
||||
const diagnostics: RecordDiagnostic[] = []
|
||||
|
||||
records.forEach((raw, index) => {
|
||||
const row = raw as ZedThreadRow
|
||||
try {
|
||||
// Zed's DataType enum is "zstd" (current save path) or "json" (legacy
|
||||
// uncompressed rows); anything else is unknown.
|
||||
if (!row.id || !row.data || (row.data_type !== 'zstd' && row.data_type !== 'json')) {
|
||||
if (row.data != null) diagnostics.push({ index, code: 'unknown-shape' })
|
||||
return
|
||||
}
|
||||
const parsedAt = new Date(row.updated_at ?? '')
|
||||
if (Number.isNaN(parsedAt.getTime())) return
|
||||
const timestamp = parsedAt.toISOString()
|
||||
|
||||
const jsonText = row.data_type === 'zstd'
|
||||
? zstdDecompressSync!(Buffer.from(row.data)).toString('utf-8')
|
||||
: Buffer.from(row.data).toString('utf-8')
|
||||
const thread = JSON.parse(jsonText) as ZedThreadJson
|
||||
const model = thread.model?.model || 'unknown'
|
||||
const userMessage = row.summary ?? ''
|
||||
|
||||
const requests = Object.entries(thread.request_token_usage ?? {}).filter(([, usage]) => usage != null && !usageIsEmpty(usage))
|
||||
// The per-request map is keyed by user message and does not cover every
|
||||
// request, so a remainder entry tops the thread up to the exact
|
||||
// cumulative counter. Threads with an empty map degrade to one
|
||||
// cumulative call.
|
||||
const entries: Array<[string, ZedTokenUsage]> = [...requests]
|
||||
const cumulative = thread.cumulative_token_usage
|
||||
if (cumulative && !usageIsEmpty(cumulative)) {
|
||||
let sumIn = 0, sumOut = 0, sumWrite = 0, sumRead = 0
|
||||
for (const [, usage] of requests) {
|
||||
sumIn += num(usage.input_tokens)
|
||||
sumOut += num(usage.output_tokens)
|
||||
sumWrite += num(usage.cache_creation_input_tokens)
|
||||
sumRead += num(usage.cache_read_input_tokens)
|
||||
}
|
||||
const remainder: ZedTokenUsage = {
|
||||
input_tokens: Math.max(0, num(cumulative.input_tokens) - sumIn),
|
||||
output_tokens: Math.max(0, num(cumulative.output_tokens) - sumOut),
|
||||
cache_creation_input_tokens: Math.max(0, num(cumulative.cache_creation_input_tokens) - sumWrite),
|
||||
cache_read_input_tokens: Math.max(0, num(cumulative.cache_read_input_tokens) - sumRead),
|
||||
}
|
||||
if (!usageIsEmpty(remainder)) entries.push(['cumulative-remainder', remainder])
|
||||
}
|
||||
|
||||
for (const [requestKey, usage] of entries) {
|
||||
const call = buildCall({ threadId: row.id, requestKey, usage, model, timestamp, userMessage })
|
||||
if (seen.has(call.deduplicationKey)) continue
|
||||
seen.add(call.deduplicationKey)
|
||||
calls.push(call)
|
||||
}
|
||||
} catch {
|
||||
diagnostics.push({ index, code: 'malformed-json' })
|
||||
}
|
||||
})
|
||||
|
||||
return { calls, diagnostics }
|
||||
}
|
||||
28
packages/core/src/providers/zed/index.ts
Normal file
28
packages/core/src/providers/zed/index.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
// @codeburn/core Zed provider.
|
||||
//
|
||||
// Two layers:
|
||||
// - Rich pure decode (`decodeZed`): host-facing, NOT part of the stable
|
||||
// minimized surface. Pure over the host-supplied `threads` rows (the sqlite
|
||||
// driver and SQL query stay CLI-side, Category B); decompresses and parses
|
||||
// the zstd/json blob itself.
|
||||
// - Minimizing transform (`toObservations`): maps the rich decode into the
|
||||
// strict observation envelope; the content-smuggling guarantees bind here.
|
||||
|
||||
export {
|
||||
decodeZed,
|
||||
type ZedDecodeInput,
|
||||
type ZedDecodeResult,
|
||||
} from './decode.js'
|
||||
|
||||
export {
|
||||
toObservations,
|
||||
type RichZedSessionDecode,
|
||||
type ZedToObservationsContext,
|
||||
} from './observations.js'
|
||||
|
||||
export type {
|
||||
ZedDecodedCall,
|
||||
ZedThreadJson,
|
||||
ZedThreadRow,
|
||||
ZedTokenUsage,
|
||||
} from './types.js'
|
||||
82
packages/core/src/providers/zed/observations.ts
Normal file
82
packages/core/src/providers/zed/observations.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
// Minimizing transform: rich Zed decode -> the strict observation envelope.
|
||||
// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool
|
||||
// names cross into the output — never the thread summary Zed captures as the
|
||||
// rich decode's user message. Zed never records file paths, so there is
|
||||
// nothing to fingerprint.
|
||||
|
||||
import { projectRef, sessionRef } from '../../fingerprint.js'
|
||||
import type { RecordDiagnostic } from '../../diagnostics.js'
|
||||
import type { CallObservation, SessionObservation } from '../../observations.js'
|
||||
import type { ZedDecodedCall } from './types.js'
|
||||
|
||||
/** One Zed thread's rich decode, as the host holds it before minimization. */
|
||||
export interface RichZedSessionDecode {
|
||||
sessionId: string
|
||||
/** Absolute project path; fingerprinted, never emitted raw. */
|
||||
projectPath: string
|
||||
/** Rich calls in decode order (one per request / cumulative-remainder). */
|
||||
calls: ZedDecodedCall[]
|
||||
}
|
||||
|
||||
export interface ZedToObservationsContext {
|
||||
/** HMAC key that scopes every fingerprint. */
|
||||
privacyKey: string
|
||||
/** Provider id stamped onto sessions/calls and folded into sessionRef. */
|
||||
provider?: string
|
||||
}
|
||||
|
||||
const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/
|
||||
|
||||
function toCallObservation(call: ZedDecodedCall, turnIndex: number): CallObservation {
|
||||
return {
|
||||
provider: call.provider,
|
||||
model: call.model,
|
||||
tokens: {
|
||||
input: call.inputTokens,
|
||||
output: call.outputTokens,
|
||||
reasoning: call.reasoningTokens,
|
||||
cacheRead: call.cacheReadInputTokens,
|
||||
cacheCreate: call.cacheCreationInputTokens,
|
||||
},
|
||||
webSearchRequests: call.webSearchRequests,
|
||||
speed: call.speed,
|
||||
costBasis: 'estimated',
|
||||
timestamp: call.timestamp,
|
||||
dedupKey: call.deduplicationKey,
|
||||
toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)),
|
||||
turnIndex,
|
||||
}
|
||||
}
|
||||
|
||||
function toSessionObservation(decode: RichZedSessionDecode, ctx: ZedToObservationsContext): SessionObservation {
|
||||
const provider = ctx.provider ?? 'zed'
|
||||
const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i))
|
||||
|
||||
const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort()
|
||||
const startedAt = timestamps[0] ?? ''
|
||||
const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : ''
|
||||
|
||||
const session: SessionObservation = {
|
||||
sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId),
|
||||
projectRef: projectRef(ctx.privacyKey, decode.projectPath),
|
||||
providerId: provider,
|
||||
startedAt,
|
||||
...(endedAt ? { endedAt } : {}),
|
||||
calls,
|
||||
turnCount: calls.length,
|
||||
}
|
||||
return session
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a rich Zed decode (one or many threads) into the minimized observation
|
||||
* layer. Returns the `sessions` array plus any per-record `diagnostics`.
|
||||
*/
|
||||
export function toObservations(
|
||||
decode: RichZedSessionDecode | RichZedSessionDecode[],
|
||||
ctx: ZedToObservationsContext,
|
||||
): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } {
|
||||
const decodes = Array.isArray(decode) ? decode : [decode]
|
||||
const sessions = decodes.map(d => toSessionObservation(d, ctx))
|
||||
return { sessions, diagnostics: [] }
|
||||
}
|
||||
50
packages/core/src/providers/zed/types.ts
Normal file
50
packages/core/src/providers/zed/types.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
// Raw record + rich-decode types for the Zed provider.
|
||||
//
|
||||
// Zed's built-in agent stores one row per thread in a single SQLite database;
|
||||
// the `data` blob is zstd-compressed JSON carrying `request_token_usage`
|
||||
// (per-request Anthropic-shaped token counts) and the thread's model. The
|
||||
// sqlite driver and SQL query stay CLI-side (Category B); the host hands this
|
||||
// decoder the raw `threads` rows, blob and all — decompression + JSON parsing
|
||||
// are pure and happen here. Format documented in issue #480.
|
||||
|
||||
/** One `threads` row, moved verbatim from the CLI. */
|
||||
export type ZedThreadRow = {
|
||||
id: string
|
||||
summary: string | null
|
||||
updated_at: string | null
|
||||
data_type: string | null
|
||||
data: Uint8Array | null
|
||||
}
|
||||
|
||||
export type ZedTokenUsage = {
|
||||
input_tokens?: number
|
||||
output_tokens?: number
|
||||
cache_creation_input_tokens?: number
|
||||
cache_read_input_tokens?: number
|
||||
}
|
||||
|
||||
export type ZedThreadJson = {
|
||||
model?: { provider?: string; model?: string }
|
||||
request_token_usage?: Record<string, ZedTokenUsage>
|
||||
cumulative_token_usage?: ZedTokenUsage
|
||||
}
|
||||
|
||||
/** The rich decode of one Zed request (per-request or cumulative-remainder), pre-pricing. */
|
||||
export type ZedDecodedCall = {
|
||||
provider: 'zed'
|
||||
model: string
|
||||
inputTokens: number
|
||||
outputTokens: number
|
||||
cacheCreationInputTokens: number
|
||||
cacheReadInputTokens: number
|
||||
cachedInputTokens: number
|
||||
reasoningTokens: number
|
||||
webSearchRequests: number
|
||||
tools: string[]
|
||||
rawBashCommands: string[]
|
||||
timestamp: string
|
||||
speed: 'standard'
|
||||
deduplicationKey: string
|
||||
userMessage: string
|
||||
sessionId: string
|
||||
}
|
||||
|
|
@ -151,6 +151,12 @@ const USER_MESSAGE_ALLOWLIST = new Set([
|
|||
'src/providers/kimicode/types.ts',
|
||||
'src/providers/pi/decode.ts',
|
||||
'src/providers/pi/types.ts',
|
||||
'src/providers/zed/decode.ts',
|
||||
'src/providers/zed/types.ts',
|
||||
'src/providers/forge/decode.ts',
|
||||
'src/providers/forge/types.ts',
|
||||
'src/providers/goose/decode.ts',
|
||||
'src/providers/goose/types.ts',
|
||||
])
|
||||
|
||||
describe('architecture gate: no classification or free text in @codeburn/core source', () => {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { readFileSync } from 'node:fs'
|
||||
import { dirname, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import zlib from 'node:zlib'
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
|
|
@ -25,7 +26,11 @@ import { decodeKimi, toObservations as toKimiObservations } from '../src/provide
|
|||
import { decodeCodeWhale, toObservations as toCodeWhaleObservations } from '../src/providers/codewhale/index.js'
|
||||
import { decodeCodebuff, toObservations as toCodebuffObservations } from '../src/providers/codebuff/index.js'
|
||||
import { decodeOpenClaw, toObservations as toOpenClawObservations } from '../src/providers/openclaw/index.js'
|
||||
import { decodeZed, toObservations as toZedObservations } from '../src/providers/zed/index.js'
|
||||
import { decodeForge, toObservations as toForgeObservations } from '../src/providers/forge/index.js'
|
||||
import { decodeGoose, toObservations as toGooseObservations } from '../src/providers/goose/index.js'
|
||||
import type { DecodeContext } from '../src/contracts.js'
|
||||
import type { ZedThreadRow } from '../src/providers/zed/index.js'
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url))
|
||||
const goldenEnvelope = JSON.parse(
|
||||
|
|
@ -687,6 +692,191 @@ describe('content-smuggling guardrail: real openclaw decode -> toObservations is
|
|||
})
|
||||
})
|
||||
|
||||
describe('content-smuggling guardrail: real zed decode -> toObservations is secret-free', () => {
|
||||
// A hostile Zed thread planting every secret in the free-text fields the
|
||||
// decode captures: the thread summary (user message) and the project path.
|
||||
// Zed never records tool calls or file paths, so there is no command-line or
|
||||
// path-fingerprint surface to probe here.
|
||||
const zedContext: DecodeContext = { privacyKey: 'test-privacy-key', providerId: 'zed', sourceRef: 'ref' }
|
||||
const zstd = (zlib as { zstdCompressSync?: (buf: Buffer) => Buffer }).zstdCompressSync
|
||||
|
||||
function decodeAndMinimize() {
|
||||
const thread = {
|
||||
model: { provider: 'anthropic', model: 'claude-opus-4-8' },
|
||||
request_token_usage: {
|
||||
'req-1': { input_tokens: 500, output_tokens: 200 },
|
||||
},
|
||||
}
|
||||
const row: ZedThreadRow = {
|
||||
id: 'thread-hostile',
|
||||
summary: `${SECRETS.prompt} ${SECRETS.apiKey} ${SECRETS.fileContent}`,
|
||||
updated_at: '2026-07-17T10:00:00.000Z',
|
||||
data_type: 'zstd',
|
||||
data: zstd!(Buffer.from(JSON.stringify(thread))),
|
||||
}
|
||||
const { calls } = decodeZed({ records: [row], context: zedContext })
|
||||
const { sessions } = toZedObservations(
|
||||
{ sessionId: 'thread-hostile', projectPath: SECRETS.absPath, calls },
|
||||
{ privacyKey: 'test-privacy-key', provider: 'zed' },
|
||||
)
|
||||
return {
|
||||
schemaVersion: OBSERVATION_SCHEMA_VERSION,
|
||||
generator: { name: '@codeburn/core', version: '0.0.0-test' },
|
||||
sessions,
|
||||
}
|
||||
}
|
||||
|
||||
it.skipIf(!zstd)('produces a schema-valid envelope from the hostile thread', () => {
|
||||
expect(ObservationEnvelope.safeParse(decodeAndMinimize()).success).toBe(true)
|
||||
})
|
||||
|
||||
it.skipIf(!zstd)('the serialized envelope contains none of the planted secrets', () => {
|
||||
const serialized = JSON.stringify(decodeAndMinimize())
|
||||
for (const secret of ALL_SECRETS) {
|
||||
expect(serialized).not.toContain(secret)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('content-smuggling guardrail: real forge decode -> toObservations is secret-free', () => {
|
||||
// A hostile Forge conversation planting every secret in the free-text fields
|
||||
// the decode captures: the user message, a Bash command, and a tool NAME
|
||||
// carrying a command line (must fail the canonical-name filter). The `context`
|
||||
// column arrives as a still-serialized JSON string, exactly as sqlite returns it.
|
||||
const forgeContext: DecodeContext = { privacyKey: 'test-privacy-key', providerId: 'forge', sourceRef: 'ref' }
|
||||
|
||||
function decodeAndMinimize() {
|
||||
const context = {
|
||||
messages: [
|
||||
{ message: { text: { role: 'User', content: `${SECRETS.prompt} ${SECRETS.apiKey} ${SECRETS.fileContent}` } } },
|
||||
{
|
||||
message: {
|
||||
text: {
|
||||
role: 'Assistant', model: 'claude-opus-4-6',
|
||||
tool_calls: [
|
||||
{ name: 'shell', call_id: 'call-1', arguments: { command: SECRETS.commandLine } },
|
||||
// A hostile tool NAME carrying a command line: fails canonical charset.
|
||||
{ name: SECRETS.commandLine, call_id: 'call-2', arguments: {} },
|
||||
],
|
||||
},
|
||||
},
|
||||
usage: { prompt_tokens: { actual: 500 }, completion_tokens: { actual: 200 } },
|
||||
},
|
||||
],
|
||||
}
|
||||
const records = [{
|
||||
conversation_id: 'conv-hostile',
|
||||
title: 'hostile',
|
||||
workspace_id: 1,
|
||||
context: JSON.stringify(context),
|
||||
created_at: '2026-07-17 10:00:00',
|
||||
updated_at: '2026-07-17 10:00:05',
|
||||
}]
|
||||
const { calls } = decodeForge({ records, context: forgeContext })
|
||||
const { sessions } = toForgeObservations(
|
||||
{ sessionId: 'conv-hostile', projectPath: SECRETS.absPath, calls },
|
||||
{ privacyKey: 'test-privacy-key', provider: 'forge' },
|
||||
)
|
||||
return {
|
||||
schemaVersion: OBSERVATION_SCHEMA_VERSION,
|
||||
generator: { name: '@codeburn/core', version: '0.0.0-test' },
|
||||
sessions,
|
||||
}
|
||||
}
|
||||
|
||||
it('produces a schema-valid envelope from the hostile conversation', () => {
|
||||
expect(ObservationEnvelope.safeParse(decodeAndMinimize()).success).toBe(true)
|
||||
})
|
||||
|
||||
it('the serialized envelope contains none of the planted secrets', () => {
|
||||
const serialized = JSON.stringify(decodeAndMinimize())
|
||||
for (const secret of ALL_SECRETS) {
|
||||
expect(serialized).not.toContain(secret)
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps canonical tool names (Bash) and drops the argument-carrying name', () => {
|
||||
const env = decodeAndMinimize()
|
||||
const allToolNames = env.sessions.flatMap(s => s.calls.flatMap(c => c.toolNames))
|
||||
expect(allToolNames).toContain('Bash')
|
||||
expect(allToolNames).not.toContain(SECRETS.commandLine)
|
||||
})
|
||||
})
|
||||
|
||||
describe('content-smuggling guardrail: real goose decode -> toObservations is secret-free', () => {
|
||||
// A hostile Goose session planting every secret in the free-text fields the
|
||||
// decode captures: the first user message, a Bash command, a read_file path,
|
||||
// and a tool NAME carrying a command line. BLOB columns arrive pre-converted
|
||||
// to text (blobToText), exactly as the host hands them over.
|
||||
const gooseContext: DecodeContext = { privacyKey: 'test-privacy-key', providerId: 'goose', sourceRef: 'ref' }
|
||||
|
||||
function decodeAndMinimize() {
|
||||
const records = [{
|
||||
sessionId: 'sess-hostile',
|
||||
session: {
|
||||
id: 'sess-hostile',
|
||||
workingDir: SECRETS.absPath,
|
||||
createdAt: '2026-07-17T10:00:00Z',
|
||||
updatedAt: '2026-07-17T10:00:05Z',
|
||||
accumulatedInputTokens: 500,
|
||||
accumulatedOutputTokens: 200,
|
||||
modelConfigJson: JSON.stringify({ model_name: 'gpt-5.5' }),
|
||||
},
|
||||
assistantToolMessages: [
|
||||
{ contentJson: JSON.stringify([
|
||||
{ type: 'toolRequest', toolCall: { value: { name: 'developer__shell', arguments: { command: SECRETS.commandLine } } } },
|
||||
{ type: 'toolRequest', toolCall: { value: { name: 'developer__read_file', arguments: { file_path: SECRETS.absPath } } } },
|
||||
// A hostile tool NAME carrying a command line: fails canonical charset.
|
||||
{ type: 'toolRequest', toolCall: { value: { name: SECRETS.commandLine, arguments: {} } } },
|
||||
]) },
|
||||
// A second turn so toolSequence (included only when it has MORE THAN ONE
|
||||
// turn) actually reaches toObservations for fingerprinting.
|
||||
{ contentJson: JSON.stringify([
|
||||
{ type: 'toolRequest', toolCall: { value: { name: 'developer__list_directory', arguments: {} } } },
|
||||
]) },
|
||||
],
|
||||
firstUserMessage: { contentJson: JSON.stringify([{ type: 'text', text: `${SECRETS.prompt} ${SECRETS.apiKey} ${SECRETS.fileContent}` }]) },
|
||||
}]
|
||||
const { calls } = decodeGoose({ records, context: gooseContext })
|
||||
const { sessions } = toGooseObservations(
|
||||
{ sessionId: 'sess-hostile', projectPath: SECRETS.absPath, calls },
|
||||
{ privacyKey: 'test-privacy-key', provider: 'goose' },
|
||||
)
|
||||
return {
|
||||
schemaVersion: OBSERVATION_SCHEMA_VERSION,
|
||||
generator: { name: '@codeburn/core', version: '0.0.0-test' },
|
||||
sessions,
|
||||
}
|
||||
}
|
||||
|
||||
it('produces a schema-valid envelope from the hostile session', () => {
|
||||
expect(ObservationEnvelope.safeParse(decodeAndMinimize()).success).toBe(true)
|
||||
})
|
||||
|
||||
it('the serialized envelope contains none of the planted secrets', () => {
|
||||
const serialized = JSON.stringify(decodeAndMinimize())
|
||||
for (const secret of ALL_SECRETS) {
|
||||
expect(serialized).not.toContain(secret)
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps canonical tool names (Bash/Read) and drops the argument-carrying name', () => {
|
||||
const env = decodeAndMinimize()
|
||||
const allToolNames = env.sessions.flatMap(s => s.calls.flatMap(c => c.toolNames))
|
||||
expect(allToolNames).toContain('Bash')
|
||||
expect(allToolNames).toContain('Read')
|
||||
expect(allToolNames).not.toContain(SECRETS.commandLine)
|
||||
})
|
||||
|
||||
it('fingerprints the read_file path into a 16-hex resourceRead, never the raw path', () => {
|
||||
const env = decodeAndMinimize()
|
||||
const reads = env.sessions.flatMap(s => s.calls.flatMap(c => c.resourceReads ?? []))
|
||||
expect(reads.length).toBeGreaterThan(0)
|
||||
for (const ref of reads) expect(ref.resourceId).toMatch(/^[0-9a-f]{16}$/)
|
||||
expect(allStrings(reads)).not.toContain(SECRETS.absPath)
|
||||
})
|
||||
})
|
||||
|
||||
describe('content-smuggling guardrail: diagnostic detail rejects paths', () => {
|
||||
it('rejects an absolute path', () => {
|
||||
expect(DiagnosticDetail.safeParse(SECRETS.absPath).success).toBe(false)
|
||||
|
|
|
|||
78
packages/core/tests/providers/crush-decode.test.ts
Normal file
78
packages/core/tests/providers/crush-decode.test.ts
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { decodeCrush, toObservations } from '../../src/providers/crush/index.js'
|
||||
import { ObservationEnvelope } from '../../src/observations.js'
|
||||
import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js'
|
||||
import type { DecodeContext } from '../../src/contracts.js'
|
||||
import type { CrushRawRecord } from '../../src/providers/crush/index.js'
|
||||
|
||||
const context: DecodeContext = { privacyKey: 'k', providerId: 'crush', sourceRef: 'ref' }
|
||||
|
||||
function record(overrides: Partial<CrushRawRecord> = {}): CrushRawRecord {
|
||||
return {
|
||||
id: 'sess-a',
|
||||
prompt_tokens: 1234,
|
||||
completion_tokens: 567,
|
||||
cost: 0.0789,
|
||||
created_at: 1_700_000_010,
|
||||
updated_at: 1_700_000_999,
|
||||
message_count: 3,
|
||||
model: 'claude-sonnet-4-6',
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
describe('crush rich decode (moved to @codeburn/core)', () => {
|
||||
it('decodes a session row with a real cost into a measured, cost-free rich call', () => {
|
||||
const { calls } = decodeCrush({ records: [record()], context })
|
||||
expect(calls).toHaveLength(1)
|
||||
const call = calls[0]!
|
||||
|
||||
// No pricing crosses into the decode layer as a costBasis marker; the
|
||||
// dollar figure is carried as `measuredCostUSD` only, for the host to map.
|
||||
expect(call).not.toHaveProperty('costBasis')
|
||||
expect(call.measuredCostUSD).toBeCloseTo(0.0789, 6)
|
||||
expect(call.model).toBe('claude-sonnet-4-6')
|
||||
expect(call.inputTokens).toBe(1234)
|
||||
expect(call.outputTokens).toBe(567)
|
||||
expect(call.deduplicationKey).toBe('crush:sess-a')
|
||||
expect(call.sessionId).toBe('sess-a')
|
||||
// Crush stores epoch seconds; 1_700_000_999 sec -> 2023-11-14T22:29:59.000Z.
|
||||
expect(call.timestamp).toBe(new Date(1_700_000_999 * 1000).toISOString())
|
||||
})
|
||||
|
||||
it('omits measuredCostUSD when cost is zero (estimated fallback)', () => {
|
||||
const { calls } = decodeCrush({ records: [record({ cost: 0, prompt_tokens: 5, completion_tokens: 5 })], context })
|
||||
expect(calls).toHaveLength(1)
|
||||
expect(calls[0]).not.toHaveProperty('measuredCostUSD')
|
||||
})
|
||||
|
||||
it('skips a session with zero tokens and zero cost', () => {
|
||||
const { calls } = decodeCrush({ records: [record({ prompt_tokens: 0, completion_tokens: 0, cost: 0 })], context })
|
||||
expect(calls).toEqual([])
|
||||
})
|
||||
|
||||
it('threads a live seenKeys set so a repeated session id across passes drops', () => {
|
||||
const seen = new Set<string>()
|
||||
const first = decodeCrush({ records: [record()], context, seenKeys: seen }).calls
|
||||
expect(first).toHaveLength(1)
|
||||
const again = decodeCrush({ records: [record()], context, seenKeys: seen }).calls
|
||||
expect(again).toEqual([])
|
||||
})
|
||||
|
||||
it('toObservations produces a schema-valid, content-free envelope', () => {
|
||||
const { calls } = decodeCrush({ records: [record()], context })
|
||||
const { sessions } = toObservations(
|
||||
{ sessionId: 'sess-a', projectPath: '/Users/t/alpha', calls },
|
||||
{ privacyKey: 'test-privacy-key', provider: 'crush' },
|
||||
)
|
||||
const envelope = {
|
||||
schemaVersion: OBSERVATION_SCHEMA_VERSION,
|
||||
generator: { name: '@codeburn/core', version: '0.0.0-test' },
|
||||
sessions,
|
||||
}
|
||||
expect(ObservationEnvelope.safeParse(envelope).success).toBe(true)
|
||||
expect(sessions[0]?.calls[0]?.costBasis).toBe('measured')
|
||||
expect(sessions[0]?.calls[0]?.measuredCostUSD).toBeCloseTo(0.0789, 6)
|
||||
})
|
||||
})
|
||||
104
packages/core/tests/providers/forge-decode.test.ts
Normal file
104
packages/core/tests/providers/forge-decode.test.ts
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { decodeForge, toObservations } from '../../src/providers/forge/index.js'
|
||||
import { ObservationEnvelope } from '../../src/observations.js'
|
||||
import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js'
|
||||
import type { DecodeContext } from '../../src/contracts.js'
|
||||
import type { ForgeConversationRow } from '../../src/providers/forge/index.js'
|
||||
|
||||
const context: DecodeContext = { privacyKey: 'k', providerId: 'forge', sourceRef: 'ref' }
|
||||
|
||||
function row(context_: unknown, overrides: Partial<ForgeConversationRow> = {}): ForgeConversationRow {
|
||||
return {
|
||||
conversation_id: 'conv-1',
|
||||
title: 'Forge Project',
|
||||
workspace_id: 123,
|
||||
context: JSON.stringify(context_),
|
||||
created_at: '2026-05-06 15:00:00',
|
||||
updated_at: '2026-05-06 15:20:41.379094',
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
describe('forge rich decode (moved to @codeburn/core)', () => {
|
||||
it('maps tool calls, attributes the nearest previous user message, and emits raw bash commands', () => {
|
||||
const conv = {
|
||||
messages: [
|
||||
{ message: { text: { role: 'User', content: 'implement forge' } } },
|
||||
{
|
||||
message: {
|
||||
text: {
|
||||
role: 'Assistant', content: '', model: 'claude-opus-4-6',
|
||||
tool_calls: [
|
||||
{ name: 'shell', call_id: 'call-1', arguments: { command: 'git status && npm test' } },
|
||||
{ name: 'Read', call_id: 'call-2', arguments: { file_path: '/tmp/a' } },
|
||||
],
|
||||
},
|
||||
},
|
||||
usage: { prompt_tokens: { actual: 1200 }, completion_tokens: { actual: 300 }, cached_tokens: { actual: 200 } },
|
||||
},
|
||||
],
|
||||
}
|
||||
const { calls } = decodeForge({ records: [row(conv)], context })
|
||||
expect(calls).toHaveLength(1)
|
||||
const call = calls[0]!
|
||||
expect(call).not.toHaveProperty('costUSD')
|
||||
expect(call).not.toHaveProperty('costBasis')
|
||||
expect(call.model).toBe('claude-opus-4-6')
|
||||
expect(call.inputTokens).toBe(1000) // 1200 - 200 cached
|
||||
expect(call.cacheReadInputTokens).toBe(200)
|
||||
expect(call.tools).toEqual(['Bash', 'Read'])
|
||||
// Raw command strings survive host-side; base-name extraction is the CLI's job.
|
||||
expect(call.rawBashCommands).toEqual(['git status && npm test'])
|
||||
expect(call.userMessage).toBe('implement forge')
|
||||
expect(call.deduplicationKey).toBe('forge:conv-1:call-1')
|
||||
expect(call.timestamp).toBe('2026-05-06T15:20:41.379Z')
|
||||
})
|
||||
|
||||
it('skips zero-token assistant messages', () => {
|
||||
const conv = {
|
||||
messages: [
|
||||
{ message: { text: { role: 'User', content: 'zero tokens' } } },
|
||||
{ message: { text: { role: 'Assistant', model: 'claude-opus-4-6' } }, usage: { prompt_tokens: { actual: 0 }, completion_tokens: { actual: 0 } } },
|
||||
],
|
||||
}
|
||||
expect(decodeForge({ records: [row(conv)], context }).calls).toEqual([])
|
||||
})
|
||||
|
||||
it('yields nothing for invalid JSON context without throwing', () => {
|
||||
expect(decodeForge({ records: [row(null, { context: '{invalid' })], context }).calls).toEqual([])
|
||||
})
|
||||
|
||||
it('threads a live seenKeys set so a repeated call_id across passes drops', () => {
|
||||
const conv = {
|
||||
messages: [
|
||||
{ message: { text: { role: 'User', content: 'hi' } } },
|
||||
{ message: { text: { role: 'Assistant', model: 'm', tool_calls: [{ name: 'shell', call_id: 'call-1', arguments: {} }] } }, usage: { prompt_tokens: { actual: 5 }, completion_tokens: { actual: 5 } } },
|
||||
],
|
||||
}
|
||||
const seen = new Set<string>()
|
||||
expect(decodeForge({ records: [row(conv)], context, seenKeys: seen }).calls).toHaveLength(1)
|
||||
expect(decodeForge({ records: [row(conv)], context, seenKeys: seen }).calls).toEqual([])
|
||||
})
|
||||
|
||||
it('toObservations produces a schema-valid, content-free envelope', () => {
|
||||
const conv = {
|
||||
messages: [
|
||||
{ message: { text: { role: 'User', content: 'implement forge' } } },
|
||||
{ message: { text: { role: 'Assistant', model: 'claude-opus-4-6', tool_calls: [{ name: 'shell', call_id: 'call-1', arguments: { command: 'npm test' } }] } }, usage: { prompt_tokens: { actual: 100 }, completion_tokens: { actual: 20 } } },
|
||||
],
|
||||
}
|
||||
const { calls } = decodeForge({ records: [row(conv)], context })
|
||||
const { sessions } = toObservations(
|
||||
{ sessionId: 'conv-1', projectPath: '/Users/t/proj', calls },
|
||||
{ privacyKey: 'test-privacy-key', provider: 'forge' },
|
||||
)
|
||||
const envelope = {
|
||||
schemaVersion: OBSERVATION_SCHEMA_VERSION,
|
||||
generator: { name: '@codeburn/core', version: '0.0.0-test' },
|
||||
sessions,
|
||||
}
|
||||
expect(ObservationEnvelope.safeParse(envelope).success).toBe(true)
|
||||
expect(JSON.stringify(envelope)).not.toContain('implement forge')
|
||||
})
|
||||
})
|
||||
107
packages/core/tests/providers/goose-decode.test.ts
Normal file
107
packages/core/tests/providers/goose-decode.test.ts
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { decodeGoose, toObservations } from '../../src/providers/goose/index.js'
|
||||
import { ObservationEnvelope } from '../../src/observations.js'
|
||||
import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js'
|
||||
import type { DecodeContext } from '../../src/contracts.js'
|
||||
import type { GooseSessionRecords } from '../../src/providers/goose/index.js'
|
||||
|
||||
const context: DecodeContext = { privacyKey: 'k', providerId: 'goose', sourceRef: 'ref' }
|
||||
|
||||
function sessionRecords(overrides: Partial<GooseSessionRecords> = {}): GooseSessionRecords {
|
||||
return {
|
||||
sessionId: 'sess-1',
|
||||
session: {
|
||||
id: 'sess-1',
|
||||
workingDir: '/Users/me/project',
|
||||
createdAt: '2026-06-01T10:00:00Z',
|
||||
updatedAt: '2026-06-01T10:05:30Z',
|
||||
accumulatedInputTokens: 1500,
|
||||
accumulatedOutputTokens: 400,
|
||||
modelConfigJson: JSON.stringify({ model_name: 'gpt-5.5', reasoning: true }),
|
||||
},
|
||||
assistantToolMessages: [
|
||||
{ contentJson: JSON.stringify([
|
||||
{ type: 'text', text: 'working on it' },
|
||||
{ type: 'toolRequest', toolCall: { value: { name: 'developer__shell', arguments: { command: 'npm test && npm run lint' } } } },
|
||||
{ type: 'toolRequest', toolCall: { value: { name: 'developer__read_file', arguments: { file_path: 'src/index.ts' } } } },
|
||||
]) },
|
||||
{ contentJson: JSON.stringify([
|
||||
{ type: 'toolRequest', toolCall: { value: { name: 'some_native_tool', arguments: {} } } },
|
||||
]) },
|
||||
],
|
||||
firstUserMessage: { contentJson: JSON.stringify([{ type: 'text', text: 'please refactor the shell wrapper and run tests' }]) },
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
describe('goose rich decode (moved to @codeburn/core)', () => {
|
||||
it('decodes a session into a single rich call with mapped tools, raw commands, and a two-turn toolSequence', () => {
|
||||
const { calls } = decodeGoose({ records: [sessionRecords()], context })
|
||||
expect(calls).toHaveLength(1)
|
||||
const call = calls[0]!
|
||||
|
||||
expect(call).not.toHaveProperty('costUSD')
|
||||
expect(call).not.toHaveProperty('costBasis')
|
||||
expect(call.model).toBe('gpt-5.5')
|
||||
expect(call.inputTokens).toBe(1500)
|
||||
expect(call.outputTokens).toBe(400)
|
||||
expect(call.tools).toEqual(['Bash', 'Read', 'some_native_tool'])
|
||||
expect(call.rawBashCommands).toEqual(['npm test && npm run lint'])
|
||||
expect(call.userMessage).toBe('please refactor the shell wrapper and run tests')
|
||||
expect(call.deduplicationKey).toBe('goose:sess-1')
|
||||
expect(call.timestamp).toBe('2026-06-01T10:05:30.000Z')
|
||||
// toolSequence included only when it has MORE than one turn.
|
||||
expect(call.toolSequence).toHaveLength(2)
|
||||
expect(call.toolSequence?.[0]?.[1]).toEqual({ tool: 'Read', file: 'src/index.ts' })
|
||||
})
|
||||
|
||||
it('omits toolSequence when only one turn used tools', () => {
|
||||
const records = sessionRecords({
|
||||
assistantToolMessages: [
|
||||
{ contentJson: JSON.stringify([{ type: 'toolRequest', toolCall: { value: { name: 'developer__shell', arguments: { command: 'ls' } } } }]) },
|
||||
],
|
||||
})
|
||||
const { calls } = decodeGoose({ records: [records], context })
|
||||
expect(calls[0]!.toolSequence).toBeUndefined()
|
||||
})
|
||||
|
||||
it('falls back to "unknown" model when model_config_json is missing/malformed', () => {
|
||||
const records = sessionRecords({ session: { ...sessionRecords().session, modelConfigJson: null } })
|
||||
expect(decodeGoose({ records: [records], context }).calls[0]!.model).toBe('unknown')
|
||||
})
|
||||
|
||||
it('skips a session with zero accumulated tokens', () => {
|
||||
const records = sessionRecords({ session: { ...sessionRecords().session, accumulatedInputTokens: 0, accumulatedOutputTokens: 0 } })
|
||||
expect(decodeGoose({ records: [records], context }).calls).toEqual([])
|
||||
})
|
||||
|
||||
it('resolves to an empty timestamp (never the clock) when both timestamp columns are unparseable', () => {
|
||||
const records = sessionRecords({ session: { ...sessionRecords().session, createdAt: 'not-a-date', updatedAt: null } })
|
||||
expect(decodeGoose({ records: [records], context }).calls[0]!.timestamp).toBe('')
|
||||
})
|
||||
|
||||
it('threads a live seenKeys set so a repeated session id across passes drops', () => {
|
||||
const seen = new Set<string>()
|
||||
expect(decodeGoose({ records: [sessionRecords()], context, seenKeys: seen }).calls).toHaveLength(1)
|
||||
expect(decodeGoose({ records: [sessionRecords()], context, seenKeys: seen }).calls).toEqual([])
|
||||
})
|
||||
|
||||
it('toObservations produces a schema-valid, content-free envelope and fingerprints the read_file path', () => {
|
||||
const { calls } = decodeGoose({ records: [sessionRecords()], context })
|
||||
const { sessions } = toObservations(
|
||||
{ sessionId: 'sess-1', projectPath: '/Users/me/project', calls },
|
||||
{ privacyKey: 'test-privacy-key', provider: 'goose' },
|
||||
)
|
||||
const envelope = {
|
||||
schemaVersion: OBSERVATION_SCHEMA_VERSION,
|
||||
generator: { name: '@codeburn/core', version: '0.0.0-test' },
|
||||
sessions,
|
||||
}
|
||||
expect(ObservationEnvelope.safeParse(envelope).success).toBe(true)
|
||||
const reads = sessions.flatMap(s => s.calls.flatMap(c => c.resourceReads ?? []))
|
||||
expect(reads.length).toBeGreaterThan(0)
|
||||
for (const ref of reads) expect(ref.resourceId).toMatch(/^[0-9a-f]{16}$/)
|
||||
expect(JSON.stringify(envelope)).not.toContain('please refactor')
|
||||
})
|
||||
})
|
||||
76
packages/core/tests/providers/zcode-decode.test.ts
Normal file
76
packages/core/tests/providers/zcode-decode.test.ts
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { decodeZcode, toObservations } from '../../src/providers/zcode/index.js'
|
||||
import { ObservationEnvelope } from '../../src/observations.js'
|
||||
import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js'
|
||||
import type { DecodeContext } from '../../src/contracts.js'
|
||||
import type { ZcodeSessionRecords } from '../../src/providers/zcode/index.js'
|
||||
|
||||
const context: DecodeContext = { privacyKey: 'k', providerId: 'zcode', sourceRef: 'ref' }
|
||||
|
||||
function records(): ZcodeSessionRecords {
|
||||
return {
|
||||
sessionId: 'sess-1',
|
||||
usageRows: [
|
||||
{ id: 'mu-1', turn_id: 'turn-1', model_id: 'GLM-5.2', input_tokens: 9125, output_tokens: 27, reasoning_tokens: 12, cache_creation_input_tokens: 0, cache_read_input_tokens: 8064, started_at: 1781981181862, completed_at: 1781981202412 },
|
||||
{ id: 'mu-2', turn_id: 'turn-1', model_id: 'GLM-5.2', input_tokens: 200, output_tokens: 40, reasoning_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0, started_at: 1781981210000, completed_at: 1781981220000 },
|
||||
// Zero-token row: skipped.
|
||||
{ id: 'mu-zero', turn_id: 'turn-2', model_id: 'GLM-5.2', input_tokens: 0, output_tokens: 0, reasoning_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0, started_at: 1781981230000, completed_at: 1781981231000 },
|
||||
// No turn_id: no tools attached.
|
||||
{ id: 'mu-3', turn_id: null, model_id: 'GLM-5.2', input_tokens: 500, output_tokens: 60, reasoning_tokens: 0, cache_creation_input_tokens: 100, cache_read_input_tokens: 0, started_at: 1781981240000, completed_at: null },
|
||||
],
|
||||
toolRows: [
|
||||
{ turn_id: 'turn-1', tool_name: 'Bash' },
|
||||
{ turn_id: 'turn-1', tool_name: 'Read' },
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
describe('zcode rich decode (moved to @codeburn/core)', () => {
|
||||
it('decodes usage rows, splitting cached tokens and skipping zero-token rows', () => {
|
||||
const { calls } = decodeZcode({ records: [records()], context })
|
||||
expect(calls).toHaveLength(3)
|
||||
|
||||
const [first, second, third] = calls
|
||||
expect(first).not.toHaveProperty('costUSD')
|
||||
expect(first).not.toHaveProperty('costBasis')
|
||||
expect(first!.inputTokens).toBe(1061) // 9125 - 8064 cached
|
||||
expect(first!.cacheReadInputTokens).toBe(8064)
|
||||
expect(first!.reasoningTokens).toBe(12)
|
||||
expect(first!.tools).toEqual(['Bash', 'Read'])
|
||||
expect(first!.turnId).toBe('turn-1')
|
||||
expect(first!.deduplicationKey).toBe('zcode:mu-1')
|
||||
|
||||
// Same turn's second row gets no tools (already attached to the first).
|
||||
expect(second!.tools).toEqual([])
|
||||
expect(second!.turnId).toBe('turn-1')
|
||||
|
||||
// No turn_id -> no tools, turnId undefined.
|
||||
expect(third!.tools).toEqual([])
|
||||
expect(third!.turnId).toBeUndefined()
|
||||
expect(third!.cacheCreationInputTokens).toBe(100)
|
||||
})
|
||||
|
||||
it('threads a live seenKeys set so a repeated row id across passes drops', () => {
|
||||
const seen = new Set<string>()
|
||||
const first = decodeZcode({ records: [records()], context, seenKeys: seen }).calls
|
||||
expect(first).toHaveLength(3)
|
||||
const again = decodeZcode({ records: [records()], context, seenKeys: seen }).calls
|
||||
expect(again).toEqual([])
|
||||
})
|
||||
|
||||
it('toObservations produces a schema-valid, content-free envelope', () => {
|
||||
const { calls } = decodeZcode({ records: [records()], context })
|
||||
const { sessions } = toObservations(
|
||||
{ sessionId: 'sess-1', projectPath: '/Users/me/proj', calls },
|
||||
{ privacyKey: 'test-privacy-key', provider: 'zcode' },
|
||||
)
|
||||
const envelope = {
|
||||
schemaVersion: OBSERVATION_SCHEMA_VERSION,
|
||||
generator: { name: '@codeburn/core', version: '0.0.0-test' },
|
||||
sessions,
|
||||
}
|
||||
expect(ObservationEnvelope.safeParse(envelope).success).toBe(true)
|
||||
expect(sessions[0]?.calls.every(c => c.costBasis === 'estimated')).toBe(true)
|
||||
})
|
||||
})
|
||||
106
packages/core/tests/providers/zed-decode.test.ts
Normal file
106
packages/core/tests/providers/zed-decode.test.ts
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
import zlib from 'node:zlib'
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { decodeZed, toObservations } from '../../src/providers/zed/index.js'
|
||||
import { ObservationEnvelope } from '../../src/observations.js'
|
||||
import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js'
|
||||
import type { DecodeContext } from '../../src/contracts.js'
|
||||
import type { ZedThreadRow } from '../../src/providers/zed/index.js'
|
||||
|
||||
const context: DecodeContext = { privacyKey: 'k', providerId: 'zed', sourceRef: 'ref' }
|
||||
|
||||
const zstd = (zlib as { zstdCompressSync?: (buf: Buffer) => Buffer }).zstdCompressSync
|
||||
const skipReason = !zstd ? 'zlib zstd not available — needs Node 22.15+; skipping' : null
|
||||
|
||||
function zstdRow(id: string, thread: unknown, opts: Partial<ZedThreadRow> = {}): ZedThreadRow {
|
||||
return {
|
||||
id,
|
||||
summary: opts.summary ?? 'a thread',
|
||||
updated_at: opts.updated_at ?? '2026-06-20T10:00:00Z',
|
||||
data_type: 'zstd',
|
||||
data: zstd!(Buffer.from(JSON.stringify(thread))),
|
||||
...opts,
|
||||
}
|
||||
}
|
||||
|
||||
describe.skipIf(skipReason !== null)('zed rich decode (moved to @codeburn/core)', () => {
|
||||
it('emits one call per request, plus a cumulative-remainder entry when the map undercounts', () => {
|
||||
const row = zstdRow('thread-1', {
|
||||
model: { provider: 'anthropic', model: 'claude-opus-4-8' },
|
||||
request_token_usage: {
|
||||
'req-1': { input_tokens: 1200, output_tokens: 300, cache_creation_input_tokens: 5000, cache_read_input_tokens: 90000 },
|
||||
'req-2': { input_tokens: 800, output_tokens: 150 },
|
||||
},
|
||||
cumulative_token_usage: { input_tokens: 2000, output_tokens: 450 },
|
||||
}, { summary: 'refactor the parser', updated_at: '2026-06-21T09:30:00Z' })
|
||||
|
||||
const { calls } = decodeZed({ records: [row], context })
|
||||
expect(calls).toHaveLength(2)
|
||||
const first = calls.find(c => c.deduplicationKey === 'zed:thread-1:req-1')!
|
||||
expect(first.inputTokens).toBe(1200)
|
||||
expect(first.cacheCreationInputTokens).toBe(5000)
|
||||
expect(first.model).toBe('claude-opus-4-8')
|
||||
expect(first.userMessage).toBe('refactor the parser')
|
||||
expect(first.timestamp).toBe('2026-06-21T09:30:00.000Z')
|
||||
expect(calls.reduce((s, c) => s + c.inputTokens, 0)).toBe(2000)
|
||||
})
|
||||
|
||||
it('skips non-zstd/json rows and malformed blobs, recording a diagnostic without dropping healthy threads', () => {
|
||||
const badType: ZedThreadRow = { id: 'bad-type', summary: null, updated_at: '2026-06-20T10:00:00Z', data_type: 'protobuf', data: Buffer.from('{}') }
|
||||
const badBlob: ZedThreadRow = { id: 'bad-blob', summary: null, updated_at: '2026-06-20T10:00:00Z', data_type: 'zstd', data: Buffer.from('not zstd at all') }
|
||||
const good = zstdRow('good', { model: { model: 'claude-opus-4-8' }, request_token_usage: { 'req-1': { input_tokens: 10, output_tokens: 5 } } })
|
||||
|
||||
const { calls, diagnostics } = decodeZed({ records: [badType, badBlob, good], context })
|
||||
expect(calls).toHaveLength(1)
|
||||
expect(calls[0]!.sessionId).toBe('good')
|
||||
expect(diagnostics.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('reads legacy uncompressed json rows', () => {
|
||||
const row: ZedThreadRow = {
|
||||
id: 'legacy', summary: null, updated_at: '2026-06-22T11:00:00Z', data_type: 'json',
|
||||
data: Buffer.from(JSON.stringify({ model: { model: 'claude-sonnet-4-6' }, request_token_usage: { 'req-1': { input_tokens: 40, output_tokens: 8 } } })),
|
||||
}
|
||||
const { calls } = decodeZed({ records: [row], context })
|
||||
expect(calls).toHaveLength(1)
|
||||
expect(calls[0]!.inputTokens).toBe(40)
|
||||
expect(calls[0]!.model).toBe('claude-sonnet-4-6')
|
||||
})
|
||||
|
||||
it('skips threads whose usage is entirely zero', () => {
|
||||
const row = zstdRow('zero', {
|
||||
model: { model: 'claude-opus-4-8' },
|
||||
request_token_usage: { 'req-1': { input_tokens: 0, output_tokens: 0 } },
|
||||
cumulative_token_usage: { input_tokens: 0, output_tokens: 0 },
|
||||
})
|
||||
expect(decodeZed({ records: [row], context }).calls).toEqual([])
|
||||
})
|
||||
|
||||
it('threads a live seenKeys set so a repeated thread across passes drops', () => {
|
||||
const row = zstdRow('thread-3', { model: { model: 'claude-opus-4-8' }, request_token_usage: { 'req-1': { input_tokens: 100, output_tokens: 50 } } })
|
||||
const seen = new Set<string>()
|
||||
expect(decodeZed({ records: [row], context, seenKeys: seen }).calls).toHaveLength(1)
|
||||
expect(decodeZed({ records: [row], context, seenKeys: seen }).calls).toEqual([])
|
||||
})
|
||||
|
||||
it('toObservations produces a schema-valid, content-free envelope', () => {
|
||||
const row = zstdRow('thread-4', {
|
||||
model: { model: 'claude-opus-4-8' },
|
||||
request_token_usage: { 'req-1': { input_tokens: 100, output_tokens: 50 } },
|
||||
}, { summary: 'do the thing' })
|
||||
const { calls } = decodeZed({ records: [row], context })
|
||||
const { sessions } = toObservations(
|
||||
{ sessionId: 'thread-4', projectPath: '/Users/t/proj', calls },
|
||||
{ privacyKey: 'test-privacy-key', provider: 'zed' },
|
||||
)
|
||||
const envelope = {
|
||||
schemaVersion: OBSERVATION_SCHEMA_VERSION,
|
||||
generator: { name: '@codeburn/core', version: '0.0.0-test' },
|
||||
sessions,
|
||||
}
|
||||
expect(ObservationEnvelope.safeParse(envelope).success).toBe(true)
|
||||
// The thread summary must never cross into the envelope.
|
||||
expect(JSON.stringify(envelope)).not.toContain('do the thing')
|
||||
})
|
||||
})
|
||||
|
|
@ -27,6 +27,11 @@ export default defineConfig({
|
|||
'src/providers/gemini/index.ts',
|
||||
'src/providers/kimicode/index.ts',
|
||||
'src/providers/pi/index.ts',
|
||||
'src/providers/crush/index.ts',
|
||||
'src/providers/zcode/index.ts',
|
||||
'src/providers/zed/index.ts',
|
||||
'src/providers/forge/index.ts',
|
||||
'src/providers/goose/index.ts',
|
||||
],
|
||||
format: ['esm'],
|
||||
target: 'node20',
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue