diff --git a/packages/cli/src/providers/codewhale.ts b/packages/cli/src/providers/codewhale.ts index 882c94a..3727e51 100644 --- a/packages/cli/src/providers/codewhale.ts +++ b/packages/cli/src/providers/codewhale.ts @@ -2,87 +2,17 @@ import { open, readdir, stat } from 'fs/promises' import { homedir } from 'os' import { join } from 'path' +import { decodeCodeWhale, mapCodeWhaleToolName } from '@codeburn/core/providers/codewhale' +import type { CodeWhaleDecodedCall, CodeWhaleSessionRecords, CodeWhaleMetadata, CodeWhaleMessage } from '@codeburn/core/providers/codewhale' + import { extractBashCommands } from '../bash-utils.js' import { readSessionFile } from '../fs-utils.js' import { getShortModelName } from '../models.js' -import type { ToolCall } from '../types.js' -import type { ParsedProviderCall, Provider, SessionParser, SessionSource } from './types.js' +import { createBridgedProvider } from './bridge.js' +import type { Provider, SessionSource, ParsedProviderCall } from './types.js' const METADATA_PREFIX_BYTES = 64 * 1024 -type CodeWhaleCost = { - session_cost_usd?: number - subagent_cost_usd?: number -} - -type CodeWhaleMetadata = { - id: string - created_at?: string - updated_at?: string - total_tokens?: number - model?: string - model_provider?: string - workspace?: string - cost?: CodeWhaleCost -} - -type CodeWhaleContentBlock = { - type?: string - text?: string - name?: string - input?: unknown -} - -type CodeWhaleMessage = { - role?: string - content?: string | CodeWhaleContentBlock[] -} - -type CodeWhaleSession = { - metadata?: unknown - messages?: unknown -} - -const toolNameMap: Record = { - exec_shell: 'Bash', - exec_shell_wait: 'Bash', - exec_shell_interact: 'Bash', - exec_shell_cancel: 'Bash', - task_shell_start: 'Bash', - task_shell_wait: 'Bash', - terminal_run: 'Bash', - terminal_send: 'Bash', - terminal_wait: 'Bash', - terminal_cancel: 'Bash', - read_file: 'Read', - write_file: 'Write', - edit_file: 'Edit', - fim_edit: 'Edit', - apply_patch: 'Edit', - list_dir: 'Glob', - grep_files: 'Grep', - web_search: 'WebSearch', - fetch_url: 'WebFetch', - 'web.run': 'WebSearch', - agent: 'Agent', - 'agents/list': 'Agent', - 'agents/message': 'Agent', - 'agents/followup': 'Agent', - 'agents/interrupt': 'Agent', - 'agents/wait': 'Agent', - todo_write: 'TodoWrite', - todo_add: 'TodoWrite', - todo_update: 'TodoWrite', - todo_list: 'TodoWrite', - checklist_write: 'TodoWrite', - checklist_add: 'TodoWrite', - checklist_update: 'TodoWrite', - checklist_list: 'TodoWrite', - update_plan: 'TodoWrite', - load_skill: 'Skill', - request_user_input: 'AskUser', -} - function isRecord(value: unknown): value is Record { return value !== null && typeof value === 'object' && !Array.isArray(value) } @@ -112,7 +42,7 @@ function parseMetadata(value: unknown): CodeWhaleMetadata | null { model: nonEmptyString(value['model']), model_provider: nonEmptyString(value['model_provider']), workspace: nonEmptyString(value['workspace']), - cost: isRecord(value['cost']) ? value['cost'] as CodeWhaleCost : undefined, + cost: isRecord(value['cost']) ? (value['cost'] as CodeWhaleMetadata['cost']) : undefined, } } @@ -214,7 +144,7 @@ async function readSessionMetadata(filePath: string): Promise block?.type === 'text' && typeof block.text === 'string') - .map(block => block.text) - .join(' ') - : '' - if (text.trim()) return Array.from(text.trim()).slice(0, 500).join('') - } - return '' -} - -function mapToolName(rawName: string): string { - if (rawName.startsWith('mcp__')) return rawName - if (rawName.startsWith('agents/')) return 'Agent' - return Object.prototype.hasOwnProperty.call(toolNameMap, rawName) - ? toolNameMap[rawName]! - : rawName -} - -function toolInput(value: unknown): Record { - return isRecord(value) ? value : {} -} - -function firstString(input: Record, keys: string[]): string | undefined { - for (const key of keys) { - const value = nonEmptyString(input[key]) - if (value) return value - } - return undefined -} - -function collectTools(messages: CodeWhaleMessage[]): { - tools: string[] - bashCommands: string[] - toolSequence: ToolCall[][] - skills: string[] - subagentTypes: string[] - webSearchRequests: number -} { - const tools: string[] = [] - const bashCommands: string[] = [] - const toolSequence: ToolCall[][] = [] - const skills: string[] = [] - const subagentTypes: string[] = [] - let webSearchRequests = 0 - - for (const message of messages) { - if (message.role !== 'assistant' || !Array.isArray(message.content)) continue - const turnTools: ToolCall[] = [] - - for (const block of message.content) { - if (block?.type !== 'tool_use' && block?.type !== 'server_tool_use') continue - const rawName = nonEmptyString(block.name) - if (!rawName) continue - const mapped = mapToolName(rawName) - const input = toolInput(block.input) - const toolCall: ToolCall = { tool: mapped } - - const file = firstString(input, ['file_path', 'path', 'target_file', 'file']) - if (file) toolCall.file = file - const command = firstString(input, ['command', 'cmd']) - if (command) toolCall.command = command - - if (mapped === 'Bash' && command) { - bashCommands.push(...extractBashCommands(command)) - } - if (mapped === 'Skill') { - const skill = firstString(input, ['name', 'skill', 'skill_name']) - if (skill) skills.push(skill) - } - if (mapped === 'Agent') { - const subagentType = firstString(input, ['type', 'agent_type', 'profile']) - if (subagentType) subagentTypes.push(subagentType) - } - if (mapped === 'WebSearch') webSearchRequests++ - - tools.push(mapped) - turnTools.push(toolCall) - } - - if (turnTools.length > 0) toolSequence.push(turnTools) - } - - return { tools, bashCommands, toolSequence, skills, subagentTypes, webSearchRequests } -} - -function reportedCost(cost: CodeWhaleCost | undefined): { value: number; exact: boolean } { - if (!cost || typeof cost !== 'object') return { value: 0, exact: false } - const hasSessionCost = Object.prototype.hasOwnProperty.call(cost, 'session_cost_usd') - const hasSubagentCost = Object.prototype.hasOwnProperty.call(cost, 'subagent_cost_usd') - if (!hasSessionCost && !hasSubagentCost) return { value: 0, exact: false } +function toProviderCall(rich: CodeWhaleDecodedCall): ParsedProviderCall { + const measured = rich.measuredCostUSD !== undefined return { - value: safeNonNegativeNumber(cost.session_cost_usd) + safeNonNegativeNumber(cost.subagent_cost_usd), - exact: true, - } -} - -function createParser(source: SessionSource, seenKeys: Set): SessionParser { - return { - async *parse(): AsyncGenerator { - const raw = await readSessionFile(source.path) - let metadata: CodeWhaleMetadata | null = null - let messages: CodeWhaleMessage[] = [] - - if (raw !== null) { - try { - const saved = JSON.parse(raw) as CodeWhaleSession - metadata = parseMetadata(saved.metadata) - messages = Array.isArray(saved.messages) - ? saved.messages.filter(isRecord) as CodeWhaleMessage[] - : [] - } catch { - // A truncated transcript can still have complete, authoritative - // aggregate metadata at the front of the file. - } - } - metadata ??= await readSessionMetadata(source.path) - if (!metadata) return - const totalTokens = safeTokenCount(metadata.total_tokens) - const model = metadata.model ?? metadata.model_provider ?? 'unknown' - const localCost = reportedCost(metadata.cost) - // Match the pre-lift guard exactly: it skipped zero-token sessions whose - // COMPUTED cost was 0 — which included an exact recorded cost of 0. - if (totalTokens === 0 && (!localCost.exact || localCost.value === 0)) return - - const deduplicationKey = `codewhale:${metadata.id}` - if (seenKeys.has(deduplicationKey)) return - seenKeys.add(deduplicationKey) - - let timestamp = normalizeTimestamp(metadata.updated_at) || normalizeTimestamp(metadata.created_at) - if (!timestamp) { - const fileStat = await stat(source.path).catch(() => null) - timestamp = fileStat?.mtime.toISOString() ?? '' - } - - const { tools, bashCommands, toolSequence, skills, subagentTypes, webSearchRequests } = collectTools(messages) - const workspace = metadata.workspace - - yield { - provider: 'codewhale', - model, - // CodeWhale persists only one aggregate token counter. Preserve it - // losslessly in the input column instead of inventing a split. - inputTokens: totalTokens, - outputTokens: 0, - cacheCreationInputTokens: 0, - cacheReadInputTokens: 0, - cachedInputTokens: 0, - reasoningTokens: 0, - webSearchRequests, - ...(localCost.exact - ? { costUSD: localCost.value, costBasis: 'measured' as const } - : { costBasis: 'estimated' as const }), - costIsEstimated: !localCost.exact, - tools, - bashCommands, - skills, - subagentTypes, - timestamp, - speed: 'standard', - deduplicationKey, - turnId: `${metadata.id}:session`, - toolSequence: toolSequence.length > 0 ? toolSequence : undefined, - userMessage: firstUserMessage(messages), - sessionId: metadata.id, - project: projectName(workspace), - projectPath: workspace, - } - }, + provider: 'codewhale', + model: rich.model, + inputTokens: rich.inputTokens, + outputTokens: rich.outputTokens, + cacheCreationInputTokens: rich.cacheCreationInputTokens, + cacheReadInputTokens: rich.cacheReadInputTokens, + cachedInputTokens: rich.cachedInputTokens, + reasoningTokens: rich.reasoningTokens, + webSearchRequests: rich.webSearchRequests, + ...(measured + ? { costUSD: rich.measuredCostUSD, costBasis: 'measured' as const } + : { costBasis: 'estimated' as const }), + costIsEstimated: !measured, + tools: rich.tools, + // The legacy codewhale decode deduped nothing: it pushed every extracted base + // command into a flat list. Preserve that (no Set) so per-command counts match. + bashCommands: rich.rawBashCommands.flatMap(c => extractBashCommands(c)), + skills: rich.skills, + subagentTypes: rich.subagentTypes, + timestamp: rich.timestamp, + speed: rich.speed, + deduplicationKey: rich.deduplicationKey, + turnId: rich.turnId, + toolSequence: rich.toolSequence, + userMessage: rich.userMessage, + sessionId: rich.sessionId, + project: rich.project, + projectPath: rich.projectPath, } } @@ -446,7 +227,7 @@ export function createCodeWhaleProvider(overrideDirs?: string | string[]): Provi ? undefined : Array.isArray(overrideDirs) ? overrideDirs : [overrideDirs] - return { + return createBridgedProvider({ name: 'codewhale', displayName: 'CodeWhale', @@ -455,15 +236,13 @@ export function createCodeWhaleProvider(overrideDirs?: string | string[]): Provi }, toolDisplayName(rawTool: string): string { - return mapToolName(rawTool) + return mapCodeWhaleToolName(rawTool) }, async discoverSessions(): Promise { const seenSessionIds = new Set() const sources: SessionSource[] = [] - // Primary comes before legacy so an id already migrated by CodeWhale is - // not counted twice and the primary copy wins without modifying either. for (const dir of configuredDirs ?? defaultSessionDirs()) { for (const candidate of await discoverInDir(dir)) { if (seenSessionIds.has(candidate.id)) continue @@ -474,10 +253,36 @@ export function createCodeWhaleProvider(overrideDirs?: string | string[]): Provi return sources }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { - return createParser(source, seenKeys) + async readRecords(source: SessionSource): Promise { + const [raw, fileStat] = await Promise.all([ + readSessionFile(source.path), + stat(source.path).catch(() => null), + ]) + + let metadata: CodeWhaleMetadata | null = null + let messages: CodeWhaleMessage[] = [] + + if (raw !== null) { + try { + const saved = JSON.parse(raw) as { metadata?: unknown; messages?: unknown } + metadata = parseMetadata(saved.metadata) + messages = Array.isArray(saved.messages) ? (saved.messages.filter(isRecord) as CodeWhaleMessage[]) : [] + } catch { + // A truncated transcript can still have complete aggregate metadata at + // the front of the file. + } + } + metadata ??= await readSessionMetadata(source.path) + if (!metadata) return null + + const fileMtime = fileStat?.mtime.toISOString() ?? '' + const record: CodeWhaleSessionRecords = { metadata, messages, fileMtime } + return [record] }, - } + + decode: decodeCodeWhale, + toProviderCall, + }) } export const codewhale = createCodeWhaleProvider() diff --git a/packages/cli/src/providers/grok.ts b/packages/cli/src/providers/grok.ts index 271f15d..760e94d 100644 --- a/packages/cli/src/providers/grok.ts +++ b/packages/cli/src/providers/grok.ts @@ -2,66 +2,20 @@ import { readdir, stat } from 'fs/promises' import { basename, dirname, join } from 'path' import { homedir } from 'os' +import { decodeGrok, grokToolNameMap } from '@codeburn/core/providers/grok' +import type { GrokDecodedCall, GrokSessionRecords, GrokSignals, GrokSummary } from '@codeburn/core/providers/grok' + import { readSessionFile } from '../fs-utils.js' import { getShortModelName } from '../models.js' import { extractBashCommands } from '../bash-utils.js' -import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js' - -// Grok Build (xAI's coding CLI) stores one session per directory at -// /sessions///, where grok-home is $GROK_HOME -// or ~/.grok. Each session dir holds summary.json, signals.json, and the ACP -// log updates.jsonl. -// -// Grok does NOT record billable input/output tokens. signals.json carries -// `contextTokensUsed` (current context fill) and updates.jsonl carries a running -// `_meta.totalTokens` per streamed chunk; there is no per-call input/output -// split. We reconstruct an ESTIMATE from the per-turn totalTokens curve. Agentic -// turns re-send the growing context every call, and that re-sent context is -// cached server-side, so we bill the unique context (summed per compaction segment) as fresh input, -// the re-sent remainder as cache reads, and the per-turn growth as output. Cost -// is flagged estimated; grok-build is priced via its grok-build-0.1 alias. - -const toolNameMap: Record = { - bash: 'Bash', - run_terminal_command: 'Bash', - read_file: 'Read', - read: 'Read', - write_file: 'Write', - edit_file: 'Edit', - edit: 'Edit', - list_dir: 'Glob', - glob: 'Glob', - grep: 'Grep', - search: 'WebSearch', - web_search: 'WebSearch', - fetch: 'WebFetch', - task: 'Agent', - search_replace: 'Edit', - todo_write: 'TodoWrite', - spawn_subagent: 'Agent', -} +import { createBridgedProvider } from './bridge.js' +import type { Provider, SessionSource, ParsedProviderCall } from './types.js' function defaultSessionsDir(): string { const home = process.env['GROK_HOME'] ?? join(homedir(), '.grok') return join(home, 'sessions') } -type GrokSummary = { - info?: { id?: string; cwd?: string } - created_at?: string - updated_at?: string - last_active_at?: string - current_model_id?: string - session_summary?: string - generated_title?: string -} - -type GrokSignals = { - primaryModelId?: string - modelsUsed?: string[] - toolsUsed?: string[] -} - async function readJson(path: string): Promise { const content = await readSessionFile(path) if (content === null) return null @@ -80,138 +34,6 @@ function safeDecode(name: string): string { } } -// updates.jsonl is one ACP JSON-RPC notification per line; streamed chunks carry -// params._meta.{totalTokens, promptId}. totalTokens is the running context size, -// so grouping by promptId (one per turn) gives each turn's first/last value. -type GrokUpdate = { - params?: { - _meta?: { totalTokens?: number; promptId?: string } - update?: { sessionUpdate?: string; title?: string; rawInput?: { command?: unknown; subagent_type?: unknown } } - } -} - -// Single pass over updates.jsonl: per-turn totalTokens for the cost estimate, -// plus the real tool calls (each tool_call's title -> a tool, and -// run_terminal_command's rawInput.command -> shell commands). -function parseUpdates(updates: string): { - input: number - cacheRead: number - output: number - tools: string[] - bashCommands: string[] - subagentTypes: string[] -} { - const turns = new Map() - const tools: string[] = [] - const bashCommands: string[] = [] - const subagentTypes: string[] = [] - // Compaction-aware fresh input: a large drop in totalTokens means the context - // was compacted and rebuilt, so we sum each segment's peak rather than the - // single global peak (which would lose everything before the last compaction). - let prevTotal = -1 - let segmentPeak = 0 - let inputFresh = 0 - - for (const line of updates.split('\n')) { - if (!line.trim()) continue - let params: GrokUpdate['params'] - try { - params = (JSON.parse(line) as GrokUpdate).params - } catch { - continue - } - if (!params) continue - - const total = params._meta?.totalTokens - if (typeof total === 'number') { - if (prevTotal >= 0 && total < prevTotal * 0.5) { - inputFresh += segmentPeak // close the segment a compaction just ended - segmentPeak = 0 - } - if (total > segmentPeak) segmentPeak = total - prevTotal = total - - const promptId = params._meta?.promptId - if (promptId) { - const turn = turns.get(promptId) - if (!turn) turns.set(promptId, { first: total, last: total }) - else turn.last = total - } - } - - const update = params.update - if (update?.sessionUpdate === 'tool_call' && typeof update.title === 'string') { - tools.push(toolNameMap[update.title] ?? update.title) - if (update.title === 'run_terminal_command' && typeof update.rawInput?.command === 'string') { - bashCommands.push(...extractBashCommands(update.rawInput.command)) - } - if (update.title === 'spawn_subagent' && typeof update.rawInput?.subagent_type === 'string') { - subagentTypes.push(update.rawInput.subagent_type) - } - } - } - - inputFresh += segmentPeak // close the final segment - let sumFirst = 0 - let output = 0 - for (const { first, last } of turns.values()) { - sumFirst += first - output += Math.max(0, last - first) - } - // Fresh input (summed segment peaks) is billed once; the rest of the per-turn - // re-sends are cache reads (Grok caches them, even though it reports nothing). - const cacheRead = Math.max(0, sumFirst - inputFresh) - return { input: inputFresh, cacheRead, output, tools, bashCommands, subagentTypes } -} - -function createParser(source: SessionSource, seenKeys: Set): SessionParser { - return { - async *parse(): AsyncGenerator { - const dir = dirname(source.path) - const summary = await readJson(join(dir, 'summary.json')) - const updates = await readSessionFile(source.path) - if (!summary || updates === null) return - - const { input, cacheRead, output, tools, bashCommands, subagentTypes } = parseUpdates(updates) - if (input === 0 && output === 0) return - - const signals = await readJson(join(dir, 'signals.json')) - const model = - summary.current_model_id ?? signals?.primaryModelId ?? signals?.modelsUsed?.[0] ?? 'grok-build' - const timestamp = summary.updated_at ?? summary.last_active_at ?? summary.created_at ?? '' - const sessionId = summary.info?.id ?? basename(dir) - - const dedupKey = `${source.provider}:${dir}:${timestamp}:${sessionId}` - if (seenKeys.has(dedupKey)) return - seenKeys.add(dedupKey) - - yield { - provider: source.provider, - model, - inputTokens: input, - outputTokens: output, - cacheCreationInputTokens: 0, - cacheReadInputTokens: cacheRead, - cachedInputTokens: cacheRead, - reasoningTokens: 0, - webSearchRequests: 0, - costBasis: 'estimated', - costIsEstimated: true, - tools, - bashCommands, - subagentTypes, - timestamp, - speed: 'standard', - deduplicationKey: dedupKey, - userMessage: summary.session_summary ?? summary.generated_title ?? '', - sessionId, - project: source.project, - projectPath: summary.info?.cwd, - } - }, - } -} - async function discoverSessions(sessionsDir: string): Promise { const sources: SessionSource[] = [] @@ -250,10 +72,38 @@ async function discoverSessions(sessionsDir: string): Promise { return sources } +function toProviderCall(rich: GrokDecodedCall): ParsedProviderCall { + return { + provider: 'grok', + model: rich.model, + inputTokens: rich.inputTokens, + outputTokens: rich.outputTokens, + cacheCreationInputTokens: rich.cacheCreationInputTokens, + cacheReadInputTokens: rich.cacheReadInputTokens, + cachedInputTokens: rich.cachedInputTokens, + reasoningTokens: rich.reasoningTokens, + webSearchRequests: rich.webSearchRequests, + costBasis: 'estimated', + costIsEstimated: true, + tools: rich.tools, + // The legacy grok decode deduped nothing: it pushed every extracted base + // command into a flat list. Preserve that (no Set) so per-command counts match. + bashCommands: rich.rawBashCommands.flatMap(c => extractBashCommands(c)), + subagentTypes: rich.subagentTypes, + timestamp: rich.timestamp, + speed: rich.speed, + deduplicationKey: rich.deduplicationKey, + userMessage: rich.userMessage, + sessionId: rich.sessionId, + project: rich.project, + projectPath: rich.projectPath, + } +} + export function createGrokProvider(sessionsDir?: string): Provider { const dir = sessionsDir ?? defaultSessionsDir() - return { + return createBridgedProvider({ name: 'grok', displayName: 'Grok Build', @@ -263,17 +113,38 @@ export function createGrokProvider(sessionsDir?: string): Provider { }, toolDisplayName(rawTool: string): string { - return toolNameMap[rawTool] ?? rawTool + return grokToolNameMap[rawTool] ?? rawTool }, async discoverSessions(): Promise { return discoverSessions(dir) }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { - return createParser(source, seenKeys) + async readRecords(source: SessionSource): Promise { + const sessionDir = dirname(source.path) + const [summary, updatesRaw] = await Promise.all([ + readJson(join(sessionDir, 'summary.json')), + readSessionFile(source.path), + ]) + if (!summary || updatesRaw === null) return null + + const signals = await readJson(join(sessionDir, 'signals.json')) + const cwdEncoded = basename(dirname(sessionDir)) + const cwd = summary.info?.cwd ?? safeDecode(cwdEncoded) + const record: GrokSessionRecords = { + summary, + signals, + updatesLines: updatesRaw.split('\n').filter(l => l.trim()), + sourceDir: sessionDir, + sessionName: basename(sessionDir), + project: basename(cwd), + } + return [record] }, - } + + decode: decodeGrok, + toProviderCall, + }) } export const grok = createGrokProvider() diff --git a/packages/cli/src/providers/kimi.ts b/packages/cli/src/providers/kimi.ts index 3a8350d..b65ef7b 100644 --- a/packages/cli/src/providers/kimi.ts +++ b/packages/cli/src/providers/kimi.ts @@ -3,40 +3,19 @@ import { readdir, readFile, stat } from 'fs/promises' import { basename, dirname, join } from 'path' import { homedir } from 'os' +import { decodeKimi, kimiToolNameMap } from '@codeburn/core/providers/kimi' +import type { KimiDecodedCall } from '@codeburn/core/providers/kimi' + import { extractBashCommands } from '../bash-utils.js' -import { readSessionLines } from '../fs-utils.js' +import { readSessionFile } from '../fs-utils.js' import { getShortModelName } from '../models.js' -import type { ParsedProviderCall, Provider, SessionParser, SessionSource } from './types.js' +import { createBridgedProvider } from './bridge.js' +import type { Provider, SessionSource, ParsedProviderCall } from './types.js' type JsonObject = Record -const toolNameMap: Record = { - Shell: 'Bash', - Bash: 'Bash', - bash: 'Bash', - ReadFile: 'Read', - ReadMediaFile: 'Read', - WriteFile: 'Write', - StrReplaceFile: 'Edit', - Grep: 'Grep', - Glob: 'Glob', - SearchWeb: 'WebSearch', - FetchURL: 'WebFetch', - Agent: 'Agent', - AgentTool: 'Agent', - TaskList: 'Agent', - TaskOutput: 'Agent', - TaskStop: 'Agent', - AskUserQuestion: 'AskUser', - SetTodoList: 'TodoWrite', - Think: 'Think', - EnterPlanMode: 'EnterPlanMode', - ExitPlanMode: 'ExitPlanMode', - SendDMail: 'DMail', -} - function asObject(value: unknown): JsonObject | null { - return value && typeof value === 'object' && !Array.isArray(value) ? value as JsonObject : null + return value && typeof value === 'object' && !Array.isArray(value) ? (value as JsonObject) : null } function stringField(obj: JsonObject | null, key: string): string | undefined { @@ -44,15 +23,6 @@ function stringField(obj: JsonObject | null, key: string): string | undefined { return typeof value === 'string' ? value : undefined } -function numericField(obj: JsonObject, ...keys: string[]): number { - for (const key of keys) { - const raw = obj[key] - const n = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : NaN - if (Number.isFinite(n) && n > 0) return Math.trunc(n) - } - return 0 -} - function getShareDir(overrideDir?: string): string { return overrideDir ?? process.env['KIMI_SHARE_DIR'] ?? join(homedir(), '.kimi') } @@ -164,177 +134,38 @@ async function getConfiguredModel(shareDir: string): Promise { return parseModelIdForKey(raw, defaultModel) ?? defaultModel } -function parseJsonObject(text: string | undefined): JsonObject | null { - if (!text) return null - try { - return asObject(JSON.parse(text)) - } catch { - return null - } -} - -function extractUserText(value: unknown): string { - if (typeof value === 'string') return value.slice(0, 500) - if (!Array.isArray(value)) return '' - - return value - .map(part => stringField(asObject(part), 'text') ?? '') - .filter(Boolean) - .join(' ') - .slice(0, 500) -} - -function timestampToIso(value: unknown): string { - if (typeof value === 'string') return value - if (typeof value !== 'number' || !Number.isFinite(value)) return '' - - const millis = value > 1_000_000_000_000 ? value : value * 1000 - const date = new Date(millis) - return Number.isFinite(date.getTime()) ? date.toISOString() : '' -} - -function extractEnvelope(record: JsonObject): { type: string; payload: JsonObject; timestamp: string } | null { - const message = asObject(record['message']) - const envelope = message ?? record - const type = stringField(envelope, 'type') - const payload = asObject(envelope['payload']) - if (!type || !payload) return null - return { type, payload, timestamp: timestampToIso(record['timestamp']) } -} - -function extractUsage(payload: JsonObject): { - inputTokens: number - outputTokens: number - cacheReadInputTokens: number - cacheCreationInputTokens: number -} | null { - const usage = asObject(payload['token_usage']) ?? asObject(payload['usage']) - if (!usage) return null - - const cacheReadInputTokens = numericField(usage, 'input_cache_read', 'cache_read_input_tokens', 'cached_input_tokens') - const cacheCreationInputTokens = numericField(usage, 'input_cache_creation', 'cache_creation_input_tokens') - let inputTokens = numericField(usage, 'input_other', 'input_tokens') - if (inputTokens === 0) { - const totalInput = numericField(usage, 'input') - inputTokens = Math.max(0, totalInput - cacheReadInputTokens - cacheCreationInputTokens) - } - const outputTokens = numericField(usage, 'output', 'output_tokens') - - if (inputTokens === 0 && outputTokens === 0 && cacheReadInputTokens === 0 && cacheCreationInputTokens === 0) { - return null - } - - return { inputTokens, outputTokens, cacheReadInputTokens, cacheCreationInputTokens } -} - -function extractTool(payload: JsonObject): { tool: string; bashCommands: string[] } | null { - const fn = asObject(payload['function']) - const rawName = stringField(fn, 'name') ?? stringField(payload, 'name') - if (!rawName) return null - - const tool = toolNameMap[rawName] ?? rawName - const argsText = stringField(fn, 'arguments') ?? stringField(payload, 'arguments') - const args = parseJsonObject(argsText) - const command = stringField(args, 'command') - const bashCommands = tool === 'Bash' && command ? extractBashCommands(command) : [] - - return { tool, bashCommands } -} - -function createParser(source: SessionSource, shareDir: string, seenKeys: Set): SessionParser { - return { - async *parse(): AsyncGenerator { - const configuredModel = await getConfiguredModel(shareDir) - const tools = new Set() - const bashCommands = new Set() - let currentUserMessage = '' - const sessionId = basename(dirname(source.path)) - let index = 0 - - for await (const line of readSessionLines(source.path)) { - if (!line.trim()) continue - - let record: JsonObject | null = null - try { - record = asObject(JSON.parse(line)) - } catch { - continue - } - if (!record) continue - - const envelope = extractEnvelope(record) - if (!envelope || envelope.type === 'metadata') continue - - if (envelope.type === 'TurnBegin' || envelope.type === 'SteerInput') { - currentUserMessage = extractUserText(envelope.payload['user_input']) - continue - } - - if (envelope.type === 'TurnEnd') { - currentUserMessage = '' - tools.clear() - bashCommands.clear() - continue - } - - if (envelope.type === 'ToolCall' || envelope.type === 'ToolCallRequest') { - const extracted = extractTool(envelope.payload) - if (!extracted) continue - tools.add(extracted.tool) - for (const command of extracted.bashCommands) bashCommands.add(command) - continue - } - - if (envelope.type !== 'StatusUpdate') continue - - const usage = extractUsage(envelope.payload) - if (!usage) continue - - const rawMessageId = stringField(envelope.payload, 'message_id') - const dedupKey = `kimi:${sessionId}:${rawMessageId ?? index}` - index++ - if (seenKeys.has(dedupKey)) continue - seenKeys.add(dedupKey) - - const model = stringField(envelope.payload, 'model') ?? stringField(envelope.payload, 'model_name') ?? configuredModel - - yield { - provider: 'kimi', - model, - inputTokens: usage.inputTokens, - outputTokens: usage.outputTokens, - cacheCreationInputTokens: usage.cacheCreationInputTokens, - cacheReadInputTokens: usage.cacheReadInputTokens, - cachedInputTokens: usage.cacheReadInputTokens, - reasoningTokens: 0, - webSearchRequests: 0, - costBasis: 'estimated', - tools: [...tools], - bashCommands: [...bashCommands], - timestamp: envelope.timestamp, - speed: 'standard', - deduplicationKey: dedupKey, - userMessage: currentUserMessage, - sessionId, - } - - tools.clear() - bashCommands.clear() - } - }, - } -} - async function addWireSource(sources: SessionSource[], filePath: string, project: string): Promise { const s = await stat(filePath).catch(() => null) if (!s?.isFile()) return sources.push({ path: filePath, project, provider: 'kimi' }) } +function toProviderCall(rich: KimiDecodedCall): ParsedProviderCall { + return { + provider: 'kimi', + model: rich.model, + inputTokens: rich.inputTokens, + outputTokens: rich.outputTokens, + cacheCreationInputTokens: rich.cacheCreationInputTokens, + cacheReadInputTokens: rich.cacheReadInputTokens, + cachedInputTokens: rich.cachedInputTokens, + reasoningTokens: rich.reasoningTokens, + webSearchRequests: rich.webSearchRequests, + costBasis: 'estimated', + tools: rich.tools, + bashCommands: [...new Set(rich.rawBashCommands.flatMap(c => extractBashCommands(c)))], + timestamp: rich.timestamp, + speed: rich.speed, + deduplicationKey: rich.deduplicationKey, + userMessage: rich.userMessage, + sessionId: rich.sessionId, + } +} + export function createKimiProvider(overrideDir?: string): Provider { const shareDir = getShareDir(overrideDir) - return { + return createBridgedProvider({ name: 'kimi', displayName: 'Kimi', @@ -343,7 +174,7 @@ export function createKimiProvider(overrideDir?: string): Provider { }, toolDisplayName(rawTool: string): string { - return toolNameMap[rawTool] ?? rawTool + return kimiToolNameMap[rawTool] ?? rawTool }, async discoverSessions(): Promise { @@ -377,10 +208,22 @@ export function createKimiProvider(overrideDir?: string): Provider { return sources }, - createSessionParser(source: SessionSource, seenKeys: Set): SessionParser { - return createParser(source, shareDir, seenKeys) + async readRecords(source: SessionSource): Promise { + const [configuredModel, raw] = await Promise.all([ + getConfiguredModel(shareDir), + readSessionFile(source.path), + ]) + if (raw === null) return null + return [{ + lines: raw.split('\n').filter(l => l.trim()), + configuredModel, + sessionName: basename(dirname(source.path)), + }] }, - } + + decode: decodeKimi, + toProviderCall, + }) } export const kimi = createKimiProvider() diff --git a/packages/cli/tests/fixtures/codewhale-parity/sessions/full.json b/packages/cli/tests/fixtures/codewhale-parity/sessions/full.json new file mode 100644 index 0000000..5056eb3 --- /dev/null +++ b/packages/cli/tests/fixtures/codewhale-parity/sessions/full.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "metadata": { + "id": "session-full", + "title": "CodeWhale test session", + "created_at": "2026-07-14T10:00:00.000Z", + "updated_at": "2026-07-15T12:34:56.000Z", + "message_count": 6, + "total_tokens": 12345, + "model": "anthropic/claude-sonnet-4-6", + "model_provider": "anthropic", + "workspace": "/Users/alice/codewhale-demo", + "mode": "agent", + "cost": { + "session_cost_usd": 0.75, + "subagent_cost_usd": 0.20, + "displayed_cost_high_water_usd": 2 + } + }, + "messages": [ + { "role": "user", "content": [{ "type": "text", "text": "Implement the parser" }] }, + { + "role": "assistant", + "content": [ + { "type": "thinking", "thinking": "..." }, + { "type": "tool_use", "id": "t1", "name": "read_file", "input": { "file_path": "src/app.ts" } }, + { "type": "tool_use", "id": "t2", "name": "exec_shell", "input": { "command": "npm test && git status && npm run build" } }, + { "type": "tool_use", "id": "t3", "name": "edit_file", "input": { "path": "src/app.ts" } }, + { "type": "tool_use", "id": "t3b", "name": "read_file", "input": { "file_path": "src/other.ts" } } + ] + }, + { + "role": "assistant", + "content": [ + { "type": "tool_use", "id": "t4", "name": "load_skill", "input": { "name": "typescript" } }, + { "type": "tool_use", "id": "t5", "name": "agent", "input": { "type": "reviewer" } }, + { "type": "server_tool_use", "id": "t6", "name": "web_search", "input": { "query": "CodeWhale" } } + ] + } + ], + "system_prompt": null +} diff --git a/packages/cli/tests/fixtures/grok-parity/%2FUsers%2Ftest/019edf9c-0000-7000-8000-000000000001/signals.json b/packages/cli/tests/fixtures/grok-parity/%2FUsers%2Ftest/019edf9c-0000-7000-8000-000000000001/signals.json new file mode 100644 index 0000000..e0ee031 --- /dev/null +++ b/packages/cli/tests/fixtures/grok-parity/%2FUsers%2Ftest/019edf9c-0000-7000-8000-000000000001/signals.json @@ -0,0 +1,7 @@ +{ + "primaryModelId": "grok-build", + "modelsUsed": ["grok-build"], + "toolsUsed": ["read_file", "run_terminal_command", "grep"], + "contextTokensUsed": 40000, + "contextWindowTokens": 512000 +} diff --git a/packages/cli/tests/fixtures/grok-parity/%2FUsers%2Ftest/019edf9c-0000-7000-8000-000000000001/summary.json b/packages/cli/tests/fixtures/grok-parity/%2FUsers%2Ftest/019edf9c-0000-7000-8000-000000000001/summary.json new file mode 100644 index 0000000..6ae7ff2 --- /dev/null +++ b/packages/cli/tests/fixtures/grok-parity/%2FUsers%2Ftest/019edf9c-0000-7000-8000-000000000001/summary.json @@ -0,0 +1,13 @@ +{ + "info": { + "id": "019edf9c-0000-7000-8000-000000000001", + "cwd": "/Users/test/myproject" + }, + "created_at": "2026-06-19T11:20:40.686261Z", + "updated_at": "2026-06-19T11:31:12.282793Z", + "last_active_at": "2026-06-19T11:31:12.222328Z", + "num_messages": 42, + "current_model_id": "grok-build", + "session_summary": "User asks about the repo", + "generated_title": "User asks about the repo" +} diff --git a/packages/cli/tests/fixtures/grok-parity/%2FUsers%2Ftest/019edf9c-0000-7000-8000-000000000001/updates.jsonl b/packages/cli/tests/fixtures/grok-parity/%2FUsers%2Ftest/019edf9c-0000-7000-8000-000000000001/updates.jsonl new file mode 100644 index 0000000..481aca0 --- /dev/null +++ b/packages/cli/tests/fixtures/grok-parity/%2FUsers%2Ftest/019edf9c-0000-7000-8000-000000000001/updates.jsonl @@ -0,0 +1,11 @@ +{"timestamp":"2026-06-19T11:30:00.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"hi"}},"_meta":{"totalTokens":20000,"promptId":"p1","updateType":"AgentMessageChunk","modelId":"grok-build"}}} +{"timestamp":"2026-06-19T11:30:00.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"hi"}},"_meta":{"totalTokens":25000,"promptId":"p1","updateType":"AgentMessageChunk","modelId":"grok-build"}}} +{"timestamp":"2026-06-19T11:30:00.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"hi"}},"_meta":{"totalTokens":30000,"promptId":"p2","updateType":"AgentMessageChunk","modelId":"grok-build"}}} +{"timestamp":"2026-06-19T11:30:00.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"hi"}},"_meta":{"totalTokens":35000,"promptId":"p2","updateType":"AgentMessageChunk","modelId":"grok-build"}}} +{"timestamp":"2026-06-19T11:30:00.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"hi"}},"_meta":{"totalTokens":40000,"promptId":"p3","updateType":"AgentMessageChunk","modelId":"grok-build"}}} +{"timestamp":"2026-06-19T11:30:00.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"hi"}},"_meta":{"totalTokens":45000,"promptId":"p3","updateType":"AgentMessageChunk","modelId":"grok-build"}}} +{"timestamp":"2026-06-19T11:30:05.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"tool_call","toolCallId":"c1","title":"read_file","rawInput":{"target_directory":"."}}}} +{"timestamp":"2026-06-19T11:30:05.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"tool_call","toolCallId":"c2","title":"grep","rawInput":{"pattern":"x"}}}} +{"timestamp":"2026-06-19T11:30:05.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"tool_call","toolCallId":"c3","title":"run_terminal_command","rawInput":{"command":"git status && git log"}}}} +{"timestamp":"2026-06-19T11:30:05.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"tool_call","toolCallId":"c4","title":"run_terminal_command","rawInput":{"command":"git diff"}}}} +{"timestamp":"2026-06-19T11:30:05.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"tool_call","toolCallId":"c5","title":"spawn_subagent","rawInput":{"subagent_type":"general-purpose","prompt":"x"}}}} diff --git a/packages/cli/tests/fixtures/kimi-parity/config.toml b/packages/cli/tests/fixtures/kimi-parity/config.toml new file mode 100644 index 0000000..4f280c9 --- /dev/null +++ b/packages/cli/tests/fixtures/kimi-parity/config.toml @@ -0,0 +1,4 @@ +default_model = "kimi-code/k2" + +[models."kimi-code/k2"] +model = "kimi-k2-thinking-turbo" diff --git a/packages/cli/tests/fixtures/kimi-parity/kimi.json b/packages/cli/tests/fixtures/kimi-parity/kimi.json new file mode 100644 index 0000000..c6eeabc --- /dev/null +++ b/packages/cli/tests/fixtures/kimi-parity/kimi.json @@ -0,0 +1,5 @@ +{ + "work_dirs": [ + { "path": "/Users/test/work/app", "kaos": "local", "last_session_id": "sess-1" } + ] +} diff --git a/packages/cli/tests/fixtures/kimi-parity/sessions/98a2ba96a5242fc8b3fa26717ef660ba/sess-1/wire.jsonl b/packages/cli/tests/fixtures/kimi-parity/sessions/98a2ba96a5242fc8b3fa26717ef660ba/sess-1/wire.jsonl new file mode 100644 index 0000000..ba27376 --- /dev/null +++ b/packages/cli/tests/fixtures/kimi-parity/sessions/98a2ba96a5242fc8b3fa26717ef660ba/sess-1/wire.jsonl @@ -0,0 +1,7 @@ +{"type":"metadata","protocol_version":"2"} +{"timestamp":1776162400,"message":{"type":"TurnBegin","payload":{"user_input":"add status endpoint"}}} +{"timestamp":1776162401,"message":{"type":"ToolCall","payload":{"type":"function","id":"call-shell","function":{"name":"Shell","arguments":"{\"command\":\"git status && npm test\"}"}}}} +{"timestamp":1776162402,"message":{"type":"ToolCall","payload":{"type":"function","id":"call-read","function":{"name":"ReadFile","arguments":"{\"path\":\"src/index.ts\"}"}}}} +{"timestamp":1776162403,"message":{"type":"ToolCall","payload":{"type":"function","id":"call-read2","function":{"name":"ReadFile","arguments":"{\"path\":\"src/other.ts\"}"}}}} +{"timestamp":1776162404,"message":{"type":"ToolCall","payload":{"type":"function","id":"call-shell2","function":{"name":"Shell","arguments":"{\"command\":\"git status && npm test\"}"}}}} +{"timestamp":1776162405,"message":{"type":"StatusUpdate","payload":{"message_id":"msg-1","token_usage":{"input_other":100,"input_cache_read":25,"input_cache_creation":10,"output":40}}}} diff --git a/packages/cli/tests/providers/codewhale-bridge.test.ts b/packages/cli/tests/providers/codewhale-bridge.test.ts new file mode 100644 index 0000000..b4ad476 --- /dev/null +++ b/packages/cli/tests/providers/codewhale-bridge.test.ts @@ -0,0 +1,104 @@ +import { dirname, resolve } from 'path' +import { fileURLToPath } from 'url' + +import { describe, it, expect } from 'vitest' + +import { createCodeWhaleProvider } from '../../src/providers/codewhale.js' +import { priceProviderCall } from '../../src/pricing-pass.js' +import type { ParsedProviderCall, SessionSource, ToolCall } from '../../src/providers/types.js' + +// Byte-identical parity gate for the codewhale bridge migration (phase 8). The +// GOLDEN below was captured from the legacy in-CLI decode before the migration. + +const here = dirname(fileURLToPath(import.meta.url)) +const FIXTURE_DIR = resolve(here, '../fixtures/codewhale-parity/sessions') + +const GOLDEN: ParsedProviderCall[] = [ + { + provider: 'codewhale', + model: 'anthropic/claude-sonnet-4-6', + inputTokens: 12345, + outputTokens: 0, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests: 1, + costUSD: 0.95, + costBasis: 'measured', + costIsEstimated: false, + // CodeWhale dedups nothing: Read appears twice (two read_file calls) and + // 'npm' repeats across `npm test && git status && npm run build`. + tools: ['Read', 'Bash', 'Edit', 'Read', 'Skill', 'Agent', 'WebSearch'], + bashCommands: ['npm', 'git', 'npm'], + skills: ['typescript'], + subagentTypes: ['reviewer'], + timestamp: '2026-07-15T12:34:56.000Z', + speed: 'standard', + deduplicationKey: 'codewhale:session-full', + turnId: 'session-full:session', + toolSequence: [ + [ + { tool: 'Read', file: 'src/app.ts' }, + { tool: 'Bash', command: 'npm test && git status && npm run build' }, + { tool: 'Edit', file: 'src/app.ts' }, + { tool: 'Read', file: 'src/other.ts' }, + ] as ToolCall[], + [ + { tool: 'Skill' }, + { tool: 'Agent' }, + { tool: 'WebSearch' }, + ] as ToolCall[], + ], + userMessage: 'Implement the parser', + sessionId: 'session-full', + project: 'codewhale-demo', + projectPath: '/Users/alice/codewhale-demo', + }, +] + +async function collect(): Promise { + const provider = createCodeWhaleProvider(FIXTURE_DIR) + const sources: SessionSource[] = await provider.discoverSessions() + sources.sort((a, b) => a.path.localeCompare(b.path)) + const seen = new Set() + const calls: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) { + calls.push(call) + } + } + return calls +} + +describe('codewhale bridge — fixture parity', () => { + it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => { + expect(await collect()).toEqual(GOLDEN) + }) + + it('the priced output survives the pricing pass unchanged for measured cost', async () => { + const raw = await collect() + const priced = raw.map(priceProviderCall) + priced.forEach((call, i) => { + // Measured-cost calls already carry costUSD/costBasis; the pricing pass + // leaves them untouched. + expect(call).toEqual(raw[i]) + }) + }) + + it('dedup threads through the host-owned seenKeys set', async () => { + const provider = createCodeWhaleProvider(FIXTURE_DIR) + const sources = await provider.discoverSessions() + const seen = new Set() + const first: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) first.push(call) + } + const second: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) second.push(call) + } + expect(first.length).toBe(1) + expect(second).toEqual([]) + }) +}) diff --git a/packages/cli/tests/providers/grok-bridge.test.ts b/packages/cli/tests/providers/grok-bridge.test.ts new file mode 100644 index 0000000..857d290 --- /dev/null +++ b/packages/cli/tests/providers/grok-bridge.test.ts @@ -0,0 +1,90 @@ +import { dirname, resolve } from 'path' +import { fileURLToPath } from 'url' + +import { describe, it, expect } from 'vitest' + +import { createGrokProvider } from '../../src/providers/grok.js' +import { priceProviderCall } from '../../src/pricing-pass.js' +import type { ParsedProviderCall, SessionSource } from '../../src/providers/types.js' + +// Byte-identical parity gate for the grok bridge migration (phase 8). The +// GOLDEN below was captured from the legacy in-CLI decode before the migration. + +const here = dirname(fileURLToPath(import.meta.url)) +const FIXTURE_DIR = resolve(here, '../fixtures/grok-parity') + +const GOLDEN: ParsedProviderCall[] = [ + { + provider: 'grok', + model: 'grok-build', + inputTokens: 45000, + outputTokens: 15000, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 45000, + cachedInputTokens: 45000, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + costIsEstimated: true, + // Grok dedups nothing: Bash appears twice (two run_terminal_command calls) + // and 'git' repeats across `git status && git log` + `git diff`. + tools: ['Read', 'Grep', 'Bash', 'Bash', 'Agent'], + bashCommands: ['git', 'git', 'git'], + subagentTypes: ['general-purpose'], + timestamp: '2026-06-19T11:31:12.282793Z', + speed: 'standard', + deduplicationKey: 'grok:/Users/torukmakto/Projects/codeburn/codeburn/.claude/worktrees/kimi-p8a/packages/cli/tests/fixtures/grok-parity/%2FUsers%2Ftest/019edf9c-0000-7000-8000-000000000001:2026-06-19T11:31:12.282793Z:019edf9c-0000-7000-8000-000000000001', + userMessage: 'User asks about the repo', + sessionId: '019edf9c-0000-7000-8000-000000000001', + project: 'myproject', + projectPath: '/Users/test/myproject', + }, +] + +async function collect(): Promise { + const provider = createGrokProvider(FIXTURE_DIR) + const sources: SessionSource[] = await provider.discoverSessions() + sources.sort((a, b) => a.path.localeCompare(b.path)) + const seen = new Set() + const calls: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) { + calls.push(call) + } + } + return calls +} + +describe('grok bridge — fixture parity', () => { + it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => { + expect(await collect()).toEqual(GOLDEN) + }) + + it('the priced output survives the pricing pass with only costUSD added', async () => { + const raw = await collect() + const priced = raw.map(priceProviderCall) + priced.forEach((call, i) => { + expect(typeof call.costUSD).toBe('number') + expect(Number.isFinite(call.costUSD)).toBe(true) + expect(call.costBasis).toBe('estimated') + const { costUSD, ...rest } = call + expect(rest).toEqual(raw[i]) + }) + }) + + it('dedup threads through the host-owned seenKeys set', async () => { + const provider = createGrokProvider(FIXTURE_DIR) + const sources = await provider.discoverSessions() + const seen = new Set() + const first: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) first.push(call) + } + const second: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) second.push(call) + } + expect(first.length).toBe(1) + expect(second).toEqual([]) + }) +}) diff --git a/packages/cli/tests/providers/kimi-bridge.test.ts b/packages/cli/tests/providers/kimi-bridge.test.ts new file mode 100644 index 0000000..4c59c64 --- /dev/null +++ b/packages/cli/tests/providers/kimi-bridge.test.ts @@ -0,0 +1,86 @@ +import { dirname, resolve } from 'path' +import { fileURLToPath } from 'url' + +import { describe, it, expect } from 'vitest' + +import { createKimiProvider } from '../../src/providers/kimi.js' +import { priceProviderCall } from '../../src/pricing-pass.js' +import type { ParsedProviderCall, SessionSource } from '../../src/providers/types.js' + +// Byte-identical parity gate for the kimi bridge migration (phase 8). The +// GOLDEN below was captured from the legacy in-CLI decode before the migration. + +const here = dirname(fileURLToPath(import.meta.url)) +const FIXTURE_DIR = resolve(here, '../fixtures/kimi-parity') + +const GOLDEN: ParsedProviderCall[] = [ + { + provider: 'kimi', + model: 'kimi-k2-thinking-turbo', + inputTokens: 100, + outputTokens: 40, + cacheCreationInputTokens: 10, + cacheReadInputTokens: 25, + cachedInputTokens: 25, + reasoningTokens: 0, + webSearchRequests: 0, + costBasis: 'estimated', + // Kimi dedups per turn (Set): the repeated ReadFile and second Shell call + // collapse, so tools and bashCommands stay unique. + tools: ['Bash', 'Read'], + bashCommands: ['git', 'npm'], + timestamp: '2026-04-14T10:26:45.000Z', + speed: 'standard', + deduplicationKey: 'kimi:sess-1:msg-1', + userMessage: 'add status endpoint', + sessionId: 'sess-1', + }, +] + +async function collect(): Promise { + const provider = createKimiProvider(FIXTURE_DIR) + const sources: SessionSource[] = await provider.discoverSessions() + sources.sort((a, b) => a.path.localeCompare(b.path)) + const seen = new Set() + const calls: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) { + calls.push(call) + } + } + return calls +} + +describe('kimi bridge — fixture parity', () => { + it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => { + expect(await collect()).toEqual(GOLDEN) + }) + + it('the priced output survives the pricing pass with only costUSD added', async () => { + const raw = await collect() + const priced = raw.map(priceProviderCall) + priced.forEach((call, i) => { + expect(typeof call.costUSD).toBe('number') + expect(Number.isFinite(call.costUSD)).toBe(true) + expect(call.costBasis).toBe('estimated') + const { costUSD, ...rest } = call + expect(rest).toEqual(raw[i]) + }) + }) + + it('dedup threads through the host-owned seenKeys set', async () => { + const provider = createKimiProvider(FIXTURE_DIR) + const sources = await provider.discoverSessions() + const seen = new Set() + const first: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) first.push(call) + } + const second: ParsedProviderCall[] = [] + for (const source of sources) { + for await (const call of provider.createSessionParser(source, seen).parse()) second.push(call) + } + expect(first.length).toBe(1) + expect(second).toEqual([]) + }) +}) diff --git a/packages/core/package.json b/packages/core/package.json index 165bc53..75c876c 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -39,10 +39,22 @@ "types": "./dist/providers/claude/index.d.ts", "import": "./dist/providers/claude/index.js" }, + "./providers/codewhale": { + "types": "./dist/providers/codewhale/index.d.ts", + "import": "./dist/providers/codewhale/index.js" + }, "./providers/codex": { "types": "./dist/providers/codex/index.d.ts", "import": "./dist/providers/codex/index.js" }, + "./providers/grok": { + "types": "./dist/providers/grok/index.d.ts", + "import": "./dist/providers/grok/index.js" + }, + "./providers/kimi": { + "types": "./dist/providers/kimi/index.d.ts", + "import": "./dist/providers/kimi/index.js" + }, "./providers/qwen": { "types": "./dist/providers/qwen/index.d.ts", "import": "./dist/providers/qwen/index.js" diff --git a/packages/core/src/providers/codewhale/decode.ts b/packages/core/src/providers/codewhale/decode.ts new file mode 100644 index 0000000..a03ad45 --- /dev/null +++ b/packages/core/src/providers/codewhale/decode.ts @@ -0,0 +1,248 @@ +// @codeburn/core CodeWhale decoder: pure decode over a host-supplied parsed +// session. The host reads the JSON file (with an optional prefix fallback for +// oversized transcripts); this decoder is stateless, does no fs/env/clock +// access, and performs no pricing calculations. + +import type { DecodeContext } from '../../contracts.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { + CodeWhaleDecodedCall, + CodeWhaleMessage, + CodeWhaleMetadata, + CodeWhaleSessionRecords, + CodeWhaleToolCall, +} from './types.js' + +export const codeWhaleToolNameMap: Record = { + exec_shell: 'Bash', + exec_shell_wait: 'Bash', + exec_shell_interact: 'Bash', + exec_shell_cancel: 'Bash', + task_shell_start: 'Bash', + task_shell_wait: 'Bash', + terminal_run: 'Bash', + terminal_send: 'Bash', + terminal_wait: 'Bash', + terminal_cancel: 'Bash', + read_file: 'Read', + write_file: 'Write', + edit_file: 'Edit', + fim_edit: 'Edit', + apply_patch: 'Edit', + list_dir: 'Glob', + grep_files: 'Grep', + web_search: 'WebSearch', + fetch_url: 'WebFetch', + 'web.run': 'WebSearch', + agent: 'Agent', + 'agents/list': 'Agent', + 'agents/message': 'Agent', + 'agents/followup': 'Agent', + 'agents/interrupt': 'Agent', + 'agents/wait': 'Agent', + todo_write: 'TodoWrite', + todo_add: 'TodoWrite', + todo_update: 'TodoWrite', + todo_list: 'TodoWrite', + checklist_write: 'TodoWrite', + checklist_add: 'TodoWrite', + checklist_update: 'TodoWrite', + checklist_list: 'TodoWrite', + update_plan: 'TodoWrite', + load_skill: 'Skill', + request_user_input: 'AskUser', +} + +export function mapCodeWhaleToolName(rawName: string): string { + if (rawName.startsWith('mcp__')) return rawName + if (rawName.startsWith('agents/')) return 'Agent' + return Object.prototype.hasOwnProperty.call(codeWhaleToolNameMap, rawName) + ? codeWhaleToolNameMap[rawName]! + : rawName +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) +} + +function nonEmptyString(value: unknown): string | undefined { + return typeof value === 'string' && value.trim() ? value.trim() : undefined +} + +function safeNonNegativeNumber(value: unknown): number { + return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : 0 +} + +function safeTokenCount(value: unknown): number { + return Math.floor(Math.min(safeNonNegativeNumber(value), Number.MAX_SAFE_INTEGER)) +} + +function normalizeTimestamp(value: string | undefined): string { + if (!value) return '' + const timestamp = Date.parse(value) + return Number.isFinite(timestamp) ? new Date(timestamp).toISOString() : '' +} + +function firstUserMessage(messages: CodeWhaleMessage[]): string { + for (const message of messages) { + if (message.role !== 'user') continue + const text = + typeof message.content === 'string' + ? message.content + : Array.isArray(message.content) + ? message.content + .filter(block => block?.type === 'text' && typeof block.text === 'string') + .map(block => block.text) + .join(' ') + : '' + if (text.trim()) return Array.from(text.trim()).slice(0, 500).join('') + } + return '' +} + +function toolInput(value: unknown): Record { + return isRecord(value) ? value : {} +} + +function firstString(input: Record, keys: string[]): string | undefined { + for (const key of keys) { + const value = nonEmptyString(input[key]) + if (value) return value + } + return undefined +} + +function collectTools(messages: CodeWhaleMessage[]): { + tools: string[] + rawBashCommands: string[] + toolSequence: CodeWhaleToolCall[][] + skills: string[] + subagentTypes: string[] + webSearchRequests: number +} { + const tools: string[] = [] + const rawBashCommands: string[] = [] + const toolSequence: CodeWhaleToolCall[][] = [] + const skills: string[] = [] + const subagentTypes: string[] = [] + let webSearchRequests = 0 + + for (const message of messages) { + if (message.role !== 'assistant' || !Array.isArray(message.content)) continue + const turnTools: CodeWhaleToolCall[] = [] + + for (const block of message.content) { + if (block?.type !== 'tool_use' && block?.type !== 'server_tool_use') continue + const rawName = nonEmptyString(block.name) + if (!rawName) continue + const mapped = mapCodeWhaleToolName(rawName) + const input = toolInput(block.input) + const toolCall: CodeWhaleToolCall = { tool: mapped } + + const file = firstString(input, ['file_path', 'path', 'target_file', 'file']) + if (file) toolCall.file = file + const command = firstString(input, ['command', 'cmd']) + if (command) toolCall.command = command + + if (mapped === 'Bash' && command) { + rawBashCommands.push(command) + } + if (mapped === 'Skill') { + const skill = firstString(input, ['name', 'skill', 'skill_name']) + if (skill) skills.push(skill) + } + if (mapped === 'Agent') { + const subagentType = firstString(input, ['type', 'agent_type', 'profile']) + if (subagentType) subagentTypes.push(subagentType) + } + if (mapped === 'WebSearch') webSearchRequests++ + + tools.push(mapped) + turnTools.push(toolCall) + } + + if (turnTools.length > 0) toolSequence.push(turnTools) + } + + return { tools, rawBashCommands, toolSequence, skills, subagentTypes, webSearchRequests } +} + +function reportedCost(cost: CodeWhaleMetadata['cost']): { value: number; exact: boolean } { + if (!cost || typeof cost !== 'object') return { value: 0, exact: false } + const hasSessionCost = Object.prototype.hasOwnProperty.call(cost, 'session_cost_usd') + const hasSubagentCost = Object.prototype.hasOwnProperty.call(cost, 'subagent_cost_usd') + if (!hasSessionCost && !hasSubagentCost) return { value: 0, exact: false } + return { + value: safeNonNegativeNumber(cost.session_cost_usd) + safeNonNegativeNumber(cost.subagent_cost_usd), + exact: true, + } +} + +function isCodeWhaleSessionRecords(value: unknown): value is CodeWhaleSessionRecords { + return value !== null && typeof value === 'object' && 'metadata' in (value as object) +} + +export type CodeWhaleDecodeInput = { + records: unknown[] + context: DecodeContext + seenKeys?: Set +} + +export type CodeWhaleDecodeResult = { + calls: CodeWhaleDecodedCall[] + diagnostics: RecordDiagnostic[] +} + +/** + * Decode one CodeWhale session's composite record into a single rich, cost-free + * call. The host owns file I/O and the live cross-file dedup set; this function + * is pure over the supplied record. + */ +export function decodeCodeWhale({ records, seenKeys: liveSeen }: CodeWhaleDecodeInput): CodeWhaleDecodeResult { + const seen = liveSeen ?? new Set() + const session = records.find(isCodeWhaleSessionRecords) + if (!session) return { calls: [], diagnostics: [] } + + const { metadata, messages, fileMtime } = session + const totalTokens = safeTokenCount(metadata.total_tokens) + const model = metadata.model ?? metadata.model_provider ?? 'unknown' + const localCost = reportedCost(metadata.cost) + if (totalTokens === 0 && (!localCost.exact || localCost.value === 0)) return { calls: [], diagnostics: [] } + + const deduplicationKey = `codewhale:${metadata.id}` + if (seen.has(deduplicationKey)) return { calls: [], diagnostics: [] } + seen.add(deduplicationKey) + + const timestamp = normalizeTimestamp(metadata.updated_at) || normalizeTimestamp(metadata.created_at) || fileMtime + const workspace = metadata.workspace + + const { tools, rawBashCommands, toolSequence, skills, subagentTypes, webSearchRequests } = collectTools(messages) + + const call: CodeWhaleDecodedCall = { + provider: 'codewhale', + model, + inputTokens: totalTokens, + outputTokens: 0, + cacheCreationInputTokens: 0, + cacheReadInputTokens: 0, + cachedInputTokens: 0, + reasoningTokens: 0, + webSearchRequests, + ...(localCost.exact ? { measuredCostUSD: localCost.value } : {}), + tools, + rawBashCommands, + skills, + subagentTypes, + timestamp, + speed: 'standard', + deduplicationKey, + turnId: `${metadata.id}:session`, + ...(toolSequence.length > 0 ? { toolSequence } : {}), + userMessage: firstUserMessage(messages), + sessionId: metadata.id, + project: workspace ? workspace.replace(/[\\/]+$/, '').split(/[\\/]/).filter(Boolean).at(-1) ?? 'CodeWhale' : 'CodeWhale', + projectPath: workspace ?? '', + } + + return { calls: [call], diagnostics: [] } +} diff --git a/packages/core/src/providers/codewhale/index.ts b/packages/core/src/providers/codewhale/index.ts new file mode 100644 index 0000000..4bbdcaa --- /dev/null +++ b/packages/core/src/providers/codewhale/index.ts @@ -0,0 +1,30 @@ +// @codeburn/core CodeWhale provider. +// +// Two layers: +// - Rich pure decode (`decodeCodeWhale`): host-facing, NOT part of the stable +// minimized surface. Pure over the host-supplied parsed session; carries +// content in-memory but performs no pricing and no bash base-name extraction. +// - Minimizing transform (`toObservations`): maps the rich decode into the +// strict observation envelope; the content-smuggling guarantees bind here. + +export { + decodeCodeWhale, + codeWhaleToolNameMap, + mapCodeWhaleToolName, + type CodeWhaleDecodeInput, + type CodeWhaleDecodeResult, +} from './decode.js' + +export { + toObservations, + type RichCodeWhaleSessionDecode, + type CodeWhaleToObservationsContext, +} from './observations.js' + +export type { + CodeWhaleDecodedCall, + CodeWhaleSessionRecords, + CodeWhaleMetadata, + CodeWhaleMessage, + CodeWhaleToolCall, +} from './types.js' diff --git a/packages/core/src/providers/codewhale/observations.ts b/packages/core/src/providers/codewhale/observations.ts new file mode 100644 index 0000000..b726ef3 --- /dev/null +++ b/packages/core/src/providers/codewhale/observations.ts @@ -0,0 +1,91 @@ +// Minimizing transform: rich CodeWhale decode -> the strict observation envelope. +// +// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool +// names cross into the output. Project paths are fingerprinted; user messages, +// commands, and file paths stay behind. + +import { projectRef, sessionRef } from '../../fingerprint.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { CallObservation, SessionObservation } from '../../observations.js' +import { extractResourceRefs } from '../resource-refs.js' +import type { CodeWhaleDecodedCall } from './types.js' + +/** One CodeWhale session's rich decode, as the host holds it before minimization. */ +export interface RichCodeWhaleSessionDecode { + sessionId: string + /** Absolute project path (the session workspace); fingerprinted, never emitted raw. */ + projectPath: string + /** Rich, cost-free calls in decode order (CodeWhale emits one per session). */ + calls: CodeWhaleDecodedCall[] +} + +export interface CodeWhaleToObservationsContext { + /** HMAC key that scopes every fingerprint. */ + privacyKey: string + /** Provider id stamped onto sessions/calls and folded into sessionRef. */ + provider?: string +} + +const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/ + +function toCallObservation(call: CodeWhaleDecodedCall, turnIndex: number, privacyKey: string): CallObservation { + const obs: CallObservation = { + provider: call.provider, + model: call.model, + tokens: { + input: call.inputTokens, + output: call.outputTokens, + reasoning: call.reasoningTokens, + cacheRead: call.cacheReadInputTokens, + cacheCreate: call.cacheCreationInputTokens, + }, + webSearchRequests: call.webSearchRequests, + speed: call.speed, + costBasis: call.measuredCostUSD !== undefined ? 'measured' : 'estimated', + timestamp: call.timestamp, + dedupKey: call.deduplicationKey, + toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)), + turnIndex, + ...extractResourceRefs(privacyKey, call.toolSequence), + } + if (call.measuredCostUSD !== undefined) { + ;(obs as CallObservation & { measuredCostUSD: number }).measuredCostUSD = call.measuredCostUSD + } + return obs +} + +function toSessionObservation( + decode: RichCodeWhaleSessionDecode, + ctx: CodeWhaleToObservationsContext, +): SessionObservation { + const provider = ctx.provider ?? 'codewhale' + const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i, ctx.privacyKey)) + + const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort() + const startedAt = timestamps[0] ?? '' + const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : '' + + const session: SessionObservation = { + sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId), + projectRef: projectRef(ctx.privacyKey, decode.projectPath), + providerId: provider, + startedAt, + ...(endedAt ? { endedAt } : {}), + calls, + turnCount: calls.length, + } + return session +} + +/** + * Map a rich CodeWhale decode into the minimized observation layer. Returns the + * `sessions` array plus any per-record `diagnostics`. + */ +export function toObservations( + decode: RichCodeWhaleSessionDecode | RichCodeWhaleSessionDecode[], + ctx: CodeWhaleToObservationsContext, +): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } { + const decodes = Array.isArray(decode) ? decode : [decode] + const sessions = decodes.map(d => toSessionObservation(d, ctx)) + return { sessions, diagnostics: [] } +} diff --git a/packages/core/src/providers/codewhale/types.ts b/packages/core/src/providers/codewhale/types.ts new file mode 100644 index 0000000..19108ae --- /dev/null +++ b/packages/core/src/providers/codewhale/types.ts @@ -0,0 +1,74 @@ +// Raw record + rich-decode types for the CodeWhale provider. +// +// CodeWhale stores one whole JSON file per session. The host reads the file (and +// falls back to a fast prefix read for oversized transcripts); the decoder is +// pure over the parsed metadata + messages and carries no computed pricing. + +export type CodeWhaleMetadata = { + id: string + created_at?: string + updated_at?: string + total_tokens?: number + model?: string + model_provider?: string + workspace?: string + cost?: { + session_cost_usd?: number + subagent_cost_usd?: number + } +} + +export type CodeWhaleContentBlock = { + type?: string + text?: string + name?: string + input?: unknown +} + +export type CodeWhaleMessage = { + role?: string + content?: string | CodeWhaleContentBlock[] +} + +/** One tool invocation as captured from a CodeWhale transcript. */ +export type CodeWhaleToolCall = { + tool: string + file?: string + command?: string +} + +/** The composite record the host hands to the core decoder for one session. */ +export type CodeWhaleSessionRecords = { + metadata: CodeWhaleMetadata + messages: CodeWhaleMessage[] + /** File mtime as an ISO fallback when metadata timestamps are missing/invalid. */ + fileMtime: string +} + +/** The rich decode of one CodeWhale session, pre-pricing. */ +export type CodeWhaleDecodedCall = { + provider: 'codewhale' + model: string + inputTokens: number + outputTokens: number + cacheCreationInputTokens: number + cacheReadInputTokens: number + cachedInputTokens: number + reasoningTokens: number + webSearchRequests: number + /** Provider-reported dollar cost, present only when the file carried one. */ + measuredCostUSD?: number + tools: string[] + rawBashCommands: string[] + skills: string[] + subagentTypes: string[] + toolSequence?: CodeWhaleToolCall[][] + timestamp: string + speed: 'standard' + deduplicationKey: string + turnId: string + userMessage: string + sessionId: string + project: string + projectPath: string +} diff --git a/packages/core/src/providers/grok/decode.ts b/packages/core/src/providers/grok/decode.ts new file mode 100644 index 0000000..8ac68c7 --- /dev/null +++ b/packages/core/src/providers/grok/decode.ts @@ -0,0 +1,172 @@ +// @codeburn/core Grok decoder: pure decode over a host-supplied composite record. +// The host reads summary.json, signals.json, and the updates.jsonl lines; this +// decoder reconstructs the per-session token buckets and tool list with no fs, +// env, clock, or pricing. + +import type { DecodeContext } from '../../contracts.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { GrokDecodedCall, GrokSessionRecords, GrokSignals, GrokSummary, GrokUpdate } from './types.js' + +// Grok Build tool ids mapped to the canonical vocabulary. Unknown ids pass +// through unchanged so provider-native tools still appear. +export const grokToolNameMap: Record = { + bash: 'Bash', + run_terminal_command: 'Bash', + read_file: 'Read', + read: 'Read', + write_file: 'Write', + edit_file: 'Edit', + edit: 'Edit', + list_dir: 'Glob', + glob: 'Glob', + grep: 'Grep', + search: 'WebSearch', + web_search: 'WebSearch', + fetch: 'WebFetch', + task: 'Agent', + search_replace: 'Edit', + todo_write: 'TodoWrite', + spawn_subagent: 'Agent', +} + +function isGrokSessionRecords(value: unknown): value is GrokSessionRecords { + return ( + value !== null && + typeof value === 'object' && + 'updatesLines' in (value as object) && + 'summary' in (value as object) + ) +} + +// Single pass over updates.jsonl: per-turn totalTokens for the cost estimate, +// plus the real tool calls. Bash commands are kept raw; base-name extraction is +// the host's job. +function parseUpdates(updates: string): { + input: number + cacheRead: number + output: number + tools: string[] + rawBashCommands: string[] + subagentTypes: string[] +} { + const turns = new Map() + const tools: string[] = [] + const rawBashCommands: string[] = [] + const subagentTypes: string[] = [] + + let prevTotal = -1 + let segmentPeak = 0 + let inputFresh = 0 + + for (const line of updates.split('\n')) { + if (!line.trim()) continue + let params: GrokUpdate['params'] + try { + params = (JSON.parse(line) as GrokUpdate).params + } catch { + continue + } + if (!params) continue + + const total = params._meta?.totalTokens + if (typeof total === 'number') { + if (prevTotal >= 0 && total < prevTotal * 0.5) { + inputFresh += segmentPeak + segmentPeak = 0 + } + if (total > segmentPeak) segmentPeak = total + prevTotal = total + + const promptId = params._meta?.promptId + if (promptId) { + const turn = turns.get(promptId) + if (!turn) turns.set(promptId, { first: total, last: total }) + else turn.last = total + } + } + + const update = params.update + if (update?.sessionUpdate === 'tool_call' && typeof update.title === 'string') { + tools.push(grokToolNameMap[update.title] ?? update.title) + if (update.title === 'run_terminal_command' && typeof update.rawInput?.command === 'string') { + rawBashCommands.push(update.rawInput.command) + } + if (update.title === 'spawn_subagent' && typeof update.rawInput?.subagent_type === 'string') { + subagentTypes.push(update.rawInput.subagent_type) + } + } + } + + inputFresh += segmentPeak + let sumFirst = 0 + let output = 0 + for (const { first, last } of turns.values()) { + sumFirst += first + output += Math.max(0, last - first) + } + const cacheRead = Math.max(0, sumFirst - inputFresh) + return { input: inputFresh, cacheRead, output, tools, rawBashCommands, subagentTypes } +} + +export type GrokDecodeInput = { + records: unknown[] + context: DecodeContext + seenKeys?: Set +} + +export type GrokDecodeResult = { + calls: GrokDecodedCall[] + diagnostics: RecordDiagnostic[] +} + +/** + * Decode one Grok session's composite records into a single rich, cost-free call. + * The host owns file I/O and the live cross-file dedup set; this function is + * pure over the supplied record. + */ +export function decodeGrok({ records, seenKeys: liveSeen }: GrokDecodeInput): GrokDecodeResult { + const seen = liveSeen ?? new Set() + const session = records.find(isGrokSessionRecords) + if (!session) return { calls: [], diagnostics: [] } + + const { summary, signals, updatesLines, sourceDir, sessionName, project } = session + const updates = updatesLines.join('\n') + const { input, cacheRead, output, tools, rawBashCommands, subagentTypes } = parseUpdates(updates) + if (input === 0 && output === 0) return { calls: [], diagnostics: [] } + + const model = + summary.current_model_id ?? + signals?.primaryModelId ?? + signals?.modelsUsed?.[0] ?? + 'grok-build' + const timestamp = summary.updated_at ?? summary.last_active_at ?? summary.created_at ?? '' + const sessionId = summary.info?.id ?? sessionName + + const dedupKey = `grok:${sourceDir}:${timestamp}:${sessionId}` + if (seen.has(dedupKey)) return { calls: [], diagnostics: [] } + seen.add(dedupKey) + + const call: GrokDecodedCall = { + provider: 'grok', + model, + inputTokens: input, + outputTokens: output, + cacheCreationInputTokens: 0, + cacheReadInputTokens: cacheRead, + cachedInputTokens: cacheRead, + reasoningTokens: 0, + webSearchRequests: 0, + tools, + rawBashCommands, + subagentTypes, + timestamp, + speed: 'standard', + deduplicationKey: dedupKey, + userMessage: summary.session_summary ?? summary.generated_title ?? '', + sessionId, + projectPath: summary.info?.cwd ?? '', + project, + } + + return { calls: [call], diagnostics: [] } +} diff --git a/packages/core/src/providers/grok/index.ts b/packages/core/src/providers/grok/index.ts new file mode 100644 index 0000000..7bcb41d --- /dev/null +++ b/packages/core/src/providers/grok/index.ts @@ -0,0 +1,29 @@ +// @codeburn/core Grok provider. +// +// Two layers: +// - Rich pure decode (`decodeGrok`): host-facing, NOT part of the stable +// minimized surface. Pure over the host-supplied composite record; carries +// content in-memory but no pricing and no bash base-name extraction. +// - Minimizing transform (`toObservations`): maps the rich decode into the +// strict observation envelope; the content-smuggling guarantees bind here. + +export { + decodeGrok, + grokToolNameMap, + type GrokDecodeInput, + type GrokDecodeResult, +} from './decode.js' + +export { + toObservations, + type RichGrokSessionDecode, + type GrokToObservationsContext, +} from './observations.js' + +export type { + GrokDecodedCall, + GrokSessionRecords, + GrokSummary, + GrokSignals, + GrokUpdate, +} from './types.js' diff --git a/packages/core/src/providers/grok/observations.ts b/packages/core/src/providers/grok/observations.ts new file mode 100644 index 0000000..86318ae --- /dev/null +++ b/packages/core/src/providers/grok/observations.ts @@ -0,0 +1,82 @@ +// Minimizing transform: rich Grok decode -> the strict observation envelope. +// +// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool +// names cross into the output. The project path is fingerprinted; the user +// message and raw bash commands stay behind. + +import { projectRef, sessionRef } from '../../fingerprint.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { CallObservation, SessionObservation } from '../../observations.js' +import type { GrokDecodedCall } from './types.js' + +/** One Grok session's rich decode, as the host holds it before minimization. */ +export interface RichGrokSessionDecode { + sessionId: string + /** Absolute project path (the session cwd); fingerprinted, never emitted raw. */ + projectPath: string + /** Rich, cost-free calls in decode order (Grok emits one per session). */ + calls: GrokDecodedCall[] +} + +export interface GrokToObservationsContext { + /** HMAC key that scopes every fingerprint. */ + privacyKey: string + /** Provider id stamped onto sessions/calls and folded into sessionRef. */ + provider?: string +} + +const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/ + +function toCallObservation(call: GrokDecodedCall, turnIndex: number, privacyKey: string): CallObservation { + return { + provider: call.provider, + model: call.model, + tokens: { + input: call.inputTokens, + output: call.outputTokens, + reasoning: call.reasoningTokens, + cacheRead: call.cacheReadInputTokens, + cacheCreate: call.cacheCreationInputTokens, + }, + webSearchRequests: call.webSearchRequests, + speed: call.speed, + costBasis: 'estimated', + timestamp: call.timestamp, + dedupKey: call.deduplicationKey, + toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)), + turnIndex, + } +} + +function toSessionObservation(decode: RichGrokSessionDecode, ctx: GrokToObservationsContext): SessionObservation { + const provider = ctx.provider ?? 'grok' + const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i, ctx.privacyKey)) + + const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort() + const startedAt = timestamps[0] ?? '' + const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : '' + + const session: SessionObservation = { + sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId), + projectRef: projectRef(ctx.privacyKey, decode.projectPath), + providerId: provider, + startedAt, + ...(endedAt ? { endedAt } : {}), + calls, + turnCount: calls.length, + } + return session +} + +/** + * Map a rich Grok decode into the minimized observation layer. Returns the + * `sessions` array plus any per-record `diagnostics`. + */ +export function toObservations( + decode: RichGrokSessionDecode | RichGrokSessionDecode[], + ctx: GrokToObservationsContext, +): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } { + const decodes = Array.isArray(decode) ? decode : [decode] + const sessions = decodes.map(d => toSessionObservation(d, ctx)) + return { sessions, diagnostics: [] } +} diff --git a/packages/core/src/providers/grok/types.ts b/packages/core/src/providers/grok/types.ts new file mode 100644 index 0000000..92d52ea --- /dev/null +++ b/packages/core/src/providers/grok/types.ts @@ -0,0 +1,71 @@ +// Raw record + rich-decode types for the Grok Build provider. +// +// Grok stores one session per directory: /sessions/// +// with summary.json, signals.json, and updates.jsonl. The host reads those files +// and hands the decoder a single composite record; the decoder is pure over that +// record and carries no pricing. + +export type GrokSummary = { + info?: { id?: string; cwd?: string } + created_at?: string + updated_at?: string + last_active_at?: string + current_model_id?: string + session_summary?: string + generated_title?: string +} + +export type GrokSignals = { + primaryModelId?: string + modelsUsed?: string[] + toolsUsed?: string[] +} + +export type GrokUpdate = { + params?: { + _meta?: { totalTokens?: number; promptId?: string } + update?: { + sessionUpdate?: string + title?: string + rawInput?: { command?: unknown; subagent_type?: unknown } + } + } +} + +/** The composite record the host hands to the core decoder for one session. */ +export type GrokSessionRecords = { + summary: GrokSummary + signals: GrokSignals | null + updatesLines: string[] + /** Absolute session directory, used only for the host-side dedup key. */ + sourceDir: string + /** Basename of the session directory, used as a session id fallback. */ + sessionName: string + /** Project name the host derived from the session cwd. */ + project: string +} + +/** The rich decode of one Grok session, pre-pricing. */ +export type GrokDecodedCall = { + provider: 'grok' + model: string + inputTokens: number + outputTokens: number + cacheCreationInputTokens: number + cacheReadInputTokens: number + cachedInputTokens: number + reasoningTokens: number + webSearchRequests: number + tools: string[] + rawBashCommands: string[] + subagentTypes: string[] + timestamp: string + speed: 'standard' + deduplicationKey: string + userMessage: string + sessionId: string + /** Absolute project cwd; fingerprinted before it reaches an observation. */ + projectPath: string + /** Project name the host uses for display/breakdown. */ + project: string +} diff --git a/packages/core/src/providers/kimi/decode.ts b/packages/core/src/providers/kimi/decode.ts new file mode 100644 index 0000000..199cbd5 --- /dev/null +++ b/packages/core/src/providers/kimi/decode.ts @@ -0,0 +1,241 @@ +// @codeburn/core Kimi decoder: pure decode over host-supplied JSONL lines. +// The host reads the wire file and the configured model fallback; this decoder +// is stateless, does no fs/env/clock access, and carries no pricing. + +import type { DecodeContext } from '../../contracts.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { KimiDecodedCall, KimiSessionRecords, JsonObject } from './types.js' + +export const kimiToolNameMap: Record = { + Shell: 'Bash', + Bash: 'Bash', + bash: 'Bash', + ReadFile: 'Read', + ReadMediaFile: 'Read', + WriteFile: 'Write', + StrReplaceFile: 'Edit', + Grep: 'Grep', + Glob: 'Glob', + SearchWeb: 'WebSearch', + FetchURL: 'WebFetch', + Agent: 'Agent', + AgentTool: 'Agent', + TaskList: 'Agent', + TaskOutput: 'Agent', + TaskStop: 'Agent', + AskUserQuestion: 'AskUser', + SetTodoList: 'TodoWrite', + Think: 'Think', + EnterPlanMode: 'EnterPlanMode', + ExitPlanMode: 'ExitPlanMode', + SendDMail: 'DMail', +} + +function asObject(value: unknown): JsonObject | null { + return value && typeof value === 'object' && !Array.isArray(value) ? (value as JsonObject) : null +} + +function stringField(obj: JsonObject | null, key: string): string | undefined { + const value = obj?.[key] + return typeof value === 'string' ? value : undefined +} + +function numericField(obj: JsonObject, ...keys: string[]): number { + for (const key of keys) { + const raw = obj[key] + const n = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : NaN + if (Number.isFinite(n) && n > 0) return Math.trunc(n) + } + return 0 +} + +function timestampToIso(value: unknown): string { + if (typeof value === 'string') return value + if (typeof value !== 'number' || !Number.isFinite(value)) return '' + + const millis = value > 1_000_000_000_000 ? value : value * 1000 + const date = new Date(millis) + return Number.isFinite(date.getTime()) ? date.toISOString() : '' +} + +function extractUserText(value: unknown): string { + if (typeof value === 'string') return value.slice(0, 500) + if (!Array.isArray(value)) return '' + + return value + .map(part => stringField(asObject(part), 'text') ?? '') + .filter(Boolean) + .join(' ') + .slice(0, 500) +} + +function extractEnvelope(record: JsonObject): { type: string; payload: JsonObject; timestamp: string } | null { + const message = asObject(record['message']) + const envelope = message ?? record + const type = stringField(envelope, 'type') + const payload = asObject(envelope['payload']) + if (!type || !payload) return null + return { type, payload, timestamp: timestampToIso(record['timestamp']) } +} + +function extractUsage(payload: JsonObject): { + inputTokens: number + outputTokens: number + cacheReadInputTokens: number + cacheCreationInputTokens: number +} | null { + const usage = asObject(payload['token_usage']) ?? asObject(payload['usage']) + if (!usage) return null + + const cacheReadInputTokens = numericField(usage, 'input_cache_read', 'cache_read_input_tokens', 'cached_input_tokens') + const cacheCreationInputTokens = numericField(usage, 'input_cache_creation', 'cache_creation_input_tokens') + let inputTokens = numericField(usage, 'input_other', 'input_tokens') + if (inputTokens === 0) { + const totalInput = numericField(usage, 'input') + inputTokens = Math.max(0, totalInput - cacheReadInputTokens - cacheCreationInputTokens) + } + const outputTokens = numericField(usage, 'output', 'output_tokens') + + if ( + inputTokens === 0 && + outputTokens === 0 && + cacheReadInputTokens === 0 && + cacheCreationInputTokens === 0 + ) { + return null + } + + return { inputTokens, outputTokens, cacheReadInputTokens, cacheCreationInputTokens } +} + +function extractTool(payload: JsonObject): { tool: string; rawBashCommand?: string } | null { + const fn = asObject(payload['function']) + const rawName = stringField(fn, 'name') ?? stringField(payload, 'name') + if (!rawName) return null + + const tool = kimiToolNameMap[rawName] ?? rawName + const argsText = stringField(fn, 'arguments') ?? stringField(payload, 'arguments') + let args: JsonObject | null = null + if (argsText) { + try { + args = asObject(JSON.parse(argsText)) + } catch { + args = null + } + } + const command = stringField(args, 'command') + + return { tool, ...(tool === 'Bash' && command ? { rawBashCommand: command } : {}) } +} + +function isKimiSessionRecords(value: unknown): value is KimiSessionRecords { + return value !== null && typeof value === 'object' && 'lines' in (value as object) +} + +export type KimiDecodeInput = { + records: unknown[] + context: DecodeContext + seenKeys?: Set +} + +export type KimiDecodeResult = { + calls: KimiDecodedCall[] + diagnostics: RecordDiagnostic[] +} + +/** + * Decode a Kimi wire log into rich, cost-free calls. A single pass: + * TurnBegin/SteerInput set the pending user message; ToolCall records collect + tools; StatusUpdate records with usage flush a call. Dedup is keyed on + * `kimi::` against the live `seenKeys` set. + */ +export function decodeKimi({ records, seenKeys: liveSeen }: KimiDecodeInput): KimiDecodeResult { + const seen = liveSeen ?? new Set() + const session = records.find(isKimiSessionRecords) + if (!session) return { calls: [], diagnostics: [] } + + const { lines, configuredModel, sessionName } = session + const calls: KimiDecodedCall[] = [] + const tools: string[] = [] + const rawBashCommands: string[] = [] + let currentUserMessage = '' + let index = 0 + + for (const line of lines) { + if (!line.trim()) continue + + let record: JsonObject | null = null + try { + record = asObject(JSON.parse(line)) + } catch { + continue + } + if (!record) continue + + const envelope = extractEnvelope(record) + if (!envelope || envelope.type === 'metadata') continue + + if (envelope.type === 'TurnBegin' || envelope.type === 'SteerInput') { + currentUserMessage = extractUserText(envelope.payload['user_input']) + continue + } + + if (envelope.type === 'TurnEnd') { + currentUserMessage = '' + tools.length = 0 + rawBashCommands.length = 0 + continue + } + + if (envelope.type === 'ToolCall' || envelope.type === 'ToolCallRequest') { + const extracted = extractTool(envelope.payload) + if (!extracted) continue + tools.push(extracted.tool) + if (extracted.rawBashCommand) rawBashCommands.push(extracted.rawBashCommand) + continue + } + + if (envelope.type !== 'StatusUpdate') continue + + const usage = extractUsage(envelope.payload) + if (!usage) continue + + const rawMessageId = stringField(envelope.payload, 'message_id') + const dedupKey = `kimi:${sessionName}:${rawMessageId ?? index}` + index++ + if (seen.has(dedupKey)) continue + seen.add(dedupKey) + + const model = + stringField(envelope.payload, 'model') ?? + stringField(envelope.payload, 'model_name') ?? + configuredModel + + calls.push({ + provider: 'kimi', + model, + inputTokens: usage.inputTokens, + outputTokens: usage.outputTokens, + cacheCreationInputTokens: usage.cacheCreationInputTokens, + cacheReadInputTokens: usage.cacheReadInputTokens, + cachedInputTokens: usage.cacheReadInputTokens, + reasoningTokens: 0, + webSearchRequests: 0, + // The legacy kimi decode accumulated tools in a Set, deduping per turn. + // Preserve that here so the observation/host tool list matches byte-for-byte. + tools: [...new Set(tools)], + rawBashCommands: [...rawBashCommands], + timestamp: envelope.timestamp, + speed: 'standard', + deduplicationKey: dedupKey, + userMessage: currentUserMessage, + sessionId: sessionName, + projectPath: '', + }) + + tools.length = 0 + rawBashCommands.length = 0 + } + + return { calls, diagnostics: [] } +} diff --git a/packages/core/src/providers/kimi/index.ts b/packages/core/src/providers/kimi/index.ts new file mode 100644 index 0000000..368f6c7 --- /dev/null +++ b/packages/core/src/providers/kimi/index.ts @@ -0,0 +1,26 @@ +// @codeburn/core Kimi provider. +// +// Two layers: +// - Rich pure decode (`decodeKimi`): host-facing, NOT part of the stable +// minimized surface. Pure over the host-supplied JSONL lines; carries +// content in-memory but no pricing and no bash base-name extraction. +// - Minimizing transform (`toObservations`): maps the rich decode into the +// strict observation envelope; the content-smuggling guarantees bind here. + +export { + decodeKimi, + kimiToolNameMap, + type KimiDecodeInput, + type KimiDecodeResult, +} from './decode.js' + +export { + toObservations, + type RichKimiSessionDecode, + type KimiToObservationsContext, +} from './observations.js' + +export type { + KimiDecodedCall, + KimiSessionRecords, +} from './types.js' diff --git a/packages/core/src/providers/kimi/observations.ts b/packages/core/src/providers/kimi/observations.ts new file mode 100644 index 0000000..93d26f3 --- /dev/null +++ b/packages/core/src/providers/kimi/observations.ts @@ -0,0 +1,82 @@ +// Minimizing transform: rich Kimi decode -> the strict observation envelope. +// +// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool +// names cross into the output. The user message and raw bash commands stay +// behind. + +import { projectRef, sessionRef } from '../../fingerprint.js' +import type { RecordDiagnostic } from '../../diagnostics.js' +import type { CallObservation, SessionObservation } from '../../observations.js' +import type { KimiDecodedCall } from './types.js' + +/** One Kimi session's rich decode, as the host holds it before minimization. */ +export interface RichKimiSessionDecode { + sessionId: string + /** Kimi does not record a project path; an empty path is fingerprinted. */ + projectPath: string + /** Rich, cost-free calls in decode order. */ + calls: KimiDecodedCall[] +} + +export interface KimiToObservationsContext { + /** HMAC key that scopes every fingerprint. */ + privacyKey: string + /** Provider id stamped onto sessions/calls and folded into sessionRef. */ + provider?: string +} + +const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/ + +function toCallObservation(call: KimiDecodedCall, turnIndex: number, privacyKey: string): CallObservation { + return { + provider: call.provider, + model: call.model, + tokens: { + input: call.inputTokens, + output: call.outputTokens, + reasoning: call.reasoningTokens, + cacheRead: call.cacheReadInputTokens, + cacheCreate: call.cacheCreationInputTokens, + }, + webSearchRequests: call.webSearchRequests, + speed: call.speed, + costBasis: 'estimated', + timestamp: call.timestamp, + dedupKey: call.deduplicationKey, + toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)), + turnIndex, + } +} + +function toSessionObservation(decode: RichKimiSessionDecode, ctx: KimiToObservationsContext): SessionObservation { + const provider = ctx.provider ?? 'kimi' + const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i, ctx.privacyKey)) + + const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort() + const startedAt = timestamps[0] ?? '' + const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : '' + + const session: SessionObservation = { + sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId), + projectRef: projectRef(ctx.privacyKey, decode.projectPath), + providerId: provider, + startedAt, + ...(endedAt ? { endedAt } : {}), + calls, + turnCount: calls.length, + } + return session +} + +/** + * Map a rich Kimi decode into the minimized observation layer. Returns the + * `sessions` array plus any per-record `diagnostics`. + */ +export function toObservations( + decode: RichKimiSessionDecode | RichKimiSessionDecode[], + ctx: KimiToObservationsContext, +): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } { + const decodes = Array.isArray(decode) ? decode : [decode] + const sessions = decodes.map(d => toSessionObservation(d, ctx)) + return { sessions, diagnostics: [] } +} diff --git a/packages/core/src/providers/kimi/types.ts b/packages/core/src/providers/kimi/types.ts new file mode 100644 index 0000000..955b28d --- /dev/null +++ b/packages/core/src/providers/kimi/types.ts @@ -0,0 +1,38 @@ +// Raw record + rich-decode types for the Kimi provider. +// +// Kimi writes one JSONL wire log per session. The host reads the log lines and +// the configured model fallback from config.toml; the decoder is pure over that +// composite record and carries no pricing. + +export type JsonObject = Record + +export type KimiSessionRecords = { + /** Raw JSONL lines from the wire log. */ + lines: string[] + /** Model string to fall back to when no model is named on a usage record. */ + configuredModel: string + /** Basename of the session directory, used as the session id. */ + sessionName: string +} + +/** The rich decode of one Kimi call, pre-pricing. */ +export type KimiDecodedCall = { + provider: 'kimi' + model: string + inputTokens: number + outputTokens: number + cacheCreationInputTokens: number + cacheReadInputTokens: number + cachedInputTokens: number + reasoningTokens: number + webSearchRequests: number + tools: string[] + rawBashCommands: string[] + timestamp: string + speed: 'standard' + deduplicationKey: string + userMessage: string + sessionId: string + /** Kimi does not record a project path; observations fingerprint an empty path. */ + projectPath: string +} diff --git a/packages/core/tests/architecture-gate.test.ts b/packages/core/tests/architecture-gate.test.ts index da295f8..71a7aa4 100644 --- a/packages/core/tests/architecture-gate.test.ts +++ b/packages/core/tests/architecture-gate.test.ts @@ -118,8 +118,14 @@ const CORRECTION_PHRASES = [ const USER_MESSAGE_ALLOWLIST = new Set([ 'src/providers/claude/decode.ts', 'src/providers/claude/types.ts', + 'src/providers/codewhale/decode.ts', + 'src/providers/codewhale/types.ts', 'src/providers/codex/decode.ts', 'src/providers/codex/types.ts', + 'src/providers/grok/decode.ts', + 'src/providers/grok/types.ts', + 'src/providers/kimi/decode.ts', + 'src/providers/kimi/types.ts', 'src/providers/qwen/decode.ts', 'src/providers/qwen/types.ts', ]) diff --git a/packages/core/tests/content-smuggling.test.ts b/packages/core/tests/content-smuggling.test.ts index a86b213..973d24d 100644 --- a/packages/core/tests/content-smuggling.test.ts +++ b/packages/core/tests/content-smuggling.test.ts @@ -20,6 +20,9 @@ import { import type { JournalEntry, ToolResultMeta } from '../src/providers/claude/index.js' import { decodeCodex, toObservations as toCodexObservations } from '../src/providers/codex/index.js' import { decodeQwen, toObservations as toQwenObservations } from '../src/providers/qwen/index.js' +import { decodeGrok, toObservations as toGrokObservations } from '../src/providers/grok/index.js' +import { decodeKimi, toObservations as toKimiObservations } from '../src/providers/kimi/index.js' +import { decodeCodeWhale, toObservations as toCodeWhaleObservations } from '../src/providers/codewhale/index.js' import type { DecodeContext } from '../src/contracts.js' const here = dirname(fileURLToPath(import.meta.url)) @@ -357,6 +360,223 @@ describe('content-smuggling guardrail: real qwen decode -> toObservations is sec }) }) +describe('content-smuggling guardrail: real grok decode -> toObservations is secret-free', () => { + // A hostile Grok session planting every secret in the free-text fields the + // decode captures: the project path, the user message (session summary/title), + // a bash command, and a subagent type. Plus a tool NAME carrying a command + // line, which must be dropped by the canonical-name filter. + const grokContext: DecodeContext = { privacyKey: 'test-privacy-key', providerId: 'grok', sourceRef: 'ref' } + + function decodeAndMinimize() { + const records = [ + { + summary: { + info: { id: 'sess-hostile', cwd: SECRETS.absPath }, + created_at: '2026-07-17T10:00:00.000Z', + updated_at: '2026-07-17T10:00:05.000Z', + session_summary: `${SECRETS.prompt} ${SECRETS.apiKey} ${SECRETS.fileContent}`, + generated_title: SECRETS.prompt, + }, + signals: null, + updatesLines: [ + JSON.stringify({ + timestamp: '2026-07-17T10:00:05.000Z', + method: 'session/update', + params: { + sessionId: 'sess-hostile', + update: { sessionUpdate: 'tool_call', title: SECRETS.commandLine, rawInput: {} }, + }, + }), + JSON.stringify({ + timestamp: '2026-07-17T10:00:05.000Z', + method: 'session/update', + params: { + sessionId: 'sess-hostile', + update: { + sessionUpdate: 'tool_call', + title: 'run_terminal_command', + rawInput: { command: SECRETS.commandLine }, + }, + }, + }), + JSON.stringify({ + timestamp: '2026-07-17T10:00:05.000Z', + method: 'session/update', + params: { + sessionId: 'sess-hostile', + update: { + sessionUpdate: 'tool_call', + title: 'spawn_subagent', + rawInput: { subagent_type: SECRETS.fileContent }, + }, + _meta: { totalTokens: 1000, promptId: 'p1' }, + }, + }), + ], + sourceDir: '/sessions/hostile', + sessionName: 'sess-hostile', + project: 'hostile-project', + }, + ] + const { calls } = decodeGrok({ records, context: grokContext }) + const { sessions } = toGrokObservations( + { sessionId: 'sess-hostile', projectPath: SECRETS.absPath, calls }, + { privacyKey: 'test-privacy-key', provider: 'grok' }, + ) + return { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + } + + it('produces a schema-valid envelope from the hostile session', () => { + expect(ObservationEnvelope.safeParse(decodeAndMinimize()).success).toBe(true) + }) + + it('the serialized envelope contains none of the planted secrets', () => { + const serialized = JSON.stringify(decodeAndMinimize()) + for (const secret of ALL_SECRETS) { + expect(serialized).not.toContain(secret) + } + }) + + it('drops non-canonical (argument-carrying) tool names instead of emitting them', () => { + const env = decodeAndMinimize() + const allToolNames = env.sessions.flatMap(s => s.calls.flatMap(c => c.toolNames)) + expect(allToolNames).toContain('Bash') + expect(allToolNames).not.toContain(SECRETS.commandLine) + }) +}) + +describe('content-smuggling guardrail: real kimi decode -> toObservations is secret-free', () => { + // A hostile Kimi wire log planting every secret in the free-text fields the + // decode captures: the user message, a Bash command, and a tool NAME carrying + // a command line. Minimizing MUST surface none of them. + const kimiContext: DecodeContext = { privacyKey: 'test-privacy-key', providerId: 'kimi', sourceRef: 'ref' } + + function decodeAndMinimize() { + const records = [ + { + lines: [ + JSON.stringify({ timestamp: 1776162400, message: { type: 'TurnBegin', payload: { user_input: `${SECRETS.prompt} ${SECRETS.apiKey} ${SECRETS.fileContent}` } } }), + JSON.stringify({ timestamp: 1776162401, message: { type: 'ToolCall', payload: { type: 'function', id: 'call-shell', function: { name: SECRETS.commandLine, arguments: '{}' } } } }), + JSON.stringify({ timestamp: 1776162402, message: { type: 'ToolCall', payload: { type: 'function', id: 'call-bash', function: { name: 'Shell', arguments: JSON.stringify({ command: SECRETS.commandLine }) } } } }), + JSON.stringify({ timestamp: 1776162403, message: { type: 'StatusUpdate', payload: { message_id: 'msg-hostile', token_usage: { input_other: 10, output: 5 } } } }), + ], + configuredModel: 'kimi-auto', + sessionName: 'sess-hostile', + }, + ] + const { calls } = decodeKimi({ records, context: kimiContext }) + const { sessions } = toKimiObservations( + { sessionId: 'sess-hostile', projectPath: '', calls }, + { privacyKey: 'test-privacy-key', provider: 'kimi' }, + ) + return { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + } + + it('produces a schema-valid envelope from the hostile wire log', () => { + expect(ObservationEnvelope.safeParse(decodeAndMinimize()).success).toBe(true) + }) + + it('the serialized envelope contains none of the planted secrets', () => { + const serialized = JSON.stringify(decodeAndMinimize()) + for (const secret of ALL_SECRETS) { + expect(serialized).not.toContain(secret) + } + }) + + it('keeps canonical tool names (Bash) and drops the argument-carrying name', () => { + const env = decodeAndMinimize() + const allToolNames = env.sessions.flatMap(s => s.calls.flatMap(c => c.toolNames)) + expect(allToolNames).toContain('Bash') + expect(allToolNames).not.toContain(SECRETS.commandLine) + }) +}) + +describe('content-smuggling guardrail: real codewhale decode -> toObservations is secret-free', () => { + // A hostile CodeWhale session planting every secret in the free-text fields + // the decode captures: the project path, the user message, a Bash command, a + // read/edit file path, a skill name, a subagent type, and a tool NAME carrying + // a command line. Minimizing MUST surface none of them. + const codeWhaleContext: DecodeContext = { privacyKey: 'test-privacy-key', providerId: 'codewhale', sourceRef: 'ref' } + + function decodeAndMinimize() { + const records = [ + { + metadata: { + id: 'sess-hostile', + total_tokens: 1000, + workspace: SECRETS.absPath, + }, + messages: [ + { role: 'user', content: `${SECRETS.prompt} ${SECRETS.apiKey} ${SECRETS.fileContent}` }, + { + role: 'assistant', + content: [ + { type: 'tool_use', id: 't1', name: SECRETS.commandLine, input: {} }, + { type: 'tool_use', id: 't2', name: 'exec_shell', input: { command: SECRETS.commandLine } }, + { type: 'tool_use', id: 't3', name: 'read_file', input: { file_path: SECRETS.absPath } }, + { type: 'tool_use', id: 't4', name: 'edit_file', input: { path: SECRETS.absPath } }, + { type: 'tool_use', id: 't5', name: 'load_skill', input: { name: SECRETS.fileContent } }, + { type: 'tool_use', id: 't6', name: 'agent', input: { type: SECRETS.prompt } }, + ], + }, + ], + fileMtime: '2026-07-17T10:00:00.000Z', + }, + ] + const { calls } = decodeCodeWhale({ records, context: codeWhaleContext }) + const { sessions } = toCodeWhaleObservations( + { sessionId: 'sess-hostile', projectPath: SECRETS.absPath, calls }, + { privacyKey: 'test-privacy-key', provider: 'codewhale' }, + ) + return { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + } + + it('produces a schema-valid envelope from the hostile session', () => { + expect(ObservationEnvelope.safeParse(decodeAndMinimize()).success).toBe(true) + }) + + it('the serialized envelope contains none of the planted secrets', () => { + const serialized = JSON.stringify(decodeAndMinimize()) + for (const secret of ALL_SECRETS) { + expect(serialized).not.toContain(secret) + } + }) + + it('keeps canonical tool names (Bash/Read/Agent/Skill) and drops the argument-carrying name', () => { + const env = decodeAndMinimize() + const allToolNames = env.sessions.flatMap(s => s.calls.flatMap(c => c.toolNames)) + expect(allToolNames).toContain('Bash') + expect(allToolNames).toContain('Read') + expect(allToolNames).toContain('Agent') + expect(allToolNames).toContain('Skill') + expect(allToolNames).not.toContain(SECRETS.commandLine) + }) + + it('fingerprints the read/edit paths into 16-hex resource refs, never the raw paths', () => { + const env = decodeAndMinimize() + const reads = env.sessions.flatMap(s => s.calls.flatMap(c => c.resourceReads ?? [])) + const edits = env.sessions.flatMap(s => s.calls.flatMap(c => c.resourceEdits ?? [])) + expect(reads.length).toBeGreaterThan(0) + expect(edits.length).toBeGreaterThan(0) + for (const ref of [...reads, ...edits]) { + expect(ref.resourceId).toMatch(/^[0-9a-f]{16}$/) + } + expect(allStrings([...reads, ...edits])).not.toContain(SECRETS.absPath) + }) +}) + describe('content-smuggling guardrail: diagnostic detail rejects paths', () => { it('rejects an absolute path', () => { expect(DiagnosticDetail.safeParse(SECRETS.absPath).success).toBe(false) diff --git a/packages/core/tests/providers/codewhale-decode.test.ts b/packages/core/tests/providers/codewhale-decode.test.ts new file mode 100644 index 0000000..3656637 --- /dev/null +++ b/packages/core/tests/providers/codewhale-decode.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, it } from 'vitest' + +import { decodeCodeWhale, toObservations } from '../../src/providers/codewhale/index.js' +import { ObservationEnvelope } from '../../src/observations.js' +import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js' +import type { DecodeContext } from '../../src/contracts.js' +import type { CodeWhaleSessionRecords } from '../../src/providers/codewhale/types.js' + +const context: DecodeContext = { privacyKey: 'k', providerId: 'codewhale', sourceRef: 'ref' } + +function session(opts: Partial = {}): CodeWhaleSessionRecords { + return { + metadata: { + id: 'session-a', + created_at: '2026-07-14T10:00:00.000Z', + updated_at: '2026-07-15T12:34:56.000Z', + total_tokens: 12345, + model: 'anthropic/claude-sonnet-4-6', + model_provider: 'anthropic', + workspace: '/Users/alice/codewhale-demo', + cost: { session_cost_usd: 0.75, subagent_cost_usd: 0.20 }, + ...opts.metadata, + }, + messages: opts.messages ?? [], + fileMtime: opts.fileMtime ?? '2026-07-15T10:00:00.000Z', + } +} + +describe('codewhale rich decode (moved to @codeburn/core)', () => { + it('decodes one cumulative session with measured cost and tool sequence', () => { + const records: CodeWhaleSessionRecords[] = [ + session({ + messages: [ + { role: 'user', content: [{ type: 'text', text: 'Implement the parser' }] }, + { + role: 'assistant', + content: [ + { type: 'thinking', thinking: '...' }, + { type: 'tool_use', id: 't1', name: 'read_file', input: { file_path: 'src/app.ts' } }, + { type: 'tool_use', id: 't2', name: 'exec_shell', input: { command: 'npm test && git status' } }, + { type: 'tool_use', id: 't3', name: 'edit_file', input: { path: 'src/app.ts' } }, + ] as any, + }, + { + role: 'assistant', + content: [ + { type: 'tool_use', id: 't4', name: 'load_skill', input: { name: 'typescript' } }, + { type: 'tool_use', id: 't5', name: 'agent', input: { type: 'reviewer' } }, + { type: 'server_tool_use', id: 't6', name: 'web_search', input: { query: 'CodeWhale' } }, + ] as any, + }, + ], + }), + ] + + const { calls } = decodeCodeWhale({ records, context }) + expect(calls).toHaveLength(1) + + const call = calls[0]! + expect(call.provider).toBe('codewhale') + expect(call.model).toBe('anthropic/claude-sonnet-4-6') + expect(call.inputTokens).toBe(12345) + expect(call.outputTokens).toBe(0) + expect(call.measuredCostUSD).toBe(0.95) + expect(call.webSearchRequests).toBe(1) + expect(call.tools).toEqual(['Read', 'Bash', 'Edit', 'Skill', 'Agent', 'WebSearch']) + expect(call.rawBashCommands).toEqual(['npm test && git status']) + expect(call.skills).toEqual(['typescript']) + expect(call.subagentTypes).toEqual(['reviewer']) + expect(call.userMessage).toBe('Implement the parser') + expect(call.sessionId).toBe('session-a') + expect(call.project).toBe('codewhale-demo') + expect(call.projectPath).toBe('/Users/alice/codewhale-demo') + expect(call.toolSequence).toEqual([ + [ + { tool: 'Read', file: 'src/app.ts' }, + { tool: 'Bash', command: 'npm test && git status' }, + { tool: 'Edit', file: 'src/app.ts' }, + ], + [{ tool: 'Skill' }, { tool: 'Agent' }, { tool: 'WebSearch' }], + ]) + }) + + it('falls back to estimated when no cost is recorded', () => { + const records: CodeWhaleSessionRecords[] = [ + session({ + metadata: { id: 'session-b', total_tokens: 1000, cost: undefined }, + messages: [{ role: 'user', content: 'hi' }], + }), + ] + + const { calls } = decodeCodeWhale({ records, context }) + expect(calls).toHaveLength(1) + expect(calls[0]!.measuredCostUSD).toBeUndefined() + }) + + it('skips zero-token sessions with no exact cost', () => { + const records: CodeWhaleSessionRecords[] = [ + session({ + metadata: { id: 'session-c', total_tokens: 0, cost: { session_cost_usd: 0, subagent_cost_usd: 0 } }, + messages: [], + }), + ] + + expect(decodeCodeWhale({ records, context }).calls).toEqual([]) + }) + + it('deduplicates by session id using the live seenKeys set', () => { + const records: CodeWhaleSessionRecords[] = [session()] + const seen = new Set() + expect(decodeCodeWhale({ records, context, seenKeys: seen }).calls).toHaveLength(1) + expect(decodeCodeWhale({ records, context, seenKeys: seen }).calls).toEqual([]) + }) + + it('toObservations produces a schema-valid envelope with fingerprinted file refs', () => { + const records: CodeWhaleSessionRecords[] = [ + session({ + messages: [ + { role: 'user', content: 'Implement the parser' }, + { + role: 'assistant', + content: [ + { type: 'tool_use', id: 't1', name: 'read_file', input: { file_path: 'src/app.ts' } }, + { type: 'tool_use', id: 't2', name: 'edit_file', input: { path: 'src/app.ts' } }, + ] as any, + }, + ], + }), + ] + + const { calls } = decodeCodeWhale({ records, context }) + const { sessions } = toObservations( + { sessionId: 'session-a', projectPath: '/Users/alice/codewhale-demo', calls }, + { privacyKey: 'test-privacy-key', provider: 'codewhale' }, + ) + const envelope = { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + expect(ObservationEnvelope.safeParse(envelope).success).toBe(true) + const reads = sessions.flatMap(s => s.calls.flatMap(c => c.resourceReads ?? [])) + const edits = sessions.flatMap(s => s.calls.flatMap(c => c.resourceEdits ?? [])) + expect(reads.length).toBeGreaterThan(0) + expect(edits.length).toBeGreaterThan(0) + for (const ref of [...reads, ...edits]) expect(ref.resourceId).toMatch(/^[0-9a-f]{16}$/) + }) +}) diff --git a/packages/core/tests/providers/grok-decode.test.ts b/packages/core/tests/providers/grok-decode.test.ts new file mode 100644 index 0000000..6fd3f37 --- /dev/null +++ b/packages/core/tests/providers/grok-decode.test.ts @@ -0,0 +1,165 @@ +import { describe, expect, it } from 'vitest' + +import { decodeGrok, toObservations } from '../../src/providers/grok/index.js' +import { ObservationEnvelope } from '../../src/observations.js' +import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js' +import type { DecodeContext } from '../../src/contracts.js' +import type { GrokSessionRecords } from '../../src/providers/grok/types.js' + +const context: DecodeContext = { privacyKey: 'k', providerId: 'grok', sourceRef: 'ref' } + +function sessionRecord(opts: { + updatesLines: string[] + summary?: Partial + signals?: GrokSessionRecords['signals'] + sourceDir?: string + sessionName?: string + project?: string +}): GrokSessionRecords { + return { + summary: { + info: { id: 'sess-1', cwd: '/Users/test/project' }, + created_at: '2026-06-19T11:20:40.686261Z', + updated_at: '2026-06-19T11:31:12.282793Z', + last_active_at: '2026-06-19T11:31:12.222328Z', + current_model_id: 'grok-build', + session_summary: 'User asks about the repo', + generated_title: 'User asks about the repo', + ...opts.summary, + }, + signals: opts.signals ?? null, + updatesLines: opts.updatesLines, + sourceDir: opts.sourceDir ?? '/sessions/%2FUsers%2Ftest/sess-1', + sessionName: opts.sessionName ?? 'sess-1', + project: opts.project ?? 'project', + } +} + +function updateLine(opts: { + totalTokens?: number + promptId?: string + title?: string + rawInput?: Record +}): string { + return JSON.stringify({ + timestamp: '2026-06-19T11:30:00.000Z', + method: 'session/update', + params: { + sessionId: 'sess-1', + update: { + sessionUpdate: opts.title ? 'tool_call' : 'agent_message_chunk', + ...(opts.title ? { title: opts.title, toolCallId: 'c1', rawInput: opts.rawInput } : { content: { type: 'text', text: 'hi' } }), + }, + _meta: { + ...(opts.totalTokens !== undefined ? { totalTokens: opts.totalTokens } : {}), + ...(opts.promptId ? { promptId: opts.promptId } : {}), + }, + }, + }) +} + +describe('grok rich decode (moved to @codeburn/core)', () => { + it('decodes one session from updates.jsonl into a cost-free rich call', () => { + const records: GrokSessionRecords[] = [ + sessionRecord({ + updatesLines: [ + updateLine({ totalTokens: 20000, promptId: 'p1' }), + updateLine({ totalTokens: 25000, promptId: 'p1' }), + updateLine({ totalTokens: 30000, promptId: 'p2' }), + updateLine({ totalTokens: 35000, promptId: 'p2' }), + updateLine({ title: 'read_file', rawInput: { target_directory: '.' } }), + updateLine({ title: 'run_terminal_command', rawInput: { command: 'npm test && git status' } }), + updateLine({ title: 'spawn_subagent', rawInput: { subagent_type: 'general-purpose' } }), + ], + }), + ] + + const { calls } = decodeGrok({ records, context }) + expect(calls).toHaveLength(1) + + const call = calls[0]! + expect(call.provider).toBe('grok') + expect(call.model).toBe('grok-build') + expect(call.inputTokens).toBe(35000) + expect(call.outputTokens).toBe(10000) + expect(call.cacheReadInputTokens).toBe(15000) + expect(call.cachedInputTokens).toBe(15000) + expect(call.tools).toEqual(['Read', 'Bash', 'Agent']) + // Raw commands stay raw; base-name extraction is the host's job. + expect(call.rawBashCommands).toEqual(['npm test && git status']) + expect(call.subagentTypes).toEqual(['general-purpose']) + expect(call.userMessage).toBe('User asks about the repo') + expect(call.sessionId).toBe('sess-1') + expect(call.project).toBe('project') + expect(call.projectPath).toBe('/Users/test/project') + }) + + it('sums fresh input across compaction segments', () => { + const records: GrokSessionRecords[] = [ + sessionRecord({ + updatesLines: [ + updateLine({ totalTokens: 100000, promptId: 'p1' }), + updateLine({ totalTokens: 400000, promptId: 'p1' }), + // Large drop (>50%) simulates compaction. + updateLine({ totalTokens: 20000, promptId: 'p2' }), + updateLine({ totalTokens: 50000, promptId: 'p2' }), + ], + }), + ] + + const { calls } = decodeGrok({ records, context }) + expect(calls).toHaveLength(1) + expect(calls[0]!.inputTokens).toBe(450000) + }) + + it('skips zero-token sessions', () => { + const records: GrokSessionRecords[] = [ + sessionRecord({ + updatesLines: [ + updateLine({ totalTokens: 0, promptId: 'p1' }), + updateLine({ totalTokens: 0, promptId: 'p1' }), + ], + }), + ] + + expect(decodeGrok({ records, context }).calls).toEqual([]) + }) + + it('threads a live seenKeys set so a repeated session drops', () => { + const records: GrokSessionRecords[] = [ + sessionRecord({ + updatesLines: [updateLine({ totalTokens: 1000, promptId: 'p1' })], + }), + ] + + const seen = new Set() + expect(decodeGrok({ records, context, seenKeys: seen }).calls).toHaveLength(1) + expect(decodeGrok({ records, context, seenKeys: seen }).calls).toEqual([]) + }) + + it('toObservations produces a schema-valid, content-free envelope', () => { + const records: GrokSessionRecords[] = [ + sessionRecord({ + updatesLines: [ + updateLine({ totalTokens: 1000, promptId: 'p1' }), + updateLine({ title: 'read_file', rawInput: { file_path: 'src/app.ts' } }), + ], + }), + ] + + const { calls } = decodeGrok({ records, context }) + const { sessions } = toObservations( + { sessionId: 'sess-1', projectPath: '/Users/test/project', calls }, + { privacyKey: 'test-privacy-key', provider: 'grok' }, + ) + const envelope = { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + expect(ObservationEnvelope.safeParse(envelope).success).toBe(true) + // Grok does not capture per-tool file paths, so there are no resource refs. + const reads = sessions.flatMap(s => s.calls.flatMap(c => c.resourceReads ?? [])) + expect(reads).toEqual([]) + }) +}) diff --git a/packages/core/tests/providers/kimi-decode.test.ts b/packages/core/tests/providers/kimi-decode.test.ts new file mode 100644 index 0000000..518aac8 --- /dev/null +++ b/packages/core/tests/providers/kimi-decode.test.ts @@ -0,0 +1,176 @@ +import { describe, expect, it } from 'vitest' + +import { decodeKimi, toObservations } from '../../src/providers/kimi/index.js' +import { ObservationEnvelope } from '../../src/observations.js' +import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js' +import type { DecodeContext } from '../../src/contracts.js' +import type { KimiSessionRecords } from '../../src/providers/kimi/types.js' + +const context: DecodeContext = { privacyKey: 'k', providerId: 'kimi', sourceRef: 'ref' } + +function record(timestamp: number, type: string, payload: Record): string { + return JSON.stringify({ timestamp, message: { type, payload } }) +} + +function session(opts: Partial = {}): KimiSessionRecords { + return { + lines: opts.lines ?? [], + configuredModel: opts.configuredModel ?? 'kimi-auto', + sessionName: opts.sessionName ?? 'sess-a', + } +} + +describe('kimi rich decode (moved to @codeburn/core)', () => { + it('decodes StatusUpdate usage, tools, and raw bash commands', () => { + const records: KimiSessionRecords[] = [ + session({ + configuredModel: 'kimi-k2-thinking-turbo', + lines: [ + record(1776162400, 'TurnBegin', { user_input: 'add status endpoint' }), + record(1776162401, 'ToolCall', { + type: 'function', + id: 'call-shell', + function: { name: 'Shell', arguments: JSON.stringify({ command: 'git status && npm test' }) }, + }), + record(1776162402, 'ToolCall', { + type: 'function', + id: 'call-read', + function: { name: 'ReadFile', arguments: JSON.stringify({ path: 'src/index.ts' }) }, + }), + record(1776162403, 'StatusUpdate', { + message_id: 'msg-1', + token_usage: { + input_other: 100, + input_cache_read: 25, + input_cache_creation: 10, + output: 40, + }, + }), + ], + }), + ] + + const { calls } = decodeKimi({ records, context }) + expect(calls).toHaveLength(1) + + const call = calls[0]! + expect(call.provider).toBe('kimi') + expect(call.model).toBe('kimi-k2-thinking-turbo') + expect(call.inputTokens).toBe(100) + expect(call.outputTokens).toBe(40) + expect(call.cacheReadInputTokens).toBe(25) + expect(call.cacheCreationInputTokens).toBe(10) + expect(call.cachedInputTokens).toBe(25) + expect(call.tools).toEqual(['Bash', 'Read']) + expect(call.rawBashCommands).toEqual(['git status && npm test']) + expect(call.userMessage).toBe('add status endpoint') + expect(call.deduplicationKey).toBe('kimi:sess-a:msg-1') + expect(call.sessionId).toBe('sess-a') + }) + + it('dedups tools per turn (Set), matching the legacy in-CLI decode', () => { + const records: KimiSessionRecords[] = [ + session({ + lines: [ + record(1776162400, 'TurnBegin', { user_input: 'x' }), + record(1776162401, 'ToolCall', { + type: 'function', + function: { name: 'ReadFile', arguments: JSON.stringify({ path: 'a.ts' }) }, + }), + record(1776162402, 'ToolCall', { + type: 'function', + function: { name: 'ReadFile', arguments: JSON.stringify({ path: 'b.ts' }) }, + }), + record(1776162403, 'ToolCall', { + type: 'function', + function: { name: 'Shell', arguments: JSON.stringify({ command: 'ls' }) }, + }), + record(1776162404, 'StatusUpdate', { + message_id: 'msg-dup', + token_usage: { input_other: 5, output: 7 }, + }), + ], + }), + ] + + const { calls } = decodeKimi({ records, context }) + expect(calls).toHaveLength(1) + // Two ReadFile calls collapse to one 'Read'; raw bash commands stay raw. + expect(calls[0]!.tools).toEqual(['Read', 'Bash']) + expect(calls[0]!.rawBashCommands).toEqual(['ls']) + }) + + it('filters thought parts and joins content parts for the user message', () => { + const records: KimiSessionRecords[] = [ + session({ + lines: [ + record(1776023300, 'TurnBegin', { + user_input: [ + { type: 'text', text: 'refactor parser' }, + { type: 'image_url', image_url: { url: 'file://diagram.png' } }, + { type: 'text', text: 'carefully' }, + ], + }), + record(1776023301, 'ToolCallRequest', { + id: 'call-write', + name: 'WriteFile', + arguments: JSON.stringify({ path: 'src/parser.ts', content: 'x' }), + }), + record(1776023302, 'StatusUpdate', { + message_id: 'msg-2', + model_name: 'kimi-k2.6', + token_usage: { input_other: 5, output: 7 }, + }), + ], + }), + ] + + const { calls } = decodeKimi({ records, context }) + expect(calls).toHaveLength(1) + expect(calls[0]!.userMessage).toBe('refactor parser carefully') + expect(calls[0]!.model).toBe('kimi-k2.6') + expect(calls[0]!.tools).toEqual(['Write']) + }) + + it('deduplicates repeated message ids using the live seenKeys set', () => { + const records: KimiSessionRecords[] = [ + session({ + lines: [ + record(1776023300, 'TurnBegin', { user_input: 'x' }), + record(1776023301, 'StatusUpdate', { message_id: 'msg-3', token_usage: { input_other: 5, output: 7 } }), + record(1776023302, 'StatusUpdate', { message_id: 'msg-3', token_usage: { input_other: 5, output: 7 } }), + ], + }), + ] + + const seen = new Set() + expect(decodeKimi({ records, context, seenKeys: seen }).calls).toHaveLength(1) + expect(decodeKimi({ records, context, seenKeys: seen }).calls).toEqual([]) + }) + + it('toObservations produces a schema-valid, content-free envelope', () => { + const records: KimiSessionRecords[] = [ + session({ + lines: [ + record(1776162400, 'TurnBegin', { user_input: 'add status endpoint' }), + record(1776162403, 'StatusUpdate', { + message_id: 'msg-1', + token_usage: { input_other: 100, output: 40 }, + }), + ], + }), + ] + + const { calls } = decodeKimi({ records, context }) + const { sessions } = toObservations( + { sessionId: 'sess-a', projectPath: '', calls }, + { privacyKey: 'test-privacy-key', provider: 'kimi' }, + ) + const envelope = { + schemaVersion: OBSERVATION_SCHEMA_VERSION, + generator: { name: '@codeburn/core', version: '0.0.0-test' }, + sessions, + } + expect(ObservationEnvelope.safeParse(envelope).success).toBe(true) + }) +}) diff --git a/packages/core/tsup.config.ts b/packages/core/tsup.config.ts index 0faec29..7cf6f24 100644 --- a/packages/core/tsup.config.ts +++ b/packages/core/tsup.config.ts @@ -12,7 +12,10 @@ export default defineConfig({ 'src/contracts.ts', 'src/detectors/index.ts', 'src/providers/claude/index.ts', + 'src/providers/codewhale/index.ts', 'src/providers/codex/index.ts', + 'src/providers/grok/index.ts', + 'src/providers/kimi/index.ts', 'src/providers/qwen/index.ts', ], format: ['esm'],