mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-08-04 05:41:29 +00:00
Merge pull request #828 from getagentseal/phase8/kimi-batch-a
refactor(core): tail migrations — grok, kimi, codewhale (phase 8)
This commit is contained in:
Resham Joshi
GitHub
commit
43058baadd
32 changed files with 2456 additions and 654 deletions
|
|
@ -2,87 +2,17 @@ import { open, readdir, stat } from 'fs/promises'
|
|||
import { homedir } from 'os'
|
||||
import { join } from 'path'
|
||||
|
||||
import { decodeCodeWhale, mapCodeWhaleToolName } from '@codeburn/core/providers/codewhale'
|
||||
import type { CodeWhaleDecodedCall, CodeWhaleSessionRecords, CodeWhaleMetadata, CodeWhaleMessage } from '@codeburn/core/providers/codewhale'
|
||||
|
||||
import { extractBashCommands } from '../bash-utils.js'
|
||||
import { readSessionFile } from '../fs-utils.js'
|
||||
import { getShortModelName } from '../models.js'
|
||||
import type { ToolCall } from '../types.js'
|
||||
import type { ParsedProviderCall, Provider, SessionParser, SessionSource } from './types.js'
|
||||
import { createBridgedProvider } from './bridge.js'
|
||||
import type { Provider, SessionSource, ParsedProviderCall } from './types.js'
|
||||
|
||||
const METADATA_PREFIX_BYTES = 64 * 1024
|
||||
|
||||
type CodeWhaleCost = {
|
||||
session_cost_usd?: number
|
||||
subagent_cost_usd?: number
|
||||
}
|
||||
|
||||
type CodeWhaleMetadata = {
|
||||
id: string
|
||||
created_at?: string
|
||||
updated_at?: string
|
||||
total_tokens?: number
|
||||
model?: string
|
||||
model_provider?: string
|
||||
workspace?: string
|
||||
cost?: CodeWhaleCost
|
||||
}
|
||||
|
||||
type CodeWhaleContentBlock = {
|
||||
type?: string
|
||||
text?: string
|
||||
name?: string
|
||||
input?: unknown
|
||||
}
|
||||
|
||||
type CodeWhaleMessage = {
|
||||
role?: string
|
||||
content?: string | CodeWhaleContentBlock[]
|
||||
}
|
||||
|
||||
type CodeWhaleSession = {
|
||||
metadata?: unknown
|
||||
messages?: unknown
|
||||
}
|
||||
|
||||
const toolNameMap: Record<string, string> = {
|
||||
exec_shell: 'Bash',
|
||||
exec_shell_wait: 'Bash',
|
||||
exec_shell_interact: 'Bash',
|
||||
exec_shell_cancel: 'Bash',
|
||||
task_shell_start: 'Bash',
|
||||
task_shell_wait: 'Bash',
|
||||
terminal_run: 'Bash',
|
||||
terminal_send: 'Bash',
|
||||
terminal_wait: 'Bash',
|
||||
terminal_cancel: 'Bash',
|
||||
read_file: 'Read',
|
||||
write_file: 'Write',
|
||||
edit_file: 'Edit',
|
||||
fim_edit: 'Edit',
|
||||
apply_patch: 'Edit',
|
||||
list_dir: 'Glob',
|
||||
grep_files: 'Grep',
|
||||
web_search: 'WebSearch',
|
||||
fetch_url: 'WebFetch',
|
||||
'web.run': 'WebSearch',
|
||||
agent: 'Agent',
|
||||
'agents/list': 'Agent',
|
||||
'agents/message': 'Agent',
|
||||
'agents/followup': 'Agent',
|
||||
'agents/interrupt': 'Agent',
|
||||
'agents/wait': 'Agent',
|
||||
todo_write: 'TodoWrite',
|
||||
todo_add: 'TodoWrite',
|
||||
todo_update: 'TodoWrite',
|
||||
todo_list: 'TodoWrite',
|
||||
checklist_write: 'TodoWrite',
|
||||
checklist_add: 'TodoWrite',
|
||||
checklist_update: 'TodoWrite',
|
||||
checklist_list: 'TodoWrite',
|
||||
update_plan: 'TodoWrite',
|
||||
load_skill: 'Skill',
|
||||
request_user_input: 'AskUser',
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
||||
}
|
||||
|
|
@ -112,7 +42,7 @@ function parseMetadata(value: unknown): CodeWhaleMetadata | null {
|
|||
model: nonEmptyString(value['model']),
|
||||
model_provider: nonEmptyString(value['model_provider']),
|
||||
workspace: nonEmptyString(value['workspace']),
|
||||
cost: isRecord(value['cost']) ? value['cost'] as CodeWhaleCost : undefined,
|
||||
cost: isRecord(value['cost']) ? (value['cost'] as CodeWhaleMetadata['cost']) : undefined,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -214,7 +144,7 @@ async function readSessionMetadata(filePath: string): Promise<CodeWhaleMetadata
|
|||
const raw = await readSessionFile(filePath)
|
||||
if (raw === null) return null
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as CodeWhaleSession
|
||||
const parsed = JSON.parse(raw) as { metadata?: unknown; messages?: unknown }
|
||||
return parseMetadata(parsed.metadata)
|
||||
} catch {
|
||||
return null
|
||||
|
|
@ -258,186 +188,37 @@ async function discoverInDir(dir: string): Promise<Array<{ source: SessionSource
|
|||
return results
|
||||
}
|
||||
|
||||
function normalizeTimestamp(value: string | undefined): string {
|
||||
if (!value) return ''
|
||||
const timestamp = Date.parse(value)
|
||||
return Number.isFinite(timestamp) ? new Date(timestamp).toISOString() : ''
|
||||
}
|
||||
|
||||
function firstUserMessage(messages: CodeWhaleMessage[]): string {
|
||||
for (const message of messages) {
|
||||
if (message.role !== 'user') continue
|
||||
const text = typeof message.content === 'string'
|
||||
? message.content
|
||||
: Array.isArray(message.content)
|
||||
? message.content
|
||||
.filter(block => block?.type === 'text' && typeof block.text === 'string')
|
||||
.map(block => block.text)
|
||||
.join(' ')
|
||||
: ''
|
||||
if (text.trim()) return Array.from(text.trim()).slice(0, 500).join('')
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function mapToolName(rawName: string): string {
|
||||
if (rawName.startsWith('mcp__')) return rawName
|
||||
if (rawName.startsWith('agents/')) return 'Agent'
|
||||
return Object.prototype.hasOwnProperty.call(toolNameMap, rawName)
|
||||
? toolNameMap[rawName]!
|
||||
: rawName
|
||||
}
|
||||
|
||||
function toolInput(value: unknown): Record<string, unknown> {
|
||||
return isRecord(value) ? value : {}
|
||||
}
|
||||
|
||||
function firstString(input: Record<string, unknown>, keys: string[]): string | undefined {
|
||||
for (const key of keys) {
|
||||
const value = nonEmptyString(input[key])
|
||||
if (value) return value
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
function collectTools(messages: CodeWhaleMessage[]): {
|
||||
tools: string[]
|
||||
bashCommands: string[]
|
||||
toolSequence: ToolCall[][]
|
||||
skills: string[]
|
||||
subagentTypes: string[]
|
||||
webSearchRequests: number
|
||||
} {
|
||||
const tools: string[] = []
|
||||
const bashCommands: string[] = []
|
||||
const toolSequence: ToolCall[][] = []
|
||||
const skills: string[] = []
|
||||
const subagentTypes: string[] = []
|
||||
let webSearchRequests = 0
|
||||
|
||||
for (const message of messages) {
|
||||
if (message.role !== 'assistant' || !Array.isArray(message.content)) continue
|
||||
const turnTools: ToolCall[] = []
|
||||
|
||||
for (const block of message.content) {
|
||||
if (block?.type !== 'tool_use' && block?.type !== 'server_tool_use') continue
|
||||
const rawName = nonEmptyString(block.name)
|
||||
if (!rawName) continue
|
||||
const mapped = mapToolName(rawName)
|
||||
const input = toolInput(block.input)
|
||||
const toolCall: ToolCall = { tool: mapped }
|
||||
|
||||
const file = firstString(input, ['file_path', 'path', 'target_file', 'file'])
|
||||
if (file) toolCall.file = file
|
||||
const command = firstString(input, ['command', 'cmd'])
|
||||
if (command) toolCall.command = command
|
||||
|
||||
if (mapped === 'Bash' && command) {
|
||||
bashCommands.push(...extractBashCommands(command))
|
||||
}
|
||||
if (mapped === 'Skill') {
|
||||
const skill = firstString(input, ['name', 'skill', 'skill_name'])
|
||||
if (skill) skills.push(skill)
|
||||
}
|
||||
if (mapped === 'Agent') {
|
||||
const subagentType = firstString(input, ['type', 'agent_type', 'profile'])
|
||||
if (subagentType) subagentTypes.push(subagentType)
|
||||
}
|
||||
if (mapped === 'WebSearch') webSearchRequests++
|
||||
|
||||
tools.push(mapped)
|
||||
turnTools.push(toolCall)
|
||||
}
|
||||
|
||||
if (turnTools.length > 0) toolSequence.push(turnTools)
|
||||
}
|
||||
|
||||
return { tools, bashCommands, toolSequence, skills, subagentTypes, webSearchRequests }
|
||||
}
|
||||
|
||||
function reportedCost(cost: CodeWhaleCost | undefined): { value: number; exact: boolean } {
|
||||
if (!cost || typeof cost !== 'object') return { value: 0, exact: false }
|
||||
const hasSessionCost = Object.prototype.hasOwnProperty.call(cost, 'session_cost_usd')
|
||||
const hasSubagentCost = Object.prototype.hasOwnProperty.call(cost, 'subagent_cost_usd')
|
||||
if (!hasSessionCost && !hasSubagentCost) return { value: 0, exact: false }
|
||||
function toProviderCall(rich: CodeWhaleDecodedCall): ParsedProviderCall {
|
||||
const measured = rich.measuredCostUSD !== undefined
|
||||
return {
|
||||
value: safeNonNegativeNumber(cost.session_cost_usd) + safeNonNegativeNumber(cost.subagent_cost_usd),
|
||||
exact: true,
|
||||
}
|
||||
}
|
||||
|
||||
function createParser(source: SessionSource, seenKeys: Set<string>): SessionParser {
|
||||
return {
|
||||
async *parse(): AsyncGenerator<ParsedProviderCall> {
|
||||
const raw = await readSessionFile(source.path)
|
||||
let metadata: CodeWhaleMetadata | null = null
|
||||
let messages: CodeWhaleMessage[] = []
|
||||
|
||||
if (raw !== null) {
|
||||
try {
|
||||
const saved = JSON.parse(raw) as CodeWhaleSession
|
||||
metadata = parseMetadata(saved.metadata)
|
||||
messages = Array.isArray(saved.messages)
|
||||
? saved.messages.filter(isRecord) as CodeWhaleMessage[]
|
||||
: []
|
||||
} catch {
|
||||
// A truncated transcript can still have complete, authoritative
|
||||
// aggregate metadata at the front of the file.
|
||||
}
|
||||
}
|
||||
metadata ??= await readSessionMetadata(source.path)
|
||||
if (!metadata) return
|
||||
const totalTokens = safeTokenCount(metadata.total_tokens)
|
||||
const model = metadata.model ?? metadata.model_provider ?? 'unknown'
|
||||
const localCost = reportedCost(metadata.cost)
|
||||
// Match the pre-lift guard exactly: it skipped zero-token sessions whose
|
||||
// COMPUTED cost was 0 — which included an exact recorded cost of 0.
|
||||
if (totalTokens === 0 && (!localCost.exact || localCost.value === 0)) return
|
||||
|
||||
const deduplicationKey = `codewhale:${metadata.id}`
|
||||
if (seenKeys.has(deduplicationKey)) return
|
||||
seenKeys.add(deduplicationKey)
|
||||
|
||||
let timestamp = normalizeTimestamp(metadata.updated_at) || normalizeTimestamp(metadata.created_at)
|
||||
if (!timestamp) {
|
||||
const fileStat = await stat(source.path).catch(() => null)
|
||||
timestamp = fileStat?.mtime.toISOString() ?? ''
|
||||
}
|
||||
|
||||
const { tools, bashCommands, toolSequence, skills, subagentTypes, webSearchRequests } = collectTools(messages)
|
||||
const workspace = metadata.workspace
|
||||
|
||||
yield {
|
||||
provider: 'codewhale',
|
||||
model,
|
||||
// CodeWhale persists only one aggregate token counter. Preserve it
|
||||
// losslessly in the input column instead of inventing a split.
|
||||
inputTokens: totalTokens,
|
||||
outputTokens: 0,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests,
|
||||
...(localCost.exact
|
||||
? { costUSD: localCost.value, costBasis: 'measured' as const }
|
||||
: { costBasis: 'estimated' as const }),
|
||||
costIsEstimated: !localCost.exact,
|
||||
tools,
|
||||
bashCommands,
|
||||
skills,
|
||||
subagentTypes,
|
||||
timestamp,
|
||||
speed: 'standard',
|
||||
deduplicationKey,
|
||||
turnId: `${metadata.id}:session`,
|
||||
toolSequence: toolSequence.length > 0 ? toolSequence : undefined,
|
||||
userMessage: firstUserMessage(messages),
|
||||
sessionId: metadata.id,
|
||||
project: projectName(workspace),
|
||||
projectPath: workspace,
|
||||
}
|
||||
},
|
||||
provider: 'codewhale',
|
||||
model: rich.model,
|
||||
inputTokens: rich.inputTokens,
|
||||
outputTokens: rich.outputTokens,
|
||||
cacheCreationInputTokens: rich.cacheCreationInputTokens,
|
||||
cacheReadInputTokens: rich.cacheReadInputTokens,
|
||||
cachedInputTokens: rich.cachedInputTokens,
|
||||
reasoningTokens: rich.reasoningTokens,
|
||||
webSearchRequests: rich.webSearchRequests,
|
||||
...(measured
|
||||
? { costUSD: rich.measuredCostUSD, costBasis: 'measured' as const }
|
||||
: { costBasis: 'estimated' as const }),
|
||||
costIsEstimated: !measured,
|
||||
tools: rich.tools,
|
||||
// The legacy codewhale decode deduped nothing: it pushed every extracted base
|
||||
// command into a flat list. Preserve that (no Set) so per-command counts match.
|
||||
bashCommands: rich.rawBashCommands.flatMap(c => extractBashCommands(c)),
|
||||
skills: rich.skills,
|
||||
subagentTypes: rich.subagentTypes,
|
||||
timestamp: rich.timestamp,
|
||||
speed: rich.speed,
|
||||
deduplicationKey: rich.deduplicationKey,
|
||||
turnId: rich.turnId,
|
||||
toolSequence: rich.toolSequence,
|
||||
userMessage: rich.userMessage,
|
||||
sessionId: rich.sessionId,
|
||||
project: rich.project,
|
||||
projectPath: rich.projectPath,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -446,7 +227,7 @@ export function createCodeWhaleProvider(overrideDirs?: string | string[]): Provi
|
|||
? undefined
|
||||
: Array.isArray(overrideDirs) ? overrideDirs : [overrideDirs]
|
||||
|
||||
return {
|
||||
return createBridgedProvider<CodeWhaleDecodedCall>({
|
||||
name: 'codewhale',
|
||||
displayName: 'CodeWhale',
|
||||
|
||||
|
|
@ -455,15 +236,13 @@ export function createCodeWhaleProvider(overrideDirs?: string | string[]): Provi
|
|||
},
|
||||
|
||||
toolDisplayName(rawTool: string): string {
|
||||
return mapToolName(rawTool)
|
||||
return mapCodeWhaleToolName(rawTool)
|
||||
},
|
||||
|
||||
async discoverSessions(): Promise<SessionSource[]> {
|
||||
const seenSessionIds = new Set<string>()
|
||||
const sources: SessionSource[] = []
|
||||
|
||||
// Primary comes before legacy so an id already migrated by CodeWhale is
|
||||
// not counted twice and the primary copy wins without modifying either.
|
||||
for (const dir of configuredDirs ?? defaultSessionDirs()) {
|
||||
for (const candidate of await discoverInDir(dir)) {
|
||||
if (seenSessionIds.has(candidate.id)) continue
|
||||
|
|
@ -474,10 +253,36 @@ export function createCodeWhaleProvider(overrideDirs?: string | string[]): Provi
|
|||
return sources
|
||||
},
|
||||
|
||||
createSessionParser(source: SessionSource, seenKeys: Set<string>): SessionParser {
|
||||
return createParser(source, seenKeys)
|
||||
async readRecords(source: SessionSource): Promise<unknown[] | null> {
|
||||
const [raw, fileStat] = await Promise.all([
|
||||
readSessionFile(source.path),
|
||||
stat(source.path).catch(() => null),
|
||||
])
|
||||
|
||||
let metadata: CodeWhaleMetadata | null = null
|
||||
let messages: CodeWhaleMessage[] = []
|
||||
|
||||
if (raw !== null) {
|
||||
try {
|
||||
const saved = JSON.parse(raw) as { metadata?: unknown; messages?: unknown }
|
||||
metadata = parseMetadata(saved.metadata)
|
||||
messages = Array.isArray(saved.messages) ? (saved.messages.filter(isRecord) as CodeWhaleMessage[]) : []
|
||||
} catch {
|
||||
// A truncated transcript can still have complete aggregate metadata at
|
||||
// the front of the file.
|
||||
}
|
||||
}
|
||||
metadata ??= await readSessionMetadata(source.path)
|
||||
if (!metadata) return null
|
||||
|
||||
const fileMtime = fileStat?.mtime.toISOString() ?? ''
|
||||
const record: CodeWhaleSessionRecords = { metadata, messages, fileMtime }
|
||||
return [record]
|
||||
},
|
||||
}
|
||||
|
||||
decode: decodeCodeWhale,
|
||||
toProviderCall,
|
||||
})
|
||||
}
|
||||
|
||||
export const codewhale = createCodeWhaleProvider()
|
||||
|
|
|
|||
|
|
@ -2,66 +2,20 @@ import { readdir, stat } from 'fs/promises'
|
|||
import { basename, dirname, join } from 'path'
|
||||
import { homedir } from 'os'
|
||||
|
||||
import { decodeGrok, grokToolNameMap } from '@codeburn/core/providers/grok'
|
||||
import type { GrokDecodedCall, GrokSessionRecords, GrokSignals, GrokSummary } from '@codeburn/core/providers/grok'
|
||||
|
||||
import { readSessionFile } from '../fs-utils.js'
|
||||
import { getShortModelName } from '../models.js'
|
||||
import { extractBashCommands } from '../bash-utils.js'
|
||||
import type { Provider, SessionSource, SessionParser, ParsedProviderCall } from './types.js'
|
||||
|
||||
// Grok Build (xAI's coding CLI) stores one session per directory at
|
||||
// <grok-home>/sessions/<url-encoded-cwd>/<uuid>/, where grok-home is $GROK_HOME
|
||||
// or ~/.grok. Each session dir holds summary.json, signals.json, and the ACP
|
||||
// log updates.jsonl.
|
||||
//
|
||||
// Grok does NOT record billable input/output tokens. signals.json carries
|
||||
// `contextTokensUsed` (current context fill) and updates.jsonl carries a running
|
||||
// `_meta.totalTokens` per streamed chunk; there is no per-call input/output
|
||||
// split. We reconstruct an ESTIMATE from the per-turn totalTokens curve. Agentic
|
||||
// turns re-send the growing context every call, and that re-sent context is
|
||||
// cached server-side, so we bill the unique context (summed per compaction segment) as fresh input,
|
||||
// the re-sent remainder as cache reads, and the per-turn growth as output. Cost
|
||||
// is flagged estimated; grok-build is priced via its grok-build-0.1 alias.
|
||||
|
||||
const toolNameMap: Record<string, string> = {
|
||||
bash: 'Bash',
|
||||
run_terminal_command: 'Bash',
|
||||
read_file: 'Read',
|
||||
read: 'Read',
|
||||
write_file: 'Write',
|
||||
edit_file: 'Edit',
|
||||
edit: 'Edit',
|
||||
list_dir: 'Glob',
|
||||
glob: 'Glob',
|
||||
grep: 'Grep',
|
||||
search: 'WebSearch',
|
||||
web_search: 'WebSearch',
|
||||
fetch: 'WebFetch',
|
||||
task: 'Agent',
|
||||
search_replace: 'Edit',
|
||||
todo_write: 'TodoWrite',
|
||||
spawn_subagent: 'Agent',
|
||||
}
|
||||
import { createBridgedProvider } from './bridge.js'
|
||||
import type { Provider, SessionSource, ParsedProviderCall } from './types.js'
|
||||
|
||||
function defaultSessionsDir(): string {
|
||||
const home = process.env['GROK_HOME'] ?? join(homedir(), '.grok')
|
||||
return join(home, 'sessions')
|
||||
}
|
||||
|
||||
type GrokSummary = {
|
||||
info?: { id?: string; cwd?: string }
|
||||
created_at?: string
|
||||
updated_at?: string
|
||||
last_active_at?: string
|
||||
current_model_id?: string
|
||||
session_summary?: string
|
||||
generated_title?: string
|
||||
}
|
||||
|
||||
type GrokSignals = {
|
||||
primaryModelId?: string
|
||||
modelsUsed?: string[]
|
||||
toolsUsed?: string[]
|
||||
}
|
||||
|
||||
async function readJson<T>(path: string): Promise<T | null> {
|
||||
const content = await readSessionFile(path)
|
||||
if (content === null) return null
|
||||
|
|
@ -80,138 +34,6 @@ function safeDecode(name: string): string {
|
|||
}
|
||||
}
|
||||
|
||||
// updates.jsonl is one ACP JSON-RPC notification per line; streamed chunks carry
|
||||
// params._meta.{totalTokens, promptId}. totalTokens is the running context size,
|
||||
// so grouping by promptId (one per turn) gives each turn's first/last value.
|
||||
type GrokUpdate = {
|
||||
params?: {
|
||||
_meta?: { totalTokens?: number; promptId?: string }
|
||||
update?: { sessionUpdate?: string; title?: string; rawInput?: { command?: unknown; subagent_type?: unknown } }
|
||||
}
|
||||
}
|
||||
|
||||
// Single pass over updates.jsonl: per-turn totalTokens for the cost estimate,
|
||||
// plus the real tool calls (each tool_call's title -> a tool, and
|
||||
// run_terminal_command's rawInput.command -> shell commands).
|
||||
function parseUpdates(updates: string): {
|
||||
input: number
|
||||
cacheRead: number
|
||||
output: number
|
||||
tools: string[]
|
||||
bashCommands: string[]
|
||||
subagentTypes: string[]
|
||||
} {
|
||||
const turns = new Map<string, { first: number; last: number }>()
|
||||
const tools: string[] = []
|
||||
const bashCommands: string[] = []
|
||||
const subagentTypes: string[] = []
|
||||
// Compaction-aware fresh input: a large drop in totalTokens means the context
|
||||
// was compacted and rebuilt, so we sum each segment's peak rather than the
|
||||
// single global peak (which would lose everything before the last compaction).
|
||||
let prevTotal = -1
|
||||
let segmentPeak = 0
|
||||
let inputFresh = 0
|
||||
|
||||
for (const line of updates.split('\n')) {
|
||||
if (!line.trim()) continue
|
||||
let params: GrokUpdate['params']
|
||||
try {
|
||||
params = (JSON.parse(line) as GrokUpdate).params
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
if (!params) continue
|
||||
|
||||
const total = params._meta?.totalTokens
|
||||
if (typeof total === 'number') {
|
||||
if (prevTotal >= 0 && total < prevTotal * 0.5) {
|
||||
inputFresh += segmentPeak // close the segment a compaction just ended
|
||||
segmentPeak = 0
|
||||
}
|
||||
if (total > segmentPeak) segmentPeak = total
|
||||
prevTotal = total
|
||||
|
||||
const promptId = params._meta?.promptId
|
||||
if (promptId) {
|
||||
const turn = turns.get(promptId)
|
||||
if (!turn) turns.set(promptId, { first: total, last: total })
|
||||
else turn.last = total
|
||||
}
|
||||
}
|
||||
|
||||
const update = params.update
|
||||
if (update?.sessionUpdate === 'tool_call' && typeof update.title === 'string') {
|
||||
tools.push(toolNameMap[update.title] ?? update.title)
|
||||
if (update.title === 'run_terminal_command' && typeof update.rawInput?.command === 'string') {
|
||||
bashCommands.push(...extractBashCommands(update.rawInput.command))
|
||||
}
|
||||
if (update.title === 'spawn_subagent' && typeof update.rawInput?.subagent_type === 'string') {
|
||||
subagentTypes.push(update.rawInput.subagent_type)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inputFresh += segmentPeak // close the final segment
|
||||
let sumFirst = 0
|
||||
let output = 0
|
||||
for (const { first, last } of turns.values()) {
|
||||
sumFirst += first
|
||||
output += Math.max(0, last - first)
|
||||
}
|
||||
// Fresh input (summed segment peaks) is billed once; the rest of the per-turn
|
||||
// re-sends are cache reads (Grok caches them, even though it reports nothing).
|
||||
const cacheRead = Math.max(0, sumFirst - inputFresh)
|
||||
return { input: inputFresh, cacheRead, output, tools, bashCommands, subagentTypes }
|
||||
}
|
||||
|
||||
function createParser(source: SessionSource, seenKeys: Set<string>): SessionParser {
|
||||
return {
|
||||
async *parse(): AsyncGenerator<ParsedProviderCall> {
|
||||
const dir = dirname(source.path)
|
||||
const summary = await readJson<GrokSummary>(join(dir, 'summary.json'))
|
||||
const updates = await readSessionFile(source.path)
|
||||
if (!summary || updates === null) return
|
||||
|
||||
const { input, cacheRead, output, tools, bashCommands, subagentTypes } = parseUpdates(updates)
|
||||
if (input === 0 && output === 0) return
|
||||
|
||||
const signals = await readJson<GrokSignals>(join(dir, 'signals.json'))
|
||||
const model =
|
||||
summary.current_model_id ?? signals?.primaryModelId ?? signals?.modelsUsed?.[0] ?? 'grok-build'
|
||||
const timestamp = summary.updated_at ?? summary.last_active_at ?? summary.created_at ?? ''
|
||||
const sessionId = summary.info?.id ?? basename(dir)
|
||||
|
||||
const dedupKey = `${source.provider}:${dir}:${timestamp}:${sessionId}`
|
||||
if (seenKeys.has(dedupKey)) return
|
||||
seenKeys.add(dedupKey)
|
||||
|
||||
yield {
|
||||
provider: source.provider,
|
||||
model,
|
||||
inputTokens: input,
|
||||
outputTokens: output,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: cacheRead,
|
||||
cachedInputTokens: cacheRead,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
costBasis: 'estimated',
|
||||
costIsEstimated: true,
|
||||
tools,
|
||||
bashCommands,
|
||||
subagentTypes,
|
||||
timestamp,
|
||||
speed: 'standard',
|
||||
deduplicationKey: dedupKey,
|
||||
userMessage: summary.session_summary ?? summary.generated_title ?? '',
|
||||
sessionId,
|
||||
project: source.project,
|
||||
projectPath: summary.info?.cwd,
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async function discoverSessions(sessionsDir: string): Promise<SessionSource[]> {
|
||||
const sources: SessionSource[] = []
|
||||
|
||||
|
|
@ -250,10 +72,38 @@ async function discoverSessions(sessionsDir: string): Promise<SessionSource[]> {
|
|||
return sources
|
||||
}
|
||||
|
||||
function toProviderCall(rich: GrokDecodedCall): ParsedProviderCall {
|
||||
return {
|
||||
provider: 'grok',
|
||||
model: rich.model,
|
||||
inputTokens: rich.inputTokens,
|
||||
outputTokens: rich.outputTokens,
|
||||
cacheCreationInputTokens: rich.cacheCreationInputTokens,
|
||||
cacheReadInputTokens: rich.cacheReadInputTokens,
|
||||
cachedInputTokens: rich.cachedInputTokens,
|
||||
reasoningTokens: rich.reasoningTokens,
|
||||
webSearchRequests: rich.webSearchRequests,
|
||||
costBasis: 'estimated',
|
||||
costIsEstimated: true,
|
||||
tools: rich.tools,
|
||||
// The legacy grok decode deduped nothing: it pushed every extracted base
|
||||
// command into a flat list. Preserve that (no Set) so per-command counts match.
|
||||
bashCommands: rich.rawBashCommands.flatMap(c => extractBashCommands(c)),
|
||||
subagentTypes: rich.subagentTypes,
|
||||
timestamp: rich.timestamp,
|
||||
speed: rich.speed,
|
||||
deduplicationKey: rich.deduplicationKey,
|
||||
userMessage: rich.userMessage,
|
||||
sessionId: rich.sessionId,
|
||||
project: rich.project,
|
||||
projectPath: rich.projectPath,
|
||||
}
|
||||
}
|
||||
|
||||
export function createGrokProvider(sessionsDir?: string): Provider {
|
||||
const dir = sessionsDir ?? defaultSessionsDir()
|
||||
|
||||
return {
|
||||
return createBridgedProvider<GrokDecodedCall>({
|
||||
name: 'grok',
|
||||
displayName: 'Grok Build',
|
||||
|
||||
|
|
@ -263,17 +113,38 @@ export function createGrokProvider(sessionsDir?: string): Provider {
|
|||
},
|
||||
|
||||
toolDisplayName(rawTool: string): string {
|
||||
return toolNameMap[rawTool] ?? rawTool
|
||||
return grokToolNameMap[rawTool] ?? rawTool
|
||||
},
|
||||
|
||||
async discoverSessions(): Promise<SessionSource[]> {
|
||||
return discoverSessions(dir)
|
||||
},
|
||||
|
||||
createSessionParser(source: SessionSource, seenKeys: Set<string>): SessionParser {
|
||||
return createParser(source, seenKeys)
|
||||
async readRecords(source: SessionSource): Promise<unknown[] | null> {
|
||||
const sessionDir = dirname(source.path)
|
||||
const [summary, updatesRaw] = await Promise.all([
|
||||
readJson<GrokSummary>(join(sessionDir, 'summary.json')),
|
||||
readSessionFile(source.path),
|
||||
])
|
||||
if (!summary || updatesRaw === null) return null
|
||||
|
||||
const signals = await readJson<GrokSignals>(join(sessionDir, 'signals.json'))
|
||||
const cwdEncoded = basename(dirname(sessionDir))
|
||||
const cwd = summary.info?.cwd ?? safeDecode(cwdEncoded)
|
||||
const record: GrokSessionRecords = {
|
||||
summary,
|
||||
signals,
|
||||
updatesLines: updatesRaw.split('\n').filter(l => l.trim()),
|
||||
sourceDir: sessionDir,
|
||||
sessionName: basename(sessionDir),
|
||||
project: basename(cwd),
|
||||
}
|
||||
return [record]
|
||||
},
|
||||
}
|
||||
|
||||
decode: decodeGrok,
|
||||
toProviderCall,
|
||||
})
|
||||
}
|
||||
|
||||
export const grok = createGrokProvider()
|
||||
|
|
|
|||
|
|
@ -3,40 +3,19 @@ import { readdir, readFile, stat } from 'fs/promises'
|
|||
import { basename, dirname, join } from 'path'
|
||||
import { homedir } from 'os'
|
||||
|
||||
import { decodeKimi, kimiToolNameMap } from '@codeburn/core/providers/kimi'
|
||||
import type { KimiDecodedCall } from '@codeburn/core/providers/kimi'
|
||||
|
||||
import { extractBashCommands } from '../bash-utils.js'
|
||||
import { readSessionLines } from '../fs-utils.js'
|
||||
import { readSessionFile } from '../fs-utils.js'
|
||||
import { getShortModelName } from '../models.js'
|
||||
import type { ParsedProviderCall, Provider, SessionParser, SessionSource } from './types.js'
|
||||
import { createBridgedProvider } from './bridge.js'
|
||||
import type { Provider, SessionSource, ParsedProviderCall } from './types.js'
|
||||
|
||||
type JsonObject = Record<string, unknown>
|
||||
|
||||
const toolNameMap: Record<string, string> = {
|
||||
Shell: 'Bash',
|
||||
Bash: 'Bash',
|
||||
bash: 'Bash',
|
||||
ReadFile: 'Read',
|
||||
ReadMediaFile: 'Read',
|
||||
WriteFile: 'Write',
|
||||
StrReplaceFile: 'Edit',
|
||||
Grep: 'Grep',
|
||||
Glob: 'Glob',
|
||||
SearchWeb: 'WebSearch',
|
||||
FetchURL: 'WebFetch',
|
||||
Agent: 'Agent',
|
||||
AgentTool: 'Agent',
|
||||
TaskList: 'Agent',
|
||||
TaskOutput: 'Agent',
|
||||
TaskStop: 'Agent',
|
||||
AskUserQuestion: 'AskUser',
|
||||
SetTodoList: 'TodoWrite',
|
||||
Think: 'Think',
|
||||
EnterPlanMode: 'EnterPlanMode',
|
||||
ExitPlanMode: 'ExitPlanMode',
|
||||
SendDMail: 'DMail',
|
||||
}
|
||||
|
||||
function asObject(value: unknown): JsonObject | null {
|
||||
return value && typeof value === 'object' && !Array.isArray(value) ? value as JsonObject : null
|
||||
return value && typeof value === 'object' && !Array.isArray(value) ? (value as JsonObject) : null
|
||||
}
|
||||
|
||||
function stringField(obj: JsonObject | null, key: string): string | undefined {
|
||||
|
|
@ -44,15 +23,6 @@ function stringField(obj: JsonObject | null, key: string): string | undefined {
|
|||
return typeof value === 'string' ? value : undefined
|
||||
}
|
||||
|
||||
function numericField(obj: JsonObject, ...keys: string[]): number {
|
||||
for (const key of keys) {
|
||||
const raw = obj[key]
|
||||
const n = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : NaN
|
||||
if (Number.isFinite(n) && n > 0) return Math.trunc(n)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
function getShareDir(overrideDir?: string): string {
|
||||
return overrideDir ?? process.env['KIMI_SHARE_DIR'] ?? join(homedir(), '.kimi')
|
||||
}
|
||||
|
|
@ -164,177 +134,38 @@ async function getConfiguredModel(shareDir: string): Promise<string> {
|
|||
return parseModelIdForKey(raw, defaultModel) ?? defaultModel
|
||||
}
|
||||
|
||||
function parseJsonObject(text: string | undefined): JsonObject | null {
|
||||
if (!text) return null
|
||||
try {
|
||||
return asObject(JSON.parse(text))
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function extractUserText(value: unknown): string {
|
||||
if (typeof value === 'string') return value.slice(0, 500)
|
||||
if (!Array.isArray(value)) return ''
|
||||
|
||||
return value
|
||||
.map(part => stringField(asObject(part), 'text') ?? '')
|
||||
.filter(Boolean)
|
||||
.join(' ')
|
||||
.slice(0, 500)
|
||||
}
|
||||
|
||||
function timestampToIso(value: unknown): string {
|
||||
if (typeof value === 'string') return value
|
||||
if (typeof value !== 'number' || !Number.isFinite(value)) return ''
|
||||
|
||||
const millis = value > 1_000_000_000_000 ? value : value * 1000
|
||||
const date = new Date(millis)
|
||||
return Number.isFinite(date.getTime()) ? date.toISOString() : ''
|
||||
}
|
||||
|
||||
function extractEnvelope(record: JsonObject): { type: string; payload: JsonObject; timestamp: string } | null {
|
||||
const message = asObject(record['message'])
|
||||
const envelope = message ?? record
|
||||
const type = stringField(envelope, 'type')
|
||||
const payload = asObject(envelope['payload'])
|
||||
if (!type || !payload) return null
|
||||
return { type, payload, timestamp: timestampToIso(record['timestamp']) }
|
||||
}
|
||||
|
||||
function extractUsage(payload: JsonObject): {
|
||||
inputTokens: number
|
||||
outputTokens: number
|
||||
cacheReadInputTokens: number
|
||||
cacheCreationInputTokens: number
|
||||
} | null {
|
||||
const usage = asObject(payload['token_usage']) ?? asObject(payload['usage'])
|
||||
if (!usage) return null
|
||||
|
||||
const cacheReadInputTokens = numericField(usage, 'input_cache_read', 'cache_read_input_tokens', 'cached_input_tokens')
|
||||
const cacheCreationInputTokens = numericField(usage, 'input_cache_creation', 'cache_creation_input_tokens')
|
||||
let inputTokens = numericField(usage, 'input_other', 'input_tokens')
|
||||
if (inputTokens === 0) {
|
||||
const totalInput = numericField(usage, 'input')
|
||||
inputTokens = Math.max(0, totalInput - cacheReadInputTokens - cacheCreationInputTokens)
|
||||
}
|
||||
const outputTokens = numericField(usage, 'output', 'output_tokens')
|
||||
|
||||
if (inputTokens === 0 && outputTokens === 0 && cacheReadInputTokens === 0 && cacheCreationInputTokens === 0) {
|
||||
return null
|
||||
}
|
||||
|
||||
return { inputTokens, outputTokens, cacheReadInputTokens, cacheCreationInputTokens }
|
||||
}
|
||||
|
||||
function extractTool(payload: JsonObject): { tool: string; bashCommands: string[] } | null {
|
||||
const fn = asObject(payload['function'])
|
||||
const rawName = stringField(fn, 'name') ?? stringField(payload, 'name')
|
||||
if (!rawName) return null
|
||||
|
||||
const tool = toolNameMap[rawName] ?? rawName
|
||||
const argsText = stringField(fn, 'arguments') ?? stringField(payload, 'arguments')
|
||||
const args = parseJsonObject(argsText)
|
||||
const command = stringField(args, 'command')
|
||||
const bashCommands = tool === 'Bash' && command ? extractBashCommands(command) : []
|
||||
|
||||
return { tool, bashCommands }
|
||||
}
|
||||
|
||||
function createParser(source: SessionSource, shareDir: string, seenKeys: Set<string>): SessionParser {
|
||||
return {
|
||||
async *parse(): AsyncGenerator<ParsedProviderCall> {
|
||||
const configuredModel = await getConfiguredModel(shareDir)
|
||||
const tools = new Set<string>()
|
||||
const bashCommands = new Set<string>()
|
||||
let currentUserMessage = ''
|
||||
const sessionId = basename(dirname(source.path))
|
||||
let index = 0
|
||||
|
||||
for await (const line of readSessionLines(source.path)) {
|
||||
if (!line.trim()) continue
|
||||
|
||||
let record: JsonObject | null = null
|
||||
try {
|
||||
record = asObject(JSON.parse(line))
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
if (!record) continue
|
||||
|
||||
const envelope = extractEnvelope(record)
|
||||
if (!envelope || envelope.type === 'metadata') continue
|
||||
|
||||
if (envelope.type === 'TurnBegin' || envelope.type === 'SteerInput') {
|
||||
currentUserMessage = extractUserText(envelope.payload['user_input'])
|
||||
continue
|
||||
}
|
||||
|
||||
if (envelope.type === 'TurnEnd') {
|
||||
currentUserMessage = ''
|
||||
tools.clear()
|
||||
bashCommands.clear()
|
||||
continue
|
||||
}
|
||||
|
||||
if (envelope.type === 'ToolCall' || envelope.type === 'ToolCallRequest') {
|
||||
const extracted = extractTool(envelope.payload)
|
||||
if (!extracted) continue
|
||||
tools.add(extracted.tool)
|
||||
for (const command of extracted.bashCommands) bashCommands.add(command)
|
||||
continue
|
||||
}
|
||||
|
||||
if (envelope.type !== 'StatusUpdate') continue
|
||||
|
||||
const usage = extractUsage(envelope.payload)
|
||||
if (!usage) continue
|
||||
|
||||
const rawMessageId = stringField(envelope.payload, 'message_id')
|
||||
const dedupKey = `kimi:${sessionId}:${rawMessageId ?? index}`
|
||||
index++
|
||||
if (seenKeys.has(dedupKey)) continue
|
||||
seenKeys.add(dedupKey)
|
||||
|
||||
const model = stringField(envelope.payload, 'model') ?? stringField(envelope.payload, 'model_name') ?? configuredModel
|
||||
|
||||
yield {
|
||||
provider: 'kimi',
|
||||
model,
|
||||
inputTokens: usage.inputTokens,
|
||||
outputTokens: usage.outputTokens,
|
||||
cacheCreationInputTokens: usage.cacheCreationInputTokens,
|
||||
cacheReadInputTokens: usage.cacheReadInputTokens,
|
||||
cachedInputTokens: usage.cacheReadInputTokens,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
costBasis: 'estimated',
|
||||
tools: [...tools],
|
||||
bashCommands: [...bashCommands],
|
||||
timestamp: envelope.timestamp,
|
||||
speed: 'standard',
|
||||
deduplicationKey: dedupKey,
|
||||
userMessage: currentUserMessage,
|
||||
sessionId,
|
||||
}
|
||||
|
||||
tools.clear()
|
||||
bashCommands.clear()
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async function addWireSource(sources: SessionSource[], filePath: string, project: string): Promise<void> {
|
||||
const s = await stat(filePath).catch(() => null)
|
||||
if (!s?.isFile()) return
|
||||
sources.push({ path: filePath, project, provider: 'kimi' })
|
||||
}
|
||||
|
||||
function toProviderCall(rich: KimiDecodedCall): ParsedProviderCall {
|
||||
return {
|
||||
provider: 'kimi',
|
||||
model: rich.model,
|
||||
inputTokens: rich.inputTokens,
|
||||
outputTokens: rich.outputTokens,
|
||||
cacheCreationInputTokens: rich.cacheCreationInputTokens,
|
||||
cacheReadInputTokens: rich.cacheReadInputTokens,
|
||||
cachedInputTokens: rich.cachedInputTokens,
|
||||
reasoningTokens: rich.reasoningTokens,
|
||||
webSearchRequests: rich.webSearchRequests,
|
||||
costBasis: 'estimated',
|
||||
tools: rich.tools,
|
||||
bashCommands: [...new Set(rich.rawBashCommands.flatMap(c => extractBashCommands(c)))],
|
||||
timestamp: rich.timestamp,
|
||||
speed: rich.speed,
|
||||
deduplicationKey: rich.deduplicationKey,
|
||||
userMessage: rich.userMessage,
|
||||
sessionId: rich.sessionId,
|
||||
}
|
||||
}
|
||||
|
||||
export function createKimiProvider(overrideDir?: string): Provider {
|
||||
const shareDir = getShareDir(overrideDir)
|
||||
|
||||
return {
|
||||
return createBridgedProvider<KimiDecodedCall>({
|
||||
name: 'kimi',
|
||||
displayName: 'Kimi',
|
||||
|
||||
|
|
@ -343,7 +174,7 @@ export function createKimiProvider(overrideDir?: string): Provider {
|
|||
},
|
||||
|
||||
toolDisplayName(rawTool: string): string {
|
||||
return toolNameMap[rawTool] ?? rawTool
|
||||
return kimiToolNameMap[rawTool] ?? rawTool
|
||||
},
|
||||
|
||||
async discoverSessions(): Promise<SessionSource[]> {
|
||||
|
|
@ -377,10 +208,22 @@ export function createKimiProvider(overrideDir?: string): Provider {
|
|||
return sources
|
||||
},
|
||||
|
||||
createSessionParser(source: SessionSource, seenKeys: Set<string>): SessionParser {
|
||||
return createParser(source, shareDir, seenKeys)
|
||||
async readRecords(source: SessionSource): Promise<unknown[] | null> {
|
||||
const [configuredModel, raw] = await Promise.all([
|
||||
getConfiguredModel(shareDir),
|
||||
readSessionFile(source.path),
|
||||
])
|
||||
if (raw === null) return null
|
||||
return [{
|
||||
lines: raw.split('\n').filter(l => l.trim()),
|
||||
configuredModel,
|
||||
sessionName: basename(dirname(source.path)),
|
||||
}]
|
||||
},
|
||||
}
|
||||
|
||||
decode: decodeKimi,
|
||||
toProviderCall,
|
||||
})
|
||||
}
|
||||
|
||||
export const kimi = createKimiProvider()
|
||||
|
|
|
|||
42
packages/cli/tests/fixtures/codewhale-parity/sessions/full.json
vendored
Normal file
42
packages/cli/tests/fixtures/codewhale-parity/sessions/full.json
vendored
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
{
|
||||
"schema_version": 1,
|
||||
"metadata": {
|
||||
"id": "session-full",
|
||||
"title": "CodeWhale test session",
|
||||
"created_at": "2026-07-14T10:00:00.000Z",
|
||||
"updated_at": "2026-07-15T12:34:56.000Z",
|
||||
"message_count": 6,
|
||||
"total_tokens": 12345,
|
||||
"model": "anthropic/claude-sonnet-4-6",
|
||||
"model_provider": "anthropic",
|
||||
"workspace": "/Users/alice/codewhale-demo",
|
||||
"mode": "agent",
|
||||
"cost": {
|
||||
"session_cost_usd": 0.75,
|
||||
"subagent_cost_usd": 0.20,
|
||||
"displayed_cost_high_water_usd": 2
|
||||
}
|
||||
},
|
||||
"messages": [
|
||||
{ "role": "user", "content": [{ "type": "text", "text": "Implement the parser" }] },
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{ "type": "thinking", "thinking": "..." },
|
||||
{ "type": "tool_use", "id": "t1", "name": "read_file", "input": { "file_path": "src/app.ts" } },
|
||||
{ "type": "tool_use", "id": "t2", "name": "exec_shell", "input": { "command": "npm test && git status && npm run build" } },
|
||||
{ "type": "tool_use", "id": "t3", "name": "edit_file", "input": { "path": "src/app.ts" } },
|
||||
{ "type": "tool_use", "id": "t3b", "name": "read_file", "input": { "file_path": "src/other.ts" } }
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": [
|
||||
{ "type": "tool_use", "id": "t4", "name": "load_skill", "input": { "name": "typescript" } },
|
||||
{ "type": "tool_use", "id": "t5", "name": "agent", "input": { "type": "reviewer" } },
|
||||
{ "type": "server_tool_use", "id": "t6", "name": "web_search", "input": { "query": "CodeWhale" } }
|
||||
]
|
||||
}
|
||||
],
|
||||
"system_prompt": null
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
{
|
||||
"primaryModelId": "grok-build",
|
||||
"modelsUsed": ["grok-build"],
|
||||
"toolsUsed": ["read_file", "run_terminal_command", "grep"],
|
||||
"contextTokensUsed": 40000,
|
||||
"contextWindowTokens": 512000
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
{
|
||||
"info": {
|
||||
"id": "019edf9c-0000-7000-8000-000000000001",
|
||||
"cwd": "/Users/test/myproject"
|
||||
},
|
||||
"created_at": "2026-06-19T11:20:40.686261Z",
|
||||
"updated_at": "2026-06-19T11:31:12.282793Z",
|
||||
"last_active_at": "2026-06-19T11:31:12.222328Z",
|
||||
"num_messages": 42,
|
||||
"current_model_id": "grok-build",
|
||||
"session_summary": "User asks about the repo",
|
||||
"generated_title": "User asks about the repo"
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
{"timestamp":"2026-06-19T11:30:00.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"hi"}},"_meta":{"totalTokens":20000,"promptId":"p1","updateType":"AgentMessageChunk","modelId":"grok-build"}}}
|
||||
{"timestamp":"2026-06-19T11:30:00.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"hi"}},"_meta":{"totalTokens":25000,"promptId":"p1","updateType":"AgentMessageChunk","modelId":"grok-build"}}}
|
||||
{"timestamp":"2026-06-19T11:30:00.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"hi"}},"_meta":{"totalTokens":30000,"promptId":"p2","updateType":"AgentMessageChunk","modelId":"grok-build"}}}
|
||||
{"timestamp":"2026-06-19T11:30:00.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"hi"}},"_meta":{"totalTokens":35000,"promptId":"p2","updateType":"AgentMessageChunk","modelId":"grok-build"}}}
|
||||
{"timestamp":"2026-06-19T11:30:00.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"hi"}},"_meta":{"totalTokens":40000,"promptId":"p3","updateType":"AgentMessageChunk","modelId":"grok-build"}}}
|
||||
{"timestamp":"2026-06-19T11:30:00.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"hi"}},"_meta":{"totalTokens":45000,"promptId":"p3","updateType":"AgentMessageChunk","modelId":"grok-build"}}}
|
||||
{"timestamp":"2026-06-19T11:30:05.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"tool_call","toolCallId":"c1","title":"read_file","rawInput":{"target_directory":"."}}}}
|
||||
{"timestamp":"2026-06-19T11:30:05.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"tool_call","toolCallId":"c2","title":"grep","rawInput":{"pattern":"x"}}}}
|
||||
{"timestamp":"2026-06-19T11:30:05.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"tool_call","toolCallId":"c3","title":"run_terminal_command","rawInput":{"command":"git status && git log"}}}}
|
||||
{"timestamp":"2026-06-19T11:30:05.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"tool_call","toolCallId":"c4","title":"run_terminal_command","rawInput":{"command":"git diff"}}}}
|
||||
{"timestamp":"2026-06-19T11:30:05.000Z","method":"session/update","params":{"sessionId":"019edf9c-0000-7000-8000-000000000001","update":{"sessionUpdate":"tool_call","toolCallId":"c5","title":"spawn_subagent","rawInput":{"subagent_type":"general-purpose","prompt":"x"}}}}
|
||||
4
packages/cli/tests/fixtures/kimi-parity/config.toml
vendored
Normal file
4
packages/cli/tests/fixtures/kimi-parity/config.toml
vendored
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
default_model = "kimi-code/k2"
|
||||
|
||||
[models."kimi-code/k2"]
|
||||
model = "kimi-k2-thinking-turbo"
|
||||
5
packages/cli/tests/fixtures/kimi-parity/kimi.json
vendored
Normal file
5
packages/cli/tests/fixtures/kimi-parity/kimi.json
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"work_dirs": [
|
||||
{ "path": "/Users/test/work/app", "kaos": "local", "last_session_id": "sess-1" }
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
{"type":"metadata","protocol_version":"2"}
|
||||
{"timestamp":1776162400,"message":{"type":"TurnBegin","payload":{"user_input":"add status endpoint"}}}
|
||||
{"timestamp":1776162401,"message":{"type":"ToolCall","payload":{"type":"function","id":"call-shell","function":{"name":"Shell","arguments":"{\"command\":\"git status && npm test\"}"}}}}
|
||||
{"timestamp":1776162402,"message":{"type":"ToolCall","payload":{"type":"function","id":"call-read","function":{"name":"ReadFile","arguments":"{\"path\":\"src/index.ts\"}"}}}}
|
||||
{"timestamp":1776162403,"message":{"type":"ToolCall","payload":{"type":"function","id":"call-read2","function":{"name":"ReadFile","arguments":"{\"path\":\"src/other.ts\"}"}}}}
|
||||
{"timestamp":1776162404,"message":{"type":"ToolCall","payload":{"type":"function","id":"call-shell2","function":{"name":"Shell","arguments":"{\"command\":\"git status && npm test\"}"}}}}
|
||||
{"timestamp":1776162405,"message":{"type":"StatusUpdate","payload":{"message_id":"msg-1","token_usage":{"input_other":100,"input_cache_read":25,"input_cache_creation":10,"output":40}}}}
|
||||
104
packages/cli/tests/providers/codewhale-bridge.test.ts
Normal file
104
packages/cli/tests/providers/codewhale-bridge.test.ts
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
import { dirname, resolve } from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
|
||||
import { describe, it, expect } from 'vitest'
|
||||
|
||||
import { createCodeWhaleProvider } from '../../src/providers/codewhale.js'
|
||||
import { priceProviderCall } from '../../src/pricing-pass.js'
|
||||
import type { ParsedProviderCall, SessionSource, ToolCall } from '../../src/providers/types.js'
|
||||
|
||||
// Byte-identical parity gate for the codewhale bridge migration (phase 8). The
|
||||
// GOLDEN below was captured from the legacy in-CLI decode before the migration.
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url))
|
||||
const FIXTURE_DIR = resolve(here, '../fixtures/codewhale-parity/sessions')
|
||||
|
||||
const GOLDEN: ParsedProviderCall[] = [
|
||||
{
|
||||
provider: 'codewhale',
|
||||
model: 'anthropic/claude-sonnet-4-6',
|
||||
inputTokens: 12345,
|
||||
outputTokens: 0,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 1,
|
||||
costUSD: 0.95,
|
||||
costBasis: 'measured',
|
||||
costIsEstimated: false,
|
||||
// CodeWhale dedups nothing: Read appears twice (two read_file calls) and
|
||||
// 'npm' repeats across `npm test && git status && npm run build`.
|
||||
tools: ['Read', 'Bash', 'Edit', 'Read', 'Skill', 'Agent', 'WebSearch'],
|
||||
bashCommands: ['npm', 'git', 'npm'],
|
||||
skills: ['typescript'],
|
||||
subagentTypes: ['reviewer'],
|
||||
timestamp: '2026-07-15T12:34:56.000Z',
|
||||
speed: 'standard',
|
||||
deduplicationKey: 'codewhale:session-full',
|
||||
turnId: 'session-full:session',
|
||||
toolSequence: [
|
||||
[
|
||||
{ tool: 'Read', file: 'src/app.ts' },
|
||||
{ tool: 'Bash', command: 'npm test && git status && npm run build' },
|
||||
{ tool: 'Edit', file: 'src/app.ts' },
|
||||
{ tool: 'Read', file: 'src/other.ts' },
|
||||
] as ToolCall[],
|
||||
[
|
||||
{ tool: 'Skill' },
|
||||
{ tool: 'Agent' },
|
||||
{ tool: 'WebSearch' },
|
||||
] as ToolCall[],
|
||||
],
|
||||
userMessage: 'Implement the parser',
|
||||
sessionId: 'session-full',
|
||||
project: 'codewhale-demo',
|
||||
projectPath: '/Users/alice/codewhale-demo',
|
||||
},
|
||||
]
|
||||
|
||||
async function collect(): Promise<ParsedProviderCall[]> {
|
||||
const provider = createCodeWhaleProvider(FIXTURE_DIR)
|
||||
const sources: SessionSource[] = await provider.discoverSessions()
|
||||
sources.sort((a, b) => a.path.localeCompare(b.path))
|
||||
const seen = new Set<string>()
|
||||
const calls: ParsedProviderCall[] = []
|
||||
for (const source of sources) {
|
||||
for await (const call of provider.createSessionParser(source, seen).parse()) {
|
||||
calls.push(call)
|
||||
}
|
||||
}
|
||||
return calls
|
||||
}
|
||||
|
||||
describe('codewhale bridge — fixture parity', () => {
|
||||
it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => {
|
||||
expect(await collect()).toEqual(GOLDEN)
|
||||
})
|
||||
|
||||
it('the priced output survives the pricing pass unchanged for measured cost', async () => {
|
||||
const raw = await collect()
|
||||
const priced = raw.map(priceProviderCall)
|
||||
priced.forEach((call, i) => {
|
||||
// Measured-cost calls already carry costUSD/costBasis; the pricing pass
|
||||
// leaves them untouched.
|
||||
expect(call).toEqual(raw[i])
|
||||
})
|
||||
})
|
||||
|
||||
it('dedup threads through the host-owned seenKeys set', async () => {
|
||||
const provider = createCodeWhaleProvider(FIXTURE_DIR)
|
||||
const sources = await provider.discoverSessions()
|
||||
const seen = new Set<string>()
|
||||
const first: ParsedProviderCall[] = []
|
||||
for (const source of sources) {
|
||||
for await (const call of provider.createSessionParser(source, seen).parse()) first.push(call)
|
||||
}
|
||||
const second: ParsedProviderCall[] = []
|
||||
for (const source of sources) {
|
||||
for await (const call of provider.createSessionParser(source, seen).parse()) second.push(call)
|
||||
}
|
||||
expect(first.length).toBe(1)
|
||||
expect(second).toEqual([])
|
||||
})
|
||||
})
|
||||
90
packages/cli/tests/providers/grok-bridge.test.ts
Normal file
90
packages/cli/tests/providers/grok-bridge.test.ts
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
import { dirname, resolve } from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
|
||||
import { describe, it, expect } from 'vitest'
|
||||
|
||||
import { createGrokProvider } from '../../src/providers/grok.js'
|
||||
import { priceProviderCall } from '../../src/pricing-pass.js'
|
||||
import type { ParsedProviderCall, SessionSource } from '../../src/providers/types.js'
|
||||
|
||||
// Byte-identical parity gate for the grok bridge migration (phase 8). The
|
||||
// GOLDEN below was captured from the legacy in-CLI decode before the migration.
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url))
|
||||
const FIXTURE_DIR = resolve(here, '../fixtures/grok-parity')
|
||||
|
||||
const GOLDEN: ParsedProviderCall[] = [
|
||||
{
|
||||
provider: 'grok',
|
||||
model: 'grok-build',
|
||||
inputTokens: 45000,
|
||||
outputTokens: 15000,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 45000,
|
||||
cachedInputTokens: 45000,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
costBasis: 'estimated',
|
||||
costIsEstimated: true,
|
||||
// Grok dedups nothing: Bash appears twice (two run_terminal_command calls)
|
||||
// and 'git' repeats across `git status && git log` + `git diff`.
|
||||
tools: ['Read', 'Grep', 'Bash', 'Bash', 'Agent'],
|
||||
bashCommands: ['git', 'git', 'git'],
|
||||
subagentTypes: ['general-purpose'],
|
||||
timestamp: '2026-06-19T11:31:12.282793Z',
|
||||
speed: 'standard',
|
||||
deduplicationKey: 'grok:/Users/torukmakto/Projects/codeburn/codeburn/.claude/worktrees/kimi-p8a/packages/cli/tests/fixtures/grok-parity/%2FUsers%2Ftest/019edf9c-0000-7000-8000-000000000001:2026-06-19T11:31:12.282793Z:019edf9c-0000-7000-8000-000000000001',
|
||||
userMessage: 'User asks about the repo',
|
||||
sessionId: '019edf9c-0000-7000-8000-000000000001',
|
||||
project: 'myproject',
|
||||
projectPath: '/Users/test/myproject',
|
||||
},
|
||||
]
|
||||
|
||||
async function collect(): Promise<ParsedProviderCall[]> {
|
||||
const provider = createGrokProvider(FIXTURE_DIR)
|
||||
const sources: SessionSource[] = await provider.discoverSessions()
|
||||
sources.sort((a, b) => a.path.localeCompare(b.path))
|
||||
const seen = new Set<string>()
|
||||
const calls: ParsedProviderCall[] = []
|
||||
for (const source of sources) {
|
||||
for await (const call of provider.createSessionParser(source, seen).parse()) {
|
||||
calls.push(call)
|
||||
}
|
||||
}
|
||||
return calls
|
||||
}
|
||||
|
||||
describe('grok bridge — fixture parity', () => {
|
||||
it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => {
|
||||
expect(await collect()).toEqual(GOLDEN)
|
||||
})
|
||||
|
||||
it('the priced output survives the pricing pass with only costUSD added', async () => {
|
||||
const raw = await collect()
|
||||
const priced = raw.map(priceProviderCall)
|
||||
priced.forEach((call, i) => {
|
||||
expect(typeof call.costUSD).toBe('number')
|
||||
expect(Number.isFinite(call.costUSD)).toBe(true)
|
||||
expect(call.costBasis).toBe('estimated')
|
||||
const { costUSD, ...rest } = call
|
||||
expect(rest).toEqual(raw[i])
|
||||
})
|
||||
})
|
||||
|
||||
it('dedup threads through the host-owned seenKeys set', async () => {
|
||||
const provider = createGrokProvider(FIXTURE_DIR)
|
||||
const sources = await provider.discoverSessions()
|
||||
const seen = new Set<string>()
|
||||
const first: ParsedProviderCall[] = []
|
||||
for (const source of sources) {
|
||||
for await (const call of provider.createSessionParser(source, seen).parse()) first.push(call)
|
||||
}
|
||||
const second: ParsedProviderCall[] = []
|
||||
for (const source of sources) {
|
||||
for await (const call of provider.createSessionParser(source, seen).parse()) second.push(call)
|
||||
}
|
||||
expect(first.length).toBe(1)
|
||||
expect(second).toEqual([])
|
||||
})
|
||||
})
|
||||
86
packages/cli/tests/providers/kimi-bridge.test.ts
Normal file
86
packages/cli/tests/providers/kimi-bridge.test.ts
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
import { dirname, resolve } from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
|
||||
import { describe, it, expect } from 'vitest'
|
||||
|
||||
import { createKimiProvider } from '../../src/providers/kimi.js'
|
||||
import { priceProviderCall } from '../../src/pricing-pass.js'
|
||||
import type { ParsedProviderCall, SessionSource } from '../../src/providers/types.js'
|
||||
|
||||
// Byte-identical parity gate for the kimi bridge migration (phase 8). The
|
||||
// GOLDEN below was captured from the legacy in-CLI decode before the migration.
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url))
|
||||
const FIXTURE_DIR = resolve(here, '../fixtures/kimi-parity')
|
||||
|
||||
const GOLDEN: ParsedProviderCall[] = [
|
||||
{
|
||||
provider: 'kimi',
|
||||
model: 'kimi-k2-thinking-turbo',
|
||||
inputTokens: 100,
|
||||
outputTokens: 40,
|
||||
cacheCreationInputTokens: 10,
|
||||
cacheReadInputTokens: 25,
|
||||
cachedInputTokens: 25,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
costBasis: 'estimated',
|
||||
// Kimi dedups per turn (Set): the repeated ReadFile and second Shell call
|
||||
// collapse, so tools and bashCommands stay unique.
|
||||
tools: ['Bash', 'Read'],
|
||||
bashCommands: ['git', 'npm'],
|
||||
timestamp: '2026-04-14T10:26:45.000Z',
|
||||
speed: 'standard',
|
||||
deduplicationKey: 'kimi:sess-1:msg-1',
|
||||
userMessage: 'add status endpoint',
|
||||
sessionId: 'sess-1',
|
||||
},
|
||||
]
|
||||
|
||||
async function collect(): Promise<ParsedProviderCall[]> {
|
||||
const provider = createKimiProvider(FIXTURE_DIR)
|
||||
const sources: SessionSource[] = await provider.discoverSessions()
|
||||
sources.sort((a, b) => a.path.localeCompare(b.path))
|
||||
const seen = new Set<string>()
|
||||
const calls: ParsedProviderCall[] = []
|
||||
for (const source of sources) {
|
||||
for await (const call of provider.createSessionParser(source, seen).parse()) {
|
||||
calls.push(call)
|
||||
}
|
||||
}
|
||||
return calls
|
||||
}
|
||||
|
||||
describe('kimi bridge — fixture parity', () => {
|
||||
it('the bridged provider reproduces the pre-migration decode byte-for-byte', async () => {
|
||||
expect(await collect()).toEqual(GOLDEN)
|
||||
})
|
||||
|
||||
it('the priced output survives the pricing pass with only costUSD added', async () => {
|
||||
const raw = await collect()
|
||||
const priced = raw.map(priceProviderCall)
|
||||
priced.forEach((call, i) => {
|
||||
expect(typeof call.costUSD).toBe('number')
|
||||
expect(Number.isFinite(call.costUSD)).toBe(true)
|
||||
expect(call.costBasis).toBe('estimated')
|
||||
const { costUSD, ...rest } = call
|
||||
expect(rest).toEqual(raw[i])
|
||||
})
|
||||
})
|
||||
|
||||
it('dedup threads through the host-owned seenKeys set', async () => {
|
||||
const provider = createKimiProvider(FIXTURE_DIR)
|
||||
const sources = await provider.discoverSessions()
|
||||
const seen = new Set<string>()
|
||||
const first: ParsedProviderCall[] = []
|
||||
for (const source of sources) {
|
||||
for await (const call of provider.createSessionParser(source, seen).parse()) first.push(call)
|
||||
}
|
||||
const second: ParsedProviderCall[] = []
|
||||
for (const source of sources) {
|
||||
for await (const call of provider.createSessionParser(source, seen).parse()) second.push(call)
|
||||
}
|
||||
expect(first.length).toBe(1)
|
||||
expect(second).toEqual([])
|
||||
})
|
||||
})
|
||||
|
|
@ -39,10 +39,22 @@
|
|||
"types": "./dist/providers/claude/index.d.ts",
|
||||
"import": "./dist/providers/claude/index.js"
|
||||
},
|
||||
"./providers/codewhale": {
|
||||
"types": "./dist/providers/codewhale/index.d.ts",
|
||||
"import": "./dist/providers/codewhale/index.js"
|
||||
},
|
||||
"./providers/codex": {
|
||||
"types": "./dist/providers/codex/index.d.ts",
|
||||
"import": "./dist/providers/codex/index.js"
|
||||
},
|
||||
"./providers/grok": {
|
||||
"types": "./dist/providers/grok/index.d.ts",
|
||||
"import": "./dist/providers/grok/index.js"
|
||||
},
|
||||
"./providers/kimi": {
|
||||
"types": "./dist/providers/kimi/index.d.ts",
|
||||
"import": "./dist/providers/kimi/index.js"
|
||||
},
|
||||
"./providers/qwen": {
|
||||
"types": "./dist/providers/qwen/index.d.ts",
|
||||
"import": "./dist/providers/qwen/index.js"
|
||||
|
|
|
|||
248
packages/core/src/providers/codewhale/decode.ts
Normal file
248
packages/core/src/providers/codewhale/decode.ts
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
// @codeburn/core CodeWhale decoder: pure decode over a host-supplied parsed
|
||||
// session. The host reads the JSON file (with an optional prefix fallback for
|
||||
// oversized transcripts); this decoder is stateless, does no fs/env/clock
|
||||
// access, and performs no pricing calculations.
|
||||
|
||||
import type { DecodeContext } from '../../contracts.js'
|
||||
import type { RecordDiagnostic } from '../../diagnostics.js'
|
||||
import type {
|
||||
CodeWhaleDecodedCall,
|
||||
CodeWhaleMessage,
|
||||
CodeWhaleMetadata,
|
||||
CodeWhaleSessionRecords,
|
||||
CodeWhaleToolCall,
|
||||
} from './types.js'
|
||||
|
||||
export const codeWhaleToolNameMap: Record<string, string> = {
|
||||
exec_shell: 'Bash',
|
||||
exec_shell_wait: 'Bash',
|
||||
exec_shell_interact: 'Bash',
|
||||
exec_shell_cancel: 'Bash',
|
||||
task_shell_start: 'Bash',
|
||||
task_shell_wait: 'Bash',
|
||||
terminal_run: 'Bash',
|
||||
terminal_send: 'Bash',
|
||||
terminal_wait: 'Bash',
|
||||
terminal_cancel: 'Bash',
|
||||
read_file: 'Read',
|
||||
write_file: 'Write',
|
||||
edit_file: 'Edit',
|
||||
fim_edit: 'Edit',
|
||||
apply_patch: 'Edit',
|
||||
list_dir: 'Glob',
|
||||
grep_files: 'Grep',
|
||||
web_search: 'WebSearch',
|
||||
fetch_url: 'WebFetch',
|
||||
'web.run': 'WebSearch',
|
||||
agent: 'Agent',
|
||||
'agents/list': 'Agent',
|
||||
'agents/message': 'Agent',
|
||||
'agents/followup': 'Agent',
|
||||
'agents/interrupt': 'Agent',
|
||||
'agents/wait': 'Agent',
|
||||
todo_write: 'TodoWrite',
|
||||
todo_add: 'TodoWrite',
|
||||
todo_update: 'TodoWrite',
|
||||
todo_list: 'TodoWrite',
|
||||
checklist_write: 'TodoWrite',
|
||||
checklist_add: 'TodoWrite',
|
||||
checklist_update: 'TodoWrite',
|
||||
checklist_list: 'TodoWrite',
|
||||
update_plan: 'TodoWrite',
|
||||
load_skill: 'Skill',
|
||||
request_user_input: 'AskUser',
|
||||
}
|
||||
|
||||
export function mapCodeWhaleToolName(rawName: string): string {
|
||||
if (rawName.startsWith('mcp__')) return rawName
|
||||
if (rawName.startsWith('agents/')) return 'Agent'
|
||||
return Object.prototype.hasOwnProperty.call(codeWhaleToolNameMap, rawName)
|
||||
? codeWhaleToolNameMap[rawName]!
|
||||
: rawName
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
||||
}
|
||||
|
||||
function nonEmptyString(value: unknown): string | undefined {
|
||||
return typeof value === 'string' && value.trim() ? value.trim() : undefined
|
||||
}
|
||||
|
||||
function safeNonNegativeNumber(value: unknown): number {
|
||||
return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : 0
|
||||
}
|
||||
|
||||
function safeTokenCount(value: unknown): number {
|
||||
return Math.floor(Math.min(safeNonNegativeNumber(value), Number.MAX_SAFE_INTEGER))
|
||||
}
|
||||
|
||||
function normalizeTimestamp(value: string | undefined): string {
|
||||
if (!value) return ''
|
||||
const timestamp = Date.parse(value)
|
||||
return Number.isFinite(timestamp) ? new Date(timestamp).toISOString() : ''
|
||||
}
|
||||
|
||||
function firstUserMessage(messages: CodeWhaleMessage[]): string {
|
||||
for (const message of messages) {
|
||||
if (message.role !== 'user') continue
|
||||
const text =
|
||||
typeof message.content === 'string'
|
||||
? message.content
|
||||
: Array.isArray(message.content)
|
||||
? message.content
|
||||
.filter(block => block?.type === 'text' && typeof block.text === 'string')
|
||||
.map(block => block.text)
|
||||
.join(' ')
|
||||
: ''
|
||||
if (text.trim()) return Array.from(text.trim()).slice(0, 500).join('')
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function toolInput(value: unknown): Record<string, unknown> {
|
||||
return isRecord(value) ? value : {}
|
||||
}
|
||||
|
||||
function firstString(input: Record<string, unknown>, keys: string[]): string | undefined {
|
||||
for (const key of keys) {
|
||||
const value = nonEmptyString(input[key])
|
||||
if (value) return value
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
function collectTools(messages: CodeWhaleMessage[]): {
|
||||
tools: string[]
|
||||
rawBashCommands: string[]
|
||||
toolSequence: CodeWhaleToolCall[][]
|
||||
skills: string[]
|
||||
subagentTypes: string[]
|
||||
webSearchRequests: number
|
||||
} {
|
||||
const tools: string[] = []
|
||||
const rawBashCommands: string[] = []
|
||||
const toolSequence: CodeWhaleToolCall[][] = []
|
||||
const skills: string[] = []
|
||||
const subagentTypes: string[] = []
|
||||
let webSearchRequests = 0
|
||||
|
||||
for (const message of messages) {
|
||||
if (message.role !== 'assistant' || !Array.isArray(message.content)) continue
|
||||
const turnTools: CodeWhaleToolCall[] = []
|
||||
|
||||
for (const block of message.content) {
|
||||
if (block?.type !== 'tool_use' && block?.type !== 'server_tool_use') continue
|
||||
const rawName = nonEmptyString(block.name)
|
||||
if (!rawName) continue
|
||||
const mapped = mapCodeWhaleToolName(rawName)
|
||||
const input = toolInput(block.input)
|
||||
const toolCall: CodeWhaleToolCall = { tool: mapped }
|
||||
|
||||
const file = firstString(input, ['file_path', 'path', 'target_file', 'file'])
|
||||
if (file) toolCall.file = file
|
||||
const command = firstString(input, ['command', 'cmd'])
|
||||
if (command) toolCall.command = command
|
||||
|
||||
if (mapped === 'Bash' && command) {
|
||||
rawBashCommands.push(command)
|
||||
}
|
||||
if (mapped === 'Skill') {
|
||||
const skill = firstString(input, ['name', 'skill', 'skill_name'])
|
||||
if (skill) skills.push(skill)
|
||||
}
|
||||
if (mapped === 'Agent') {
|
||||
const subagentType = firstString(input, ['type', 'agent_type', 'profile'])
|
||||
if (subagentType) subagentTypes.push(subagentType)
|
||||
}
|
||||
if (mapped === 'WebSearch') webSearchRequests++
|
||||
|
||||
tools.push(mapped)
|
||||
turnTools.push(toolCall)
|
||||
}
|
||||
|
||||
if (turnTools.length > 0) toolSequence.push(turnTools)
|
||||
}
|
||||
|
||||
return { tools, rawBashCommands, toolSequence, skills, subagentTypes, webSearchRequests }
|
||||
}
|
||||
|
||||
function reportedCost(cost: CodeWhaleMetadata['cost']): { value: number; exact: boolean } {
|
||||
if (!cost || typeof cost !== 'object') return { value: 0, exact: false }
|
||||
const hasSessionCost = Object.prototype.hasOwnProperty.call(cost, 'session_cost_usd')
|
||||
const hasSubagentCost = Object.prototype.hasOwnProperty.call(cost, 'subagent_cost_usd')
|
||||
if (!hasSessionCost && !hasSubagentCost) return { value: 0, exact: false }
|
||||
return {
|
||||
value: safeNonNegativeNumber(cost.session_cost_usd) + safeNonNegativeNumber(cost.subagent_cost_usd),
|
||||
exact: true,
|
||||
}
|
||||
}
|
||||
|
||||
function isCodeWhaleSessionRecords(value: unknown): value is CodeWhaleSessionRecords {
|
||||
return value !== null && typeof value === 'object' && 'metadata' in (value as object)
|
||||
}
|
||||
|
||||
export type CodeWhaleDecodeInput = {
|
||||
records: unknown[]
|
||||
context: DecodeContext
|
||||
seenKeys?: Set<string>
|
||||
}
|
||||
|
||||
export type CodeWhaleDecodeResult = {
|
||||
calls: CodeWhaleDecodedCall[]
|
||||
diagnostics: RecordDiagnostic[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode one CodeWhale session's composite record into a single rich, cost-free
|
||||
* call. The host owns file I/O and the live cross-file dedup set; this function
|
||||
* is pure over the supplied record.
|
||||
*/
|
||||
export function decodeCodeWhale({ records, seenKeys: liveSeen }: CodeWhaleDecodeInput): CodeWhaleDecodeResult {
|
||||
const seen = liveSeen ?? new Set<string>()
|
||||
const session = records.find(isCodeWhaleSessionRecords)
|
||||
if (!session) return { calls: [], diagnostics: [] }
|
||||
|
||||
const { metadata, messages, fileMtime } = session
|
||||
const totalTokens = safeTokenCount(metadata.total_tokens)
|
||||
const model = metadata.model ?? metadata.model_provider ?? 'unknown'
|
||||
const localCost = reportedCost(metadata.cost)
|
||||
if (totalTokens === 0 && (!localCost.exact || localCost.value === 0)) return { calls: [], diagnostics: [] }
|
||||
|
||||
const deduplicationKey = `codewhale:${metadata.id}`
|
||||
if (seen.has(deduplicationKey)) return { calls: [], diagnostics: [] }
|
||||
seen.add(deduplicationKey)
|
||||
|
||||
const timestamp = normalizeTimestamp(metadata.updated_at) || normalizeTimestamp(metadata.created_at) || fileMtime
|
||||
const workspace = metadata.workspace
|
||||
|
||||
const { tools, rawBashCommands, toolSequence, skills, subagentTypes, webSearchRequests } = collectTools(messages)
|
||||
|
||||
const call: CodeWhaleDecodedCall = {
|
||||
provider: 'codewhale',
|
||||
model,
|
||||
inputTokens: totalTokens,
|
||||
outputTokens: 0,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: 0,
|
||||
cachedInputTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests,
|
||||
...(localCost.exact ? { measuredCostUSD: localCost.value } : {}),
|
||||
tools,
|
||||
rawBashCommands,
|
||||
skills,
|
||||
subagentTypes,
|
||||
timestamp,
|
||||
speed: 'standard',
|
||||
deduplicationKey,
|
||||
turnId: `${metadata.id}:session`,
|
||||
...(toolSequence.length > 0 ? { toolSequence } : {}),
|
||||
userMessage: firstUserMessage(messages),
|
||||
sessionId: metadata.id,
|
||||
project: workspace ? workspace.replace(/[\\/]+$/, '').split(/[\\/]/).filter(Boolean).at(-1) ?? 'CodeWhale' : 'CodeWhale',
|
||||
projectPath: workspace ?? '',
|
||||
}
|
||||
|
||||
return { calls: [call], diagnostics: [] }
|
||||
}
|
||||
30
packages/core/src/providers/codewhale/index.ts
Normal file
30
packages/core/src/providers/codewhale/index.ts
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
// @codeburn/core CodeWhale provider.
|
||||
//
|
||||
// Two layers:
|
||||
// - Rich pure decode (`decodeCodeWhale`): host-facing, NOT part of the stable
|
||||
// minimized surface. Pure over the host-supplied parsed session; carries
|
||||
// content in-memory but performs no pricing and no bash base-name extraction.
|
||||
// - Minimizing transform (`toObservations`): maps the rich decode into the
|
||||
// strict observation envelope; the content-smuggling guarantees bind here.
|
||||
|
||||
export {
|
||||
decodeCodeWhale,
|
||||
codeWhaleToolNameMap,
|
||||
mapCodeWhaleToolName,
|
||||
type CodeWhaleDecodeInput,
|
||||
type CodeWhaleDecodeResult,
|
||||
} from './decode.js'
|
||||
|
||||
export {
|
||||
toObservations,
|
||||
type RichCodeWhaleSessionDecode,
|
||||
type CodeWhaleToObservationsContext,
|
||||
} from './observations.js'
|
||||
|
||||
export type {
|
||||
CodeWhaleDecodedCall,
|
||||
CodeWhaleSessionRecords,
|
||||
CodeWhaleMetadata,
|
||||
CodeWhaleMessage,
|
||||
CodeWhaleToolCall,
|
||||
} from './types.js'
|
||||
91
packages/core/src/providers/codewhale/observations.ts
Normal file
91
packages/core/src/providers/codewhale/observations.ts
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
// Minimizing transform: rich CodeWhale decode -> the strict observation envelope.
|
||||
//
|
||||
// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool
|
||||
// names cross into the output. Project paths are fingerprinted; user messages,
|
||||
// commands, and file paths stay behind.
|
||||
|
||||
import { projectRef, sessionRef } from '../../fingerprint.js'
|
||||
import type { RecordDiagnostic } from '../../diagnostics.js'
|
||||
import type { CallObservation, SessionObservation } from '../../observations.js'
|
||||
import { extractResourceRefs } from '../resource-refs.js'
|
||||
import type { CodeWhaleDecodedCall } from './types.js'
|
||||
|
||||
/** One CodeWhale session's rich decode, as the host holds it before minimization. */
|
||||
export interface RichCodeWhaleSessionDecode {
|
||||
sessionId: string
|
||||
/** Absolute project path (the session workspace); fingerprinted, never emitted raw. */
|
||||
projectPath: string
|
||||
/** Rich, cost-free calls in decode order (CodeWhale emits one per session). */
|
||||
calls: CodeWhaleDecodedCall[]
|
||||
}
|
||||
|
||||
export interface CodeWhaleToObservationsContext {
|
||||
/** HMAC key that scopes every fingerprint. */
|
||||
privacyKey: string
|
||||
/** Provider id stamped onto sessions/calls and folded into sessionRef. */
|
||||
provider?: string
|
||||
}
|
||||
|
||||
const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/
|
||||
|
||||
function toCallObservation(call: CodeWhaleDecodedCall, turnIndex: number, privacyKey: string): CallObservation {
|
||||
const obs: CallObservation = {
|
||||
provider: call.provider,
|
||||
model: call.model,
|
||||
tokens: {
|
||||
input: call.inputTokens,
|
||||
output: call.outputTokens,
|
||||
reasoning: call.reasoningTokens,
|
||||
cacheRead: call.cacheReadInputTokens,
|
||||
cacheCreate: call.cacheCreationInputTokens,
|
||||
},
|
||||
webSearchRequests: call.webSearchRequests,
|
||||
speed: call.speed,
|
||||
costBasis: call.measuredCostUSD !== undefined ? 'measured' : 'estimated',
|
||||
timestamp: call.timestamp,
|
||||
dedupKey: call.deduplicationKey,
|
||||
toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)),
|
||||
turnIndex,
|
||||
...extractResourceRefs(privacyKey, call.toolSequence),
|
||||
}
|
||||
if (call.measuredCostUSD !== undefined) {
|
||||
;(obs as CallObservation & { measuredCostUSD: number }).measuredCostUSD = call.measuredCostUSD
|
||||
}
|
||||
return obs
|
||||
}
|
||||
|
||||
function toSessionObservation(
|
||||
decode: RichCodeWhaleSessionDecode,
|
||||
ctx: CodeWhaleToObservationsContext,
|
||||
): SessionObservation {
|
||||
const provider = ctx.provider ?? 'codewhale'
|
||||
const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i, ctx.privacyKey))
|
||||
|
||||
const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort()
|
||||
const startedAt = timestamps[0] ?? ''
|
||||
const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : ''
|
||||
|
||||
const session: SessionObservation = {
|
||||
sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId),
|
||||
projectRef: projectRef(ctx.privacyKey, decode.projectPath),
|
||||
providerId: provider,
|
||||
startedAt,
|
||||
...(endedAt ? { endedAt } : {}),
|
||||
calls,
|
||||
turnCount: calls.length,
|
||||
}
|
||||
return session
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a rich CodeWhale decode into the minimized observation layer. Returns the
|
||||
* `sessions` array plus any per-record `diagnostics`.
|
||||
*/
|
||||
export function toObservations(
|
||||
decode: RichCodeWhaleSessionDecode | RichCodeWhaleSessionDecode[],
|
||||
ctx: CodeWhaleToObservationsContext,
|
||||
): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } {
|
||||
const decodes = Array.isArray(decode) ? decode : [decode]
|
||||
const sessions = decodes.map(d => toSessionObservation(d, ctx))
|
||||
return { sessions, diagnostics: [] }
|
||||
}
|
||||
74
packages/core/src/providers/codewhale/types.ts
Normal file
74
packages/core/src/providers/codewhale/types.ts
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
// Raw record + rich-decode types for the CodeWhale provider.
|
||||
//
|
||||
// CodeWhale stores one whole JSON file per session. The host reads the file (and
|
||||
// falls back to a fast prefix read for oversized transcripts); the decoder is
|
||||
// pure over the parsed metadata + messages and carries no computed pricing.
|
||||
|
||||
export type CodeWhaleMetadata = {
|
||||
id: string
|
||||
created_at?: string
|
||||
updated_at?: string
|
||||
total_tokens?: number
|
||||
model?: string
|
||||
model_provider?: string
|
||||
workspace?: string
|
||||
cost?: {
|
||||
session_cost_usd?: number
|
||||
subagent_cost_usd?: number
|
||||
}
|
||||
}
|
||||
|
||||
export type CodeWhaleContentBlock = {
|
||||
type?: string
|
||||
text?: string
|
||||
name?: string
|
||||
input?: unknown
|
||||
}
|
||||
|
||||
export type CodeWhaleMessage = {
|
||||
role?: string
|
||||
content?: string | CodeWhaleContentBlock[]
|
||||
}
|
||||
|
||||
/** One tool invocation as captured from a CodeWhale transcript. */
|
||||
export type CodeWhaleToolCall = {
|
||||
tool: string
|
||||
file?: string
|
||||
command?: string
|
||||
}
|
||||
|
||||
/** The composite record the host hands to the core decoder for one session. */
|
||||
export type CodeWhaleSessionRecords = {
|
||||
metadata: CodeWhaleMetadata
|
||||
messages: CodeWhaleMessage[]
|
||||
/** File mtime as an ISO fallback when metadata timestamps are missing/invalid. */
|
||||
fileMtime: string
|
||||
}
|
||||
|
||||
/** The rich decode of one CodeWhale session, pre-pricing. */
|
||||
export type CodeWhaleDecodedCall = {
|
||||
provider: 'codewhale'
|
||||
model: string
|
||||
inputTokens: number
|
||||
outputTokens: number
|
||||
cacheCreationInputTokens: number
|
||||
cacheReadInputTokens: number
|
||||
cachedInputTokens: number
|
||||
reasoningTokens: number
|
||||
webSearchRequests: number
|
||||
/** Provider-reported dollar cost, present only when the file carried one. */
|
||||
measuredCostUSD?: number
|
||||
tools: string[]
|
||||
rawBashCommands: string[]
|
||||
skills: string[]
|
||||
subagentTypes: string[]
|
||||
toolSequence?: CodeWhaleToolCall[][]
|
||||
timestamp: string
|
||||
speed: 'standard'
|
||||
deduplicationKey: string
|
||||
turnId: string
|
||||
userMessage: string
|
||||
sessionId: string
|
||||
project: string
|
||||
projectPath: string
|
||||
}
|
||||
172
packages/core/src/providers/grok/decode.ts
Normal file
172
packages/core/src/providers/grok/decode.ts
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
// @codeburn/core Grok decoder: pure decode over a host-supplied composite record.
|
||||
// The host reads summary.json, signals.json, and the updates.jsonl lines; this
|
||||
// decoder reconstructs the per-session token buckets and tool list with no fs,
|
||||
// env, clock, or pricing.
|
||||
|
||||
import type { DecodeContext } from '../../contracts.js'
|
||||
import type { RecordDiagnostic } from '../../diagnostics.js'
|
||||
import type { GrokDecodedCall, GrokSessionRecords, GrokSignals, GrokSummary, GrokUpdate } from './types.js'
|
||||
|
||||
// Grok Build tool ids mapped to the canonical vocabulary. Unknown ids pass
|
||||
// through unchanged so provider-native tools still appear.
|
||||
export const grokToolNameMap: Record<string, string> = {
|
||||
bash: 'Bash',
|
||||
run_terminal_command: 'Bash',
|
||||
read_file: 'Read',
|
||||
read: 'Read',
|
||||
write_file: 'Write',
|
||||
edit_file: 'Edit',
|
||||
edit: 'Edit',
|
||||
list_dir: 'Glob',
|
||||
glob: 'Glob',
|
||||
grep: 'Grep',
|
||||
search: 'WebSearch',
|
||||
web_search: 'WebSearch',
|
||||
fetch: 'WebFetch',
|
||||
task: 'Agent',
|
||||
search_replace: 'Edit',
|
||||
todo_write: 'TodoWrite',
|
||||
spawn_subagent: 'Agent',
|
||||
}
|
||||
|
||||
function isGrokSessionRecords(value: unknown): value is GrokSessionRecords {
|
||||
return (
|
||||
value !== null &&
|
||||
typeof value === 'object' &&
|
||||
'updatesLines' in (value as object) &&
|
||||
'summary' in (value as object)
|
||||
)
|
||||
}
|
||||
|
||||
// Single pass over updates.jsonl: per-turn totalTokens for the cost estimate,
|
||||
// plus the real tool calls. Bash commands are kept raw; base-name extraction is
|
||||
// the host's job.
|
||||
function parseUpdates(updates: string): {
|
||||
input: number
|
||||
cacheRead: number
|
||||
output: number
|
||||
tools: string[]
|
||||
rawBashCommands: string[]
|
||||
subagentTypes: string[]
|
||||
} {
|
||||
const turns = new Map<string, { first: number; last: number }>()
|
||||
const tools: string[] = []
|
||||
const rawBashCommands: string[] = []
|
||||
const subagentTypes: string[] = []
|
||||
|
||||
let prevTotal = -1
|
||||
let segmentPeak = 0
|
||||
let inputFresh = 0
|
||||
|
||||
for (const line of updates.split('\n')) {
|
||||
if (!line.trim()) continue
|
||||
let params: GrokUpdate['params']
|
||||
try {
|
||||
params = (JSON.parse(line) as GrokUpdate).params
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
if (!params) continue
|
||||
|
||||
const total = params._meta?.totalTokens
|
||||
if (typeof total === 'number') {
|
||||
if (prevTotal >= 0 && total < prevTotal * 0.5) {
|
||||
inputFresh += segmentPeak
|
||||
segmentPeak = 0
|
||||
}
|
||||
if (total > segmentPeak) segmentPeak = total
|
||||
prevTotal = total
|
||||
|
||||
const promptId = params._meta?.promptId
|
||||
if (promptId) {
|
||||
const turn = turns.get(promptId)
|
||||
if (!turn) turns.set(promptId, { first: total, last: total })
|
||||
else turn.last = total
|
||||
}
|
||||
}
|
||||
|
||||
const update = params.update
|
||||
if (update?.sessionUpdate === 'tool_call' && typeof update.title === 'string') {
|
||||
tools.push(grokToolNameMap[update.title] ?? update.title)
|
||||
if (update.title === 'run_terminal_command' && typeof update.rawInput?.command === 'string') {
|
||||
rawBashCommands.push(update.rawInput.command)
|
||||
}
|
||||
if (update.title === 'spawn_subagent' && typeof update.rawInput?.subagent_type === 'string') {
|
||||
subagentTypes.push(update.rawInput.subagent_type)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inputFresh += segmentPeak
|
||||
let sumFirst = 0
|
||||
let output = 0
|
||||
for (const { first, last } of turns.values()) {
|
||||
sumFirst += first
|
||||
output += Math.max(0, last - first)
|
||||
}
|
||||
const cacheRead = Math.max(0, sumFirst - inputFresh)
|
||||
return { input: inputFresh, cacheRead, output, tools, rawBashCommands, subagentTypes }
|
||||
}
|
||||
|
||||
export type GrokDecodeInput = {
|
||||
records: unknown[]
|
||||
context: DecodeContext
|
||||
seenKeys?: Set<string>
|
||||
}
|
||||
|
||||
export type GrokDecodeResult = {
|
||||
calls: GrokDecodedCall[]
|
||||
diagnostics: RecordDiagnostic[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode one Grok session's composite records into a single rich, cost-free call.
|
||||
* The host owns file I/O and the live cross-file dedup set; this function is
|
||||
* pure over the supplied record.
|
||||
*/
|
||||
export function decodeGrok({ records, seenKeys: liveSeen }: GrokDecodeInput): GrokDecodeResult {
|
||||
const seen = liveSeen ?? new Set<string>()
|
||||
const session = records.find(isGrokSessionRecords)
|
||||
if (!session) return { calls: [], diagnostics: [] }
|
||||
|
||||
const { summary, signals, updatesLines, sourceDir, sessionName, project } = session
|
||||
const updates = updatesLines.join('\n')
|
||||
const { input, cacheRead, output, tools, rawBashCommands, subagentTypes } = parseUpdates(updates)
|
||||
if (input === 0 && output === 0) return { calls: [], diagnostics: [] }
|
||||
|
||||
const model =
|
||||
summary.current_model_id ??
|
||||
signals?.primaryModelId ??
|
||||
signals?.modelsUsed?.[0] ??
|
||||
'grok-build'
|
||||
const timestamp = summary.updated_at ?? summary.last_active_at ?? summary.created_at ?? ''
|
||||
const sessionId = summary.info?.id ?? sessionName
|
||||
|
||||
const dedupKey = `grok:${sourceDir}:${timestamp}:${sessionId}`
|
||||
if (seen.has(dedupKey)) return { calls: [], diagnostics: [] }
|
||||
seen.add(dedupKey)
|
||||
|
||||
const call: GrokDecodedCall = {
|
||||
provider: 'grok',
|
||||
model,
|
||||
inputTokens: input,
|
||||
outputTokens: output,
|
||||
cacheCreationInputTokens: 0,
|
||||
cacheReadInputTokens: cacheRead,
|
||||
cachedInputTokens: cacheRead,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
tools,
|
||||
rawBashCommands,
|
||||
subagentTypes,
|
||||
timestamp,
|
||||
speed: 'standard',
|
||||
deduplicationKey: dedupKey,
|
||||
userMessage: summary.session_summary ?? summary.generated_title ?? '',
|
||||
sessionId,
|
||||
projectPath: summary.info?.cwd ?? '',
|
||||
project,
|
||||
}
|
||||
|
||||
return { calls: [call], diagnostics: [] }
|
||||
}
|
||||
29
packages/core/src/providers/grok/index.ts
Normal file
29
packages/core/src/providers/grok/index.ts
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
// @codeburn/core Grok provider.
|
||||
//
|
||||
// Two layers:
|
||||
// - Rich pure decode (`decodeGrok`): host-facing, NOT part of the stable
|
||||
// minimized surface. Pure over the host-supplied composite record; carries
|
||||
// content in-memory but no pricing and no bash base-name extraction.
|
||||
// - Minimizing transform (`toObservations`): maps the rich decode into the
|
||||
// strict observation envelope; the content-smuggling guarantees bind here.
|
||||
|
||||
export {
|
||||
decodeGrok,
|
||||
grokToolNameMap,
|
||||
type GrokDecodeInput,
|
||||
type GrokDecodeResult,
|
||||
} from './decode.js'
|
||||
|
||||
export {
|
||||
toObservations,
|
||||
type RichGrokSessionDecode,
|
||||
type GrokToObservationsContext,
|
||||
} from './observations.js'
|
||||
|
||||
export type {
|
||||
GrokDecodedCall,
|
||||
GrokSessionRecords,
|
||||
GrokSummary,
|
||||
GrokSignals,
|
||||
GrokUpdate,
|
||||
} from './types.js'
|
||||
82
packages/core/src/providers/grok/observations.ts
Normal file
82
packages/core/src/providers/grok/observations.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
// Minimizing transform: rich Grok decode -> the strict observation envelope.
|
||||
//
|
||||
// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool
|
||||
// names cross into the output. The project path is fingerprinted; the user
|
||||
// message and raw bash commands stay behind.
|
||||
|
||||
import { projectRef, sessionRef } from '../../fingerprint.js'
|
||||
import type { RecordDiagnostic } from '../../diagnostics.js'
|
||||
import type { CallObservation, SessionObservation } from '../../observations.js'
|
||||
import type { GrokDecodedCall } from './types.js'
|
||||
|
||||
/** One Grok session's rich decode, as the host holds it before minimization. */
|
||||
export interface RichGrokSessionDecode {
|
||||
sessionId: string
|
||||
/** Absolute project path (the session cwd); fingerprinted, never emitted raw. */
|
||||
projectPath: string
|
||||
/** Rich, cost-free calls in decode order (Grok emits one per session). */
|
||||
calls: GrokDecodedCall[]
|
||||
}
|
||||
|
||||
export interface GrokToObservationsContext {
|
||||
/** HMAC key that scopes every fingerprint. */
|
||||
privacyKey: string
|
||||
/** Provider id stamped onto sessions/calls and folded into sessionRef. */
|
||||
provider?: string
|
||||
}
|
||||
|
||||
const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/
|
||||
|
||||
function toCallObservation(call: GrokDecodedCall, turnIndex: number, privacyKey: string): CallObservation {
|
||||
return {
|
||||
provider: call.provider,
|
||||
model: call.model,
|
||||
tokens: {
|
||||
input: call.inputTokens,
|
||||
output: call.outputTokens,
|
||||
reasoning: call.reasoningTokens,
|
||||
cacheRead: call.cacheReadInputTokens,
|
||||
cacheCreate: call.cacheCreationInputTokens,
|
||||
},
|
||||
webSearchRequests: call.webSearchRequests,
|
||||
speed: call.speed,
|
||||
costBasis: 'estimated',
|
||||
timestamp: call.timestamp,
|
||||
dedupKey: call.deduplicationKey,
|
||||
toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)),
|
||||
turnIndex,
|
||||
}
|
||||
}
|
||||
|
||||
function toSessionObservation(decode: RichGrokSessionDecode, ctx: GrokToObservationsContext): SessionObservation {
|
||||
const provider = ctx.provider ?? 'grok'
|
||||
const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i, ctx.privacyKey))
|
||||
|
||||
const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort()
|
||||
const startedAt = timestamps[0] ?? ''
|
||||
const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : ''
|
||||
|
||||
const session: SessionObservation = {
|
||||
sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId),
|
||||
projectRef: projectRef(ctx.privacyKey, decode.projectPath),
|
||||
providerId: provider,
|
||||
startedAt,
|
||||
...(endedAt ? { endedAt } : {}),
|
||||
calls,
|
||||
turnCount: calls.length,
|
||||
}
|
||||
return session
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a rich Grok decode into the minimized observation layer. Returns the
|
||||
* `sessions` array plus any per-record `diagnostics`.
|
||||
*/
|
||||
export function toObservations(
|
||||
decode: RichGrokSessionDecode | RichGrokSessionDecode[],
|
||||
ctx: GrokToObservationsContext,
|
||||
): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } {
|
||||
const decodes = Array.isArray(decode) ? decode : [decode]
|
||||
const sessions = decodes.map(d => toSessionObservation(d, ctx))
|
||||
return { sessions, diagnostics: [] }
|
||||
}
|
||||
71
packages/core/src/providers/grok/types.ts
Normal file
71
packages/core/src/providers/grok/types.ts
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
// Raw record + rich-decode types for the Grok Build provider.
|
||||
//
|
||||
// Grok stores one session per directory: <grok-home>/sessions/<url-encoded-cwd>/<uuid>/
|
||||
// with summary.json, signals.json, and updates.jsonl. The host reads those files
|
||||
// and hands the decoder a single composite record; the decoder is pure over that
|
||||
// record and carries no pricing.
|
||||
|
||||
export type GrokSummary = {
|
||||
info?: { id?: string; cwd?: string }
|
||||
created_at?: string
|
||||
updated_at?: string
|
||||
last_active_at?: string
|
||||
current_model_id?: string
|
||||
session_summary?: string
|
||||
generated_title?: string
|
||||
}
|
||||
|
||||
export type GrokSignals = {
|
||||
primaryModelId?: string
|
||||
modelsUsed?: string[]
|
||||
toolsUsed?: string[]
|
||||
}
|
||||
|
||||
export type GrokUpdate = {
|
||||
params?: {
|
||||
_meta?: { totalTokens?: number; promptId?: string }
|
||||
update?: {
|
||||
sessionUpdate?: string
|
||||
title?: string
|
||||
rawInput?: { command?: unknown; subagent_type?: unknown }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The composite record the host hands to the core decoder for one session. */
|
||||
export type GrokSessionRecords = {
|
||||
summary: GrokSummary
|
||||
signals: GrokSignals | null
|
||||
updatesLines: string[]
|
||||
/** Absolute session directory, used only for the host-side dedup key. */
|
||||
sourceDir: string
|
||||
/** Basename of the session directory, used as a session id fallback. */
|
||||
sessionName: string
|
||||
/** Project name the host derived from the session cwd. */
|
||||
project: string
|
||||
}
|
||||
|
||||
/** The rich decode of one Grok session, pre-pricing. */
|
||||
export type GrokDecodedCall = {
|
||||
provider: 'grok'
|
||||
model: string
|
||||
inputTokens: number
|
||||
outputTokens: number
|
||||
cacheCreationInputTokens: number
|
||||
cacheReadInputTokens: number
|
||||
cachedInputTokens: number
|
||||
reasoningTokens: number
|
||||
webSearchRequests: number
|
||||
tools: string[]
|
||||
rawBashCommands: string[]
|
||||
subagentTypes: string[]
|
||||
timestamp: string
|
||||
speed: 'standard'
|
||||
deduplicationKey: string
|
||||
userMessage: string
|
||||
sessionId: string
|
||||
/** Absolute project cwd; fingerprinted before it reaches an observation. */
|
||||
projectPath: string
|
||||
/** Project name the host uses for display/breakdown. */
|
||||
project: string
|
||||
}
|
||||
241
packages/core/src/providers/kimi/decode.ts
Normal file
241
packages/core/src/providers/kimi/decode.ts
Normal file
|
|
@ -0,0 +1,241 @@
|
|||
// @codeburn/core Kimi decoder: pure decode over host-supplied JSONL lines.
|
||||
// The host reads the wire file and the configured model fallback; this decoder
|
||||
// is stateless, does no fs/env/clock access, and carries no pricing.
|
||||
|
||||
import type { DecodeContext } from '../../contracts.js'
|
||||
import type { RecordDiagnostic } from '../../diagnostics.js'
|
||||
import type { KimiDecodedCall, KimiSessionRecords, JsonObject } from './types.js'
|
||||
|
||||
export const kimiToolNameMap: Record<string, string> = {
|
||||
Shell: 'Bash',
|
||||
Bash: 'Bash',
|
||||
bash: 'Bash',
|
||||
ReadFile: 'Read',
|
||||
ReadMediaFile: 'Read',
|
||||
WriteFile: 'Write',
|
||||
StrReplaceFile: 'Edit',
|
||||
Grep: 'Grep',
|
||||
Glob: 'Glob',
|
||||
SearchWeb: 'WebSearch',
|
||||
FetchURL: 'WebFetch',
|
||||
Agent: 'Agent',
|
||||
AgentTool: 'Agent',
|
||||
TaskList: 'Agent',
|
||||
TaskOutput: 'Agent',
|
||||
TaskStop: 'Agent',
|
||||
AskUserQuestion: 'AskUser',
|
||||
SetTodoList: 'TodoWrite',
|
||||
Think: 'Think',
|
||||
EnterPlanMode: 'EnterPlanMode',
|
||||
ExitPlanMode: 'ExitPlanMode',
|
||||
SendDMail: 'DMail',
|
||||
}
|
||||
|
||||
function asObject(value: unknown): JsonObject | null {
|
||||
return value && typeof value === 'object' && !Array.isArray(value) ? (value as JsonObject) : null
|
||||
}
|
||||
|
||||
function stringField(obj: JsonObject | null, key: string): string | undefined {
|
||||
const value = obj?.[key]
|
||||
return typeof value === 'string' ? value : undefined
|
||||
}
|
||||
|
||||
function numericField(obj: JsonObject, ...keys: string[]): number {
|
||||
for (const key of keys) {
|
||||
const raw = obj[key]
|
||||
const n = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number(raw) : NaN
|
||||
if (Number.isFinite(n) && n > 0) return Math.trunc(n)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
function timestampToIso(value: unknown): string {
|
||||
if (typeof value === 'string') return value
|
||||
if (typeof value !== 'number' || !Number.isFinite(value)) return ''
|
||||
|
||||
const millis = value > 1_000_000_000_000 ? value : value * 1000
|
||||
const date = new Date(millis)
|
||||
return Number.isFinite(date.getTime()) ? date.toISOString() : ''
|
||||
}
|
||||
|
||||
function extractUserText(value: unknown): string {
|
||||
if (typeof value === 'string') return value.slice(0, 500)
|
||||
if (!Array.isArray(value)) return ''
|
||||
|
||||
return value
|
||||
.map(part => stringField(asObject(part), 'text') ?? '')
|
||||
.filter(Boolean)
|
||||
.join(' ')
|
||||
.slice(0, 500)
|
||||
}
|
||||
|
||||
function extractEnvelope(record: JsonObject): { type: string; payload: JsonObject; timestamp: string } | null {
|
||||
const message = asObject(record['message'])
|
||||
const envelope = message ?? record
|
||||
const type = stringField(envelope, 'type')
|
||||
const payload = asObject(envelope['payload'])
|
||||
if (!type || !payload) return null
|
||||
return { type, payload, timestamp: timestampToIso(record['timestamp']) }
|
||||
}
|
||||
|
||||
function extractUsage(payload: JsonObject): {
|
||||
inputTokens: number
|
||||
outputTokens: number
|
||||
cacheReadInputTokens: number
|
||||
cacheCreationInputTokens: number
|
||||
} | null {
|
||||
const usage = asObject(payload['token_usage']) ?? asObject(payload['usage'])
|
||||
if (!usage) return null
|
||||
|
||||
const cacheReadInputTokens = numericField(usage, 'input_cache_read', 'cache_read_input_tokens', 'cached_input_tokens')
|
||||
const cacheCreationInputTokens = numericField(usage, 'input_cache_creation', 'cache_creation_input_tokens')
|
||||
let inputTokens = numericField(usage, 'input_other', 'input_tokens')
|
||||
if (inputTokens === 0) {
|
||||
const totalInput = numericField(usage, 'input')
|
||||
inputTokens = Math.max(0, totalInput - cacheReadInputTokens - cacheCreationInputTokens)
|
||||
}
|
||||
const outputTokens = numericField(usage, 'output', 'output_tokens')
|
||||
|
||||
if (
|
||||
inputTokens === 0 &&
|
||||
outputTokens === 0 &&
|
||||
cacheReadInputTokens === 0 &&
|
||||
cacheCreationInputTokens === 0
|
||||
) {
|
||||
return null
|
||||
}
|
||||
|
||||
return { inputTokens, outputTokens, cacheReadInputTokens, cacheCreationInputTokens }
|
||||
}
|
||||
|
||||
function extractTool(payload: JsonObject): { tool: string; rawBashCommand?: string } | null {
|
||||
const fn = asObject(payload['function'])
|
||||
const rawName = stringField(fn, 'name') ?? stringField(payload, 'name')
|
||||
if (!rawName) return null
|
||||
|
||||
const tool = kimiToolNameMap[rawName] ?? rawName
|
||||
const argsText = stringField(fn, 'arguments') ?? stringField(payload, 'arguments')
|
||||
let args: JsonObject | null = null
|
||||
if (argsText) {
|
||||
try {
|
||||
args = asObject(JSON.parse(argsText))
|
||||
} catch {
|
||||
args = null
|
||||
}
|
||||
}
|
||||
const command = stringField(args, 'command')
|
||||
|
||||
return { tool, ...(tool === 'Bash' && command ? { rawBashCommand: command } : {}) }
|
||||
}
|
||||
|
||||
function isKimiSessionRecords(value: unknown): value is KimiSessionRecords {
|
||||
return value !== null && typeof value === 'object' && 'lines' in (value as object)
|
||||
}
|
||||
|
||||
export type KimiDecodeInput = {
|
||||
records: unknown[]
|
||||
context: DecodeContext
|
||||
seenKeys?: Set<string>
|
||||
}
|
||||
|
||||
export type KimiDecodeResult = {
|
||||
calls: KimiDecodedCall[]
|
||||
diagnostics: RecordDiagnostic[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a Kimi wire log into rich, cost-free calls. A single pass:
|
||||
* TurnBegin/SteerInput set the pending user message; ToolCall records collect
|
||||
tools; StatusUpdate records with usage flush a call. Dedup is keyed on
|
||||
* `kimi:<sessionId>:<messageId>` against the live `seenKeys` set.
|
||||
*/
|
||||
export function decodeKimi({ records, seenKeys: liveSeen }: KimiDecodeInput): KimiDecodeResult {
|
||||
const seen = liveSeen ?? new Set<string>()
|
||||
const session = records.find(isKimiSessionRecords)
|
||||
if (!session) return { calls: [], diagnostics: [] }
|
||||
|
||||
const { lines, configuredModel, sessionName } = session
|
||||
const calls: KimiDecodedCall[] = []
|
||||
const tools: string[] = []
|
||||
const rawBashCommands: string[] = []
|
||||
let currentUserMessage = ''
|
||||
let index = 0
|
||||
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue
|
||||
|
||||
let record: JsonObject | null = null
|
||||
try {
|
||||
record = asObject(JSON.parse(line))
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
if (!record) continue
|
||||
|
||||
const envelope = extractEnvelope(record)
|
||||
if (!envelope || envelope.type === 'metadata') continue
|
||||
|
||||
if (envelope.type === 'TurnBegin' || envelope.type === 'SteerInput') {
|
||||
currentUserMessage = extractUserText(envelope.payload['user_input'])
|
||||
continue
|
||||
}
|
||||
|
||||
if (envelope.type === 'TurnEnd') {
|
||||
currentUserMessage = ''
|
||||
tools.length = 0
|
||||
rawBashCommands.length = 0
|
||||
continue
|
||||
}
|
||||
|
||||
if (envelope.type === 'ToolCall' || envelope.type === 'ToolCallRequest') {
|
||||
const extracted = extractTool(envelope.payload)
|
||||
if (!extracted) continue
|
||||
tools.push(extracted.tool)
|
||||
if (extracted.rawBashCommand) rawBashCommands.push(extracted.rawBashCommand)
|
||||
continue
|
||||
}
|
||||
|
||||
if (envelope.type !== 'StatusUpdate') continue
|
||||
|
||||
const usage = extractUsage(envelope.payload)
|
||||
if (!usage) continue
|
||||
|
||||
const rawMessageId = stringField(envelope.payload, 'message_id')
|
||||
const dedupKey = `kimi:${sessionName}:${rawMessageId ?? index}`
|
||||
index++
|
||||
if (seen.has(dedupKey)) continue
|
||||
seen.add(dedupKey)
|
||||
|
||||
const model =
|
||||
stringField(envelope.payload, 'model') ??
|
||||
stringField(envelope.payload, 'model_name') ??
|
||||
configuredModel
|
||||
|
||||
calls.push({
|
||||
provider: 'kimi',
|
||||
model,
|
||||
inputTokens: usage.inputTokens,
|
||||
outputTokens: usage.outputTokens,
|
||||
cacheCreationInputTokens: usage.cacheCreationInputTokens,
|
||||
cacheReadInputTokens: usage.cacheReadInputTokens,
|
||||
cachedInputTokens: usage.cacheReadInputTokens,
|
||||
reasoningTokens: 0,
|
||||
webSearchRequests: 0,
|
||||
// The legacy kimi decode accumulated tools in a Set, deduping per turn.
|
||||
// Preserve that here so the observation/host tool list matches byte-for-byte.
|
||||
tools: [...new Set(tools)],
|
||||
rawBashCommands: [...rawBashCommands],
|
||||
timestamp: envelope.timestamp,
|
||||
speed: 'standard',
|
||||
deduplicationKey: dedupKey,
|
||||
userMessage: currentUserMessage,
|
||||
sessionId: sessionName,
|
||||
projectPath: '',
|
||||
})
|
||||
|
||||
tools.length = 0
|
||||
rawBashCommands.length = 0
|
||||
}
|
||||
|
||||
return { calls, diagnostics: [] }
|
||||
}
|
||||
26
packages/core/src/providers/kimi/index.ts
Normal file
26
packages/core/src/providers/kimi/index.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
// @codeburn/core Kimi provider.
|
||||
//
|
||||
// Two layers:
|
||||
// - Rich pure decode (`decodeKimi`): host-facing, NOT part of the stable
|
||||
// minimized surface. Pure over the host-supplied JSONL lines; carries
|
||||
// content in-memory but no pricing and no bash base-name extraction.
|
||||
// - Minimizing transform (`toObservations`): maps the rich decode into the
|
||||
// strict observation envelope; the content-smuggling guarantees bind here.
|
||||
|
||||
export {
|
||||
decodeKimi,
|
||||
kimiToolNameMap,
|
||||
type KimiDecodeInput,
|
||||
type KimiDecodeResult,
|
||||
} from './decode.js'
|
||||
|
||||
export {
|
||||
toObservations,
|
||||
type RichKimiSessionDecode,
|
||||
type KimiToObservationsContext,
|
||||
} from './observations.js'
|
||||
|
||||
export type {
|
||||
KimiDecodedCall,
|
||||
KimiSessionRecords,
|
||||
} from './types.js'
|
||||
82
packages/core/src/providers/kimi/observations.ts
Normal file
82
packages/core/src/providers/kimi/observations.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
// Minimizing transform: rich Kimi decode -> the strict observation envelope.
|
||||
//
|
||||
// Only opaque ids, fingerprints, enums, numbers, timestamps, and canonical tool
|
||||
// names cross into the output. The user message and raw bash commands stay
|
||||
// behind.
|
||||
|
||||
import { projectRef, sessionRef } from '../../fingerprint.js'
|
||||
import type { RecordDiagnostic } from '../../diagnostics.js'
|
||||
import type { CallObservation, SessionObservation } from '../../observations.js'
|
||||
import type { KimiDecodedCall } from './types.js'
|
||||
|
||||
/** One Kimi session's rich decode, as the host holds it before minimization. */
|
||||
export interface RichKimiSessionDecode {
|
||||
sessionId: string
|
||||
/** Kimi does not record a project path; an empty path is fingerprinted. */
|
||||
projectPath: string
|
||||
/** Rich, cost-free calls in decode order. */
|
||||
calls: KimiDecodedCall[]
|
||||
}
|
||||
|
||||
export interface KimiToObservationsContext {
|
||||
/** HMAC key that scopes every fingerprint. */
|
||||
privacyKey: string
|
||||
/** Provider id stamped onto sessions/calls and folded into sessionRef. */
|
||||
provider?: string
|
||||
}
|
||||
|
||||
const CANONICAL_TOOL_NAME = /^[A-Za-z0-9_.-]{1,64}$/
|
||||
|
||||
function toCallObservation(call: KimiDecodedCall, turnIndex: number, privacyKey: string): CallObservation {
|
||||
return {
|
||||
provider: call.provider,
|
||||
model: call.model,
|
||||
tokens: {
|
||||
input: call.inputTokens,
|
||||
output: call.outputTokens,
|
||||
reasoning: call.reasoningTokens,
|
||||
cacheRead: call.cacheReadInputTokens,
|
||||
cacheCreate: call.cacheCreationInputTokens,
|
||||
},
|
||||
webSearchRequests: call.webSearchRequests,
|
||||
speed: call.speed,
|
||||
costBasis: 'estimated',
|
||||
timestamp: call.timestamp,
|
||||
dedupKey: call.deduplicationKey,
|
||||
toolNames: call.tools.filter(t => CANONICAL_TOOL_NAME.test(t)),
|
||||
turnIndex,
|
||||
}
|
||||
}
|
||||
|
||||
function toSessionObservation(decode: RichKimiSessionDecode, ctx: KimiToObservationsContext): SessionObservation {
|
||||
const provider = ctx.provider ?? 'kimi'
|
||||
const calls: CallObservation[] = decode.calls.map((call, i) => toCallObservation(call, i, ctx.privacyKey))
|
||||
|
||||
const timestamps = calls.map(c => c.timestamp).filter(t => t.length > 0).sort()
|
||||
const startedAt = timestamps[0] ?? ''
|
||||
const endedAt = timestamps.length > 0 ? timestamps[timestamps.length - 1]! : ''
|
||||
|
||||
const session: SessionObservation = {
|
||||
sessionRef: sessionRef(ctx.privacyKey, provider, decode.sessionId),
|
||||
projectRef: projectRef(ctx.privacyKey, decode.projectPath),
|
||||
providerId: provider,
|
||||
startedAt,
|
||||
...(endedAt ? { endedAt } : {}),
|
||||
calls,
|
||||
turnCount: calls.length,
|
||||
}
|
||||
return session
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a rich Kimi decode into the minimized observation layer. Returns the
|
||||
* `sessions` array plus any per-record `diagnostics`.
|
||||
*/
|
||||
export function toObservations(
|
||||
decode: RichKimiSessionDecode | RichKimiSessionDecode[],
|
||||
ctx: KimiToObservationsContext,
|
||||
): { sessions: SessionObservation[]; diagnostics: RecordDiagnostic[] } {
|
||||
const decodes = Array.isArray(decode) ? decode : [decode]
|
||||
const sessions = decodes.map(d => toSessionObservation(d, ctx))
|
||||
return { sessions, diagnostics: [] }
|
||||
}
|
||||
38
packages/core/src/providers/kimi/types.ts
Normal file
38
packages/core/src/providers/kimi/types.ts
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
// Raw record + rich-decode types for the Kimi provider.
|
||||
//
|
||||
// Kimi writes one JSONL wire log per session. The host reads the log lines and
|
||||
// the configured model fallback from config.toml; the decoder is pure over that
|
||||
// composite record and carries no pricing.
|
||||
|
||||
export type JsonObject = Record<string, unknown>
|
||||
|
||||
export type KimiSessionRecords = {
|
||||
/** Raw JSONL lines from the wire log. */
|
||||
lines: string[]
|
||||
/** Model string to fall back to when no model is named on a usage record. */
|
||||
configuredModel: string
|
||||
/** Basename of the session directory, used as the session id. */
|
||||
sessionName: string
|
||||
}
|
||||
|
||||
/** The rich decode of one Kimi call, pre-pricing. */
|
||||
export type KimiDecodedCall = {
|
||||
provider: 'kimi'
|
||||
model: string
|
||||
inputTokens: number
|
||||
outputTokens: number
|
||||
cacheCreationInputTokens: number
|
||||
cacheReadInputTokens: number
|
||||
cachedInputTokens: number
|
||||
reasoningTokens: number
|
||||
webSearchRequests: number
|
||||
tools: string[]
|
||||
rawBashCommands: string[]
|
||||
timestamp: string
|
||||
speed: 'standard'
|
||||
deduplicationKey: string
|
||||
userMessage: string
|
||||
sessionId: string
|
||||
/** Kimi does not record a project path; observations fingerprint an empty path. */
|
||||
projectPath: string
|
||||
}
|
||||
|
|
@ -118,8 +118,14 @@ const CORRECTION_PHRASES = [
|
|||
const USER_MESSAGE_ALLOWLIST = new Set([
|
||||
'src/providers/claude/decode.ts',
|
||||
'src/providers/claude/types.ts',
|
||||
'src/providers/codewhale/decode.ts',
|
||||
'src/providers/codewhale/types.ts',
|
||||
'src/providers/codex/decode.ts',
|
||||
'src/providers/codex/types.ts',
|
||||
'src/providers/grok/decode.ts',
|
||||
'src/providers/grok/types.ts',
|
||||
'src/providers/kimi/decode.ts',
|
||||
'src/providers/kimi/types.ts',
|
||||
'src/providers/qwen/decode.ts',
|
||||
'src/providers/qwen/types.ts',
|
||||
])
|
||||
|
|
|
|||
|
|
@ -20,6 +20,9 @@ import {
|
|||
import type { JournalEntry, ToolResultMeta } from '../src/providers/claude/index.js'
|
||||
import { decodeCodex, toObservations as toCodexObservations } from '../src/providers/codex/index.js'
|
||||
import { decodeQwen, toObservations as toQwenObservations } from '../src/providers/qwen/index.js'
|
||||
import { decodeGrok, toObservations as toGrokObservations } from '../src/providers/grok/index.js'
|
||||
import { decodeKimi, toObservations as toKimiObservations } from '../src/providers/kimi/index.js'
|
||||
import { decodeCodeWhale, toObservations as toCodeWhaleObservations } from '../src/providers/codewhale/index.js'
|
||||
import type { DecodeContext } from '../src/contracts.js'
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url))
|
||||
|
|
@ -357,6 +360,223 @@ describe('content-smuggling guardrail: real qwen decode -> toObservations is sec
|
|||
})
|
||||
})
|
||||
|
||||
describe('content-smuggling guardrail: real grok decode -> toObservations is secret-free', () => {
|
||||
// A hostile Grok session planting every secret in the free-text fields the
|
||||
// decode captures: the project path, the user message (session summary/title),
|
||||
// a bash command, and a subagent type. Plus a tool NAME carrying a command
|
||||
// line, which must be dropped by the canonical-name filter.
|
||||
const grokContext: DecodeContext = { privacyKey: 'test-privacy-key', providerId: 'grok', sourceRef: 'ref' }
|
||||
|
||||
function decodeAndMinimize() {
|
||||
const records = [
|
||||
{
|
||||
summary: {
|
||||
info: { id: 'sess-hostile', cwd: SECRETS.absPath },
|
||||
created_at: '2026-07-17T10:00:00.000Z',
|
||||
updated_at: '2026-07-17T10:00:05.000Z',
|
||||
session_summary: `${SECRETS.prompt} ${SECRETS.apiKey} ${SECRETS.fileContent}`,
|
||||
generated_title: SECRETS.prompt,
|
||||
},
|
||||
signals: null,
|
||||
updatesLines: [
|
||||
JSON.stringify({
|
||||
timestamp: '2026-07-17T10:00:05.000Z',
|
||||
method: 'session/update',
|
||||
params: {
|
||||
sessionId: 'sess-hostile',
|
||||
update: { sessionUpdate: 'tool_call', title: SECRETS.commandLine, rawInput: {} },
|
||||
},
|
||||
}),
|
||||
JSON.stringify({
|
||||
timestamp: '2026-07-17T10:00:05.000Z',
|
||||
method: 'session/update',
|
||||
params: {
|
||||
sessionId: 'sess-hostile',
|
||||
update: {
|
||||
sessionUpdate: 'tool_call',
|
||||
title: 'run_terminal_command',
|
||||
rawInput: { command: SECRETS.commandLine },
|
||||
},
|
||||
},
|
||||
}),
|
||||
JSON.stringify({
|
||||
timestamp: '2026-07-17T10:00:05.000Z',
|
||||
method: 'session/update',
|
||||
params: {
|
||||
sessionId: 'sess-hostile',
|
||||
update: {
|
||||
sessionUpdate: 'tool_call',
|
||||
title: 'spawn_subagent',
|
||||
rawInput: { subagent_type: SECRETS.fileContent },
|
||||
},
|
||||
_meta: { totalTokens: 1000, promptId: 'p1' },
|
||||
},
|
||||
}),
|
||||
],
|
||||
sourceDir: '/sessions/hostile',
|
||||
sessionName: 'sess-hostile',
|
||||
project: 'hostile-project',
|
||||
},
|
||||
]
|
||||
const { calls } = decodeGrok({ records, context: grokContext })
|
||||
const { sessions } = toGrokObservations(
|
||||
{ sessionId: 'sess-hostile', projectPath: SECRETS.absPath, calls },
|
||||
{ privacyKey: 'test-privacy-key', provider: 'grok' },
|
||||
)
|
||||
return {
|
||||
schemaVersion: OBSERVATION_SCHEMA_VERSION,
|
||||
generator: { name: '@codeburn/core', version: '0.0.0-test' },
|
||||
sessions,
|
||||
}
|
||||
}
|
||||
|
||||
it('produces a schema-valid envelope from the hostile session', () => {
|
||||
expect(ObservationEnvelope.safeParse(decodeAndMinimize()).success).toBe(true)
|
||||
})
|
||||
|
||||
it('the serialized envelope contains none of the planted secrets', () => {
|
||||
const serialized = JSON.stringify(decodeAndMinimize())
|
||||
for (const secret of ALL_SECRETS) {
|
||||
expect(serialized).not.toContain(secret)
|
||||
}
|
||||
})
|
||||
|
||||
it('drops non-canonical (argument-carrying) tool names instead of emitting them', () => {
|
||||
const env = decodeAndMinimize()
|
||||
const allToolNames = env.sessions.flatMap(s => s.calls.flatMap(c => c.toolNames))
|
||||
expect(allToolNames).toContain('Bash')
|
||||
expect(allToolNames).not.toContain(SECRETS.commandLine)
|
||||
})
|
||||
})
|
||||
|
||||
describe('content-smuggling guardrail: real kimi decode -> toObservations is secret-free', () => {
|
||||
// A hostile Kimi wire log planting every secret in the free-text fields the
|
||||
// decode captures: the user message, a Bash command, and a tool NAME carrying
|
||||
// a command line. Minimizing MUST surface none of them.
|
||||
const kimiContext: DecodeContext = { privacyKey: 'test-privacy-key', providerId: 'kimi', sourceRef: 'ref' }
|
||||
|
||||
function decodeAndMinimize() {
|
||||
const records = [
|
||||
{
|
||||
lines: [
|
||||
JSON.stringify({ timestamp: 1776162400, message: { type: 'TurnBegin', payload: { user_input: `${SECRETS.prompt} ${SECRETS.apiKey} ${SECRETS.fileContent}` } } }),
|
||||
JSON.stringify({ timestamp: 1776162401, message: { type: 'ToolCall', payload: { type: 'function', id: 'call-shell', function: { name: SECRETS.commandLine, arguments: '{}' } } } }),
|
||||
JSON.stringify({ timestamp: 1776162402, message: { type: 'ToolCall', payload: { type: 'function', id: 'call-bash', function: { name: 'Shell', arguments: JSON.stringify({ command: SECRETS.commandLine }) } } } }),
|
||||
JSON.stringify({ timestamp: 1776162403, message: { type: 'StatusUpdate', payload: { message_id: 'msg-hostile', token_usage: { input_other: 10, output: 5 } } } }),
|
||||
],
|
||||
configuredModel: 'kimi-auto',
|
||||
sessionName: 'sess-hostile',
|
||||
},
|
||||
]
|
||||
const { calls } = decodeKimi({ records, context: kimiContext })
|
||||
const { sessions } = toKimiObservations(
|
||||
{ sessionId: 'sess-hostile', projectPath: '', calls },
|
||||
{ privacyKey: 'test-privacy-key', provider: 'kimi' },
|
||||
)
|
||||
return {
|
||||
schemaVersion: OBSERVATION_SCHEMA_VERSION,
|
||||
generator: { name: '@codeburn/core', version: '0.0.0-test' },
|
||||
sessions,
|
||||
}
|
||||
}
|
||||
|
||||
it('produces a schema-valid envelope from the hostile wire log', () => {
|
||||
expect(ObservationEnvelope.safeParse(decodeAndMinimize()).success).toBe(true)
|
||||
})
|
||||
|
||||
it('the serialized envelope contains none of the planted secrets', () => {
|
||||
const serialized = JSON.stringify(decodeAndMinimize())
|
||||
for (const secret of ALL_SECRETS) {
|
||||
expect(serialized).not.toContain(secret)
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps canonical tool names (Bash) and drops the argument-carrying name', () => {
|
||||
const env = decodeAndMinimize()
|
||||
const allToolNames = env.sessions.flatMap(s => s.calls.flatMap(c => c.toolNames))
|
||||
expect(allToolNames).toContain('Bash')
|
||||
expect(allToolNames).not.toContain(SECRETS.commandLine)
|
||||
})
|
||||
})
|
||||
|
||||
describe('content-smuggling guardrail: real codewhale decode -> toObservations is secret-free', () => {
|
||||
// A hostile CodeWhale session planting every secret in the free-text fields
|
||||
// the decode captures: the project path, the user message, a Bash command, a
|
||||
// read/edit file path, a skill name, a subagent type, and a tool NAME carrying
|
||||
// a command line. Minimizing MUST surface none of them.
|
||||
const codeWhaleContext: DecodeContext = { privacyKey: 'test-privacy-key', providerId: 'codewhale', sourceRef: 'ref' }
|
||||
|
||||
function decodeAndMinimize() {
|
||||
const records = [
|
||||
{
|
||||
metadata: {
|
||||
id: 'sess-hostile',
|
||||
total_tokens: 1000,
|
||||
workspace: SECRETS.absPath,
|
||||
},
|
||||
messages: [
|
||||
{ role: 'user', content: `${SECRETS.prompt} ${SECRETS.apiKey} ${SECRETS.fileContent}` },
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{ type: 'tool_use', id: 't1', name: SECRETS.commandLine, input: {} },
|
||||
{ type: 'tool_use', id: 't2', name: 'exec_shell', input: { command: SECRETS.commandLine } },
|
||||
{ type: 'tool_use', id: 't3', name: 'read_file', input: { file_path: SECRETS.absPath } },
|
||||
{ type: 'tool_use', id: 't4', name: 'edit_file', input: { path: SECRETS.absPath } },
|
||||
{ type: 'tool_use', id: 't5', name: 'load_skill', input: { name: SECRETS.fileContent } },
|
||||
{ type: 'tool_use', id: 't6', name: 'agent', input: { type: SECRETS.prompt } },
|
||||
],
|
||||
},
|
||||
],
|
||||
fileMtime: '2026-07-17T10:00:00.000Z',
|
||||
},
|
||||
]
|
||||
const { calls } = decodeCodeWhale({ records, context: codeWhaleContext })
|
||||
const { sessions } = toCodeWhaleObservations(
|
||||
{ sessionId: 'sess-hostile', projectPath: SECRETS.absPath, calls },
|
||||
{ privacyKey: 'test-privacy-key', provider: 'codewhale' },
|
||||
)
|
||||
return {
|
||||
schemaVersion: OBSERVATION_SCHEMA_VERSION,
|
||||
generator: { name: '@codeburn/core', version: '0.0.0-test' },
|
||||
sessions,
|
||||
}
|
||||
}
|
||||
|
||||
it('produces a schema-valid envelope from the hostile session', () => {
|
||||
expect(ObservationEnvelope.safeParse(decodeAndMinimize()).success).toBe(true)
|
||||
})
|
||||
|
||||
it('the serialized envelope contains none of the planted secrets', () => {
|
||||
const serialized = JSON.stringify(decodeAndMinimize())
|
||||
for (const secret of ALL_SECRETS) {
|
||||
expect(serialized).not.toContain(secret)
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps canonical tool names (Bash/Read/Agent/Skill) and drops the argument-carrying name', () => {
|
||||
const env = decodeAndMinimize()
|
||||
const allToolNames = env.sessions.flatMap(s => s.calls.flatMap(c => c.toolNames))
|
||||
expect(allToolNames).toContain('Bash')
|
||||
expect(allToolNames).toContain('Read')
|
||||
expect(allToolNames).toContain('Agent')
|
||||
expect(allToolNames).toContain('Skill')
|
||||
expect(allToolNames).not.toContain(SECRETS.commandLine)
|
||||
})
|
||||
|
||||
it('fingerprints the read/edit paths into 16-hex resource refs, never the raw paths', () => {
|
||||
const env = decodeAndMinimize()
|
||||
const reads = env.sessions.flatMap(s => s.calls.flatMap(c => c.resourceReads ?? []))
|
||||
const edits = env.sessions.flatMap(s => s.calls.flatMap(c => c.resourceEdits ?? []))
|
||||
expect(reads.length).toBeGreaterThan(0)
|
||||
expect(edits.length).toBeGreaterThan(0)
|
||||
for (const ref of [...reads, ...edits]) {
|
||||
expect(ref.resourceId).toMatch(/^[0-9a-f]{16}$/)
|
||||
}
|
||||
expect(allStrings([...reads, ...edits])).not.toContain(SECRETS.absPath)
|
||||
})
|
||||
})
|
||||
|
||||
describe('content-smuggling guardrail: diagnostic detail rejects paths', () => {
|
||||
it('rejects an absolute path', () => {
|
||||
expect(DiagnosticDetail.safeParse(SECRETS.absPath).success).toBe(false)
|
||||
|
|
|
|||
148
packages/core/tests/providers/codewhale-decode.test.ts
Normal file
148
packages/core/tests/providers/codewhale-decode.test.ts
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { decodeCodeWhale, toObservations } from '../../src/providers/codewhale/index.js'
|
||||
import { ObservationEnvelope } from '../../src/observations.js'
|
||||
import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js'
|
||||
import type { DecodeContext } from '../../src/contracts.js'
|
||||
import type { CodeWhaleSessionRecords } from '../../src/providers/codewhale/types.js'
|
||||
|
||||
const context: DecodeContext = { privacyKey: 'k', providerId: 'codewhale', sourceRef: 'ref' }
|
||||
|
||||
function session(opts: Partial<CodeWhaleSessionRecords> = {}): CodeWhaleSessionRecords {
|
||||
return {
|
||||
metadata: {
|
||||
id: 'session-a',
|
||||
created_at: '2026-07-14T10:00:00.000Z',
|
||||
updated_at: '2026-07-15T12:34:56.000Z',
|
||||
total_tokens: 12345,
|
||||
model: 'anthropic/claude-sonnet-4-6',
|
||||
model_provider: 'anthropic',
|
||||
workspace: '/Users/alice/codewhale-demo',
|
||||
cost: { session_cost_usd: 0.75, subagent_cost_usd: 0.20 },
|
||||
...opts.metadata,
|
||||
},
|
||||
messages: opts.messages ?? [],
|
||||
fileMtime: opts.fileMtime ?? '2026-07-15T10:00:00.000Z',
|
||||
}
|
||||
}
|
||||
|
||||
describe('codewhale rich decode (moved to @codeburn/core)', () => {
|
||||
it('decodes one cumulative session with measured cost and tool sequence', () => {
|
||||
const records: CodeWhaleSessionRecords[] = [
|
||||
session({
|
||||
messages: [
|
||||
{ role: 'user', content: [{ type: 'text', text: 'Implement the parser' }] },
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{ type: 'thinking', thinking: '...' },
|
||||
{ type: 'tool_use', id: 't1', name: 'read_file', input: { file_path: 'src/app.ts' } },
|
||||
{ type: 'tool_use', id: 't2', name: 'exec_shell', input: { command: 'npm test && git status' } },
|
||||
{ type: 'tool_use', id: 't3', name: 'edit_file', input: { path: 'src/app.ts' } },
|
||||
] as any,
|
||||
},
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{ type: 'tool_use', id: 't4', name: 'load_skill', input: { name: 'typescript' } },
|
||||
{ type: 'tool_use', id: 't5', name: 'agent', input: { type: 'reviewer' } },
|
||||
{ type: 'server_tool_use', id: 't6', name: 'web_search', input: { query: 'CodeWhale' } },
|
||||
] as any,
|
||||
},
|
||||
],
|
||||
}),
|
||||
]
|
||||
|
||||
const { calls } = decodeCodeWhale({ records, context })
|
||||
expect(calls).toHaveLength(1)
|
||||
|
||||
const call = calls[0]!
|
||||
expect(call.provider).toBe('codewhale')
|
||||
expect(call.model).toBe('anthropic/claude-sonnet-4-6')
|
||||
expect(call.inputTokens).toBe(12345)
|
||||
expect(call.outputTokens).toBe(0)
|
||||
expect(call.measuredCostUSD).toBe(0.95)
|
||||
expect(call.webSearchRequests).toBe(1)
|
||||
expect(call.tools).toEqual(['Read', 'Bash', 'Edit', 'Skill', 'Agent', 'WebSearch'])
|
||||
expect(call.rawBashCommands).toEqual(['npm test && git status'])
|
||||
expect(call.skills).toEqual(['typescript'])
|
||||
expect(call.subagentTypes).toEqual(['reviewer'])
|
||||
expect(call.userMessage).toBe('Implement the parser')
|
||||
expect(call.sessionId).toBe('session-a')
|
||||
expect(call.project).toBe('codewhale-demo')
|
||||
expect(call.projectPath).toBe('/Users/alice/codewhale-demo')
|
||||
expect(call.toolSequence).toEqual([
|
||||
[
|
||||
{ tool: 'Read', file: 'src/app.ts' },
|
||||
{ tool: 'Bash', command: 'npm test && git status' },
|
||||
{ tool: 'Edit', file: 'src/app.ts' },
|
||||
],
|
||||
[{ tool: 'Skill' }, { tool: 'Agent' }, { tool: 'WebSearch' }],
|
||||
])
|
||||
})
|
||||
|
||||
it('falls back to estimated when no cost is recorded', () => {
|
||||
const records: CodeWhaleSessionRecords[] = [
|
||||
session({
|
||||
metadata: { id: 'session-b', total_tokens: 1000, cost: undefined },
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
]
|
||||
|
||||
const { calls } = decodeCodeWhale({ records, context })
|
||||
expect(calls).toHaveLength(1)
|
||||
expect(calls[0]!.measuredCostUSD).toBeUndefined()
|
||||
})
|
||||
|
||||
it('skips zero-token sessions with no exact cost', () => {
|
||||
const records: CodeWhaleSessionRecords[] = [
|
||||
session({
|
||||
metadata: { id: 'session-c', total_tokens: 0, cost: { session_cost_usd: 0, subagent_cost_usd: 0 } },
|
||||
messages: [],
|
||||
}),
|
||||
]
|
||||
|
||||
expect(decodeCodeWhale({ records, context }).calls).toEqual([])
|
||||
})
|
||||
|
||||
it('deduplicates by session id using the live seenKeys set', () => {
|
||||
const records: CodeWhaleSessionRecords[] = [session()]
|
||||
const seen = new Set<string>()
|
||||
expect(decodeCodeWhale({ records, context, seenKeys: seen }).calls).toHaveLength(1)
|
||||
expect(decodeCodeWhale({ records, context, seenKeys: seen }).calls).toEqual([])
|
||||
})
|
||||
|
||||
it('toObservations produces a schema-valid envelope with fingerprinted file refs', () => {
|
||||
const records: CodeWhaleSessionRecords[] = [
|
||||
session({
|
||||
messages: [
|
||||
{ role: 'user', content: 'Implement the parser' },
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{ type: 'tool_use', id: 't1', name: 'read_file', input: { file_path: 'src/app.ts' } },
|
||||
{ type: 'tool_use', id: 't2', name: 'edit_file', input: { path: 'src/app.ts' } },
|
||||
] as any,
|
||||
},
|
||||
],
|
||||
}),
|
||||
]
|
||||
|
||||
const { calls } = decodeCodeWhale({ records, context })
|
||||
const { sessions } = toObservations(
|
||||
{ sessionId: 'session-a', projectPath: '/Users/alice/codewhale-demo', calls },
|
||||
{ privacyKey: 'test-privacy-key', provider: 'codewhale' },
|
||||
)
|
||||
const envelope = {
|
||||
schemaVersion: OBSERVATION_SCHEMA_VERSION,
|
||||
generator: { name: '@codeburn/core', version: '0.0.0-test' },
|
||||
sessions,
|
||||
}
|
||||
expect(ObservationEnvelope.safeParse(envelope).success).toBe(true)
|
||||
const reads = sessions.flatMap(s => s.calls.flatMap(c => c.resourceReads ?? []))
|
||||
const edits = sessions.flatMap(s => s.calls.flatMap(c => c.resourceEdits ?? []))
|
||||
expect(reads.length).toBeGreaterThan(0)
|
||||
expect(edits.length).toBeGreaterThan(0)
|
||||
for (const ref of [...reads, ...edits]) expect(ref.resourceId).toMatch(/^[0-9a-f]{16}$/)
|
||||
})
|
||||
})
|
||||
165
packages/core/tests/providers/grok-decode.test.ts
Normal file
165
packages/core/tests/providers/grok-decode.test.ts
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { decodeGrok, toObservations } from '../../src/providers/grok/index.js'
|
||||
import { ObservationEnvelope } from '../../src/observations.js'
|
||||
import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js'
|
||||
import type { DecodeContext } from '../../src/contracts.js'
|
||||
import type { GrokSessionRecords } from '../../src/providers/grok/types.js'
|
||||
|
||||
const context: DecodeContext = { privacyKey: 'k', providerId: 'grok', sourceRef: 'ref' }
|
||||
|
||||
function sessionRecord(opts: {
|
||||
updatesLines: string[]
|
||||
summary?: Partial<GrokSessionRecords['summary']>
|
||||
signals?: GrokSessionRecords['signals']
|
||||
sourceDir?: string
|
||||
sessionName?: string
|
||||
project?: string
|
||||
}): GrokSessionRecords {
|
||||
return {
|
||||
summary: {
|
||||
info: { id: 'sess-1', cwd: '/Users/test/project' },
|
||||
created_at: '2026-06-19T11:20:40.686261Z',
|
||||
updated_at: '2026-06-19T11:31:12.282793Z',
|
||||
last_active_at: '2026-06-19T11:31:12.222328Z',
|
||||
current_model_id: 'grok-build',
|
||||
session_summary: 'User asks about the repo',
|
||||
generated_title: 'User asks about the repo',
|
||||
...opts.summary,
|
||||
},
|
||||
signals: opts.signals ?? null,
|
||||
updatesLines: opts.updatesLines,
|
||||
sourceDir: opts.sourceDir ?? '/sessions/%2FUsers%2Ftest/sess-1',
|
||||
sessionName: opts.sessionName ?? 'sess-1',
|
||||
project: opts.project ?? 'project',
|
||||
}
|
||||
}
|
||||
|
||||
function updateLine(opts: {
|
||||
totalTokens?: number
|
||||
promptId?: string
|
||||
title?: string
|
||||
rawInput?: Record<string, unknown>
|
||||
}): string {
|
||||
return JSON.stringify({
|
||||
timestamp: '2026-06-19T11:30:00.000Z',
|
||||
method: 'session/update',
|
||||
params: {
|
||||
sessionId: 'sess-1',
|
||||
update: {
|
||||
sessionUpdate: opts.title ? 'tool_call' : 'agent_message_chunk',
|
||||
...(opts.title ? { title: opts.title, toolCallId: 'c1', rawInput: opts.rawInput } : { content: { type: 'text', text: 'hi' } }),
|
||||
},
|
||||
_meta: {
|
||||
...(opts.totalTokens !== undefined ? { totalTokens: opts.totalTokens } : {}),
|
||||
...(opts.promptId ? { promptId: opts.promptId } : {}),
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
describe('grok rich decode (moved to @codeburn/core)', () => {
|
||||
it('decodes one session from updates.jsonl into a cost-free rich call', () => {
|
||||
const records: GrokSessionRecords[] = [
|
||||
sessionRecord({
|
||||
updatesLines: [
|
||||
updateLine({ totalTokens: 20000, promptId: 'p1' }),
|
||||
updateLine({ totalTokens: 25000, promptId: 'p1' }),
|
||||
updateLine({ totalTokens: 30000, promptId: 'p2' }),
|
||||
updateLine({ totalTokens: 35000, promptId: 'p2' }),
|
||||
updateLine({ title: 'read_file', rawInput: { target_directory: '.' } }),
|
||||
updateLine({ title: 'run_terminal_command', rawInput: { command: 'npm test && git status' } }),
|
||||
updateLine({ title: 'spawn_subagent', rawInput: { subagent_type: 'general-purpose' } }),
|
||||
],
|
||||
}),
|
||||
]
|
||||
|
||||
const { calls } = decodeGrok({ records, context })
|
||||
expect(calls).toHaveLength(1)
|
||||
|
||||
const call = calls[0]!
|
||||
expect(call.provider).toBe('grok')
|
||||
expect(call.model).toBe('grok-build')
|
||||
expect(call.inputTokens).toBe(35000)
|
||||
expect(call.outputTokens).toBe(10000)
|
||||
expect(call.cacheReadInputTokens).toBe(15000)
|
||||
expect(call.cachedInputTokens).toBe(15000)
|
||||
expect(call.tools).toEqual(['Read', 'Bash', 'Agent'])
|
||||
// Raw commands stay raw; base-name extraction is the host's job.
|
||||
expect(call.rawBashCommands).toEqual(['npm test && git status'])
|
||||
expect(call.subagentTypes).toEqual(['general-purpose'])
|
||||
expect(call.userMessage).toBe('User asks about the repo')
|
||||
expect(call.sessionId).toBe('sess-1')
|
||||
expect(call.project).toBe('project')
|
||||
expect(call.projectPath).toBe('/Users/test/project')
|
||||
})
|
||||
|
||||
it('sums fresh input across compaction segments', () => {
|
||||
const records: GrokSessionRecords[] = [
|
||||
sessionRecord({
|
||||
updatesLines: [
|
||||
updateLine({ totalTokens: 100000, promptId: 'p1' }),
|
||||
updateLine({ totalTokens: 400000, promptId: 'p1' }),
|
||||
// Large drop (>50%) simulates compaction.
|
||||
updateLine({ totalTokens: 20000, promptId: 'p2' }),
|
||||
updateLine({ totalTokens: 50000, promptId: 'p2' }),
|
||||
],
|
||||
}),
|
||||
]
|
||||
|
||||
const { calls } = decodeGrok({ records, context })
|
||||
expect(calls).toHaveLength(1)
|
||||
expect(calls[0]!.inputTokens).toBe(450000)
|
||||
})
|
||||
|
||||
it('skips zero-token sessions', () => {
|
||||
const records: GrokSessionRecords[] = [
|
||||
sessionRecord({
|
||||
updatesLines: [
|
||||
updateLine({ totalTokens: 0, promptId: 'p1' }),
|
||||
updateLine({ totalTokens: 0, promptId: 'p1' }),
|
||||
],
|
||||
}),
|
||||
]
|
||||
|
||||
expect(decodeGrok({ records, context }).calls).toEqual([])
|
||||
})
|
||||
|
||||
it('threads a live seenKeys set so a repeated session drops', () => {
|
||||
const records: GrokSessionRecords[] = [
|
||||
sessionRecord({
|
||||
updatesLines: [updateLine({ totalTokens: 1000, promptId: 'p1' })],
|
||||
}),
|
||||
]
|
||||
|
||||
const seen = new Set<string>()
|
||||
expect(decodeGrok({ records, context, seenKeys: seen }).calls).toHaveLength(1)
|
||||
expect(decodeGrok({ records, context, seenKeys: seen }).calls).toEqual([])
|
||||
})
|
||||
|
||||
it('toObservations produces a schema-valid, content-free envelope', () => {
|
||||
const records: GrokSessionRecords[] = [
|
||||
sessionRecord({
|
||||
updatesLines: [
|
||||
updateLine({ totalTokens: 1000, promptId: 'p1' }),
|
||||
updateLine({ title: 'read_file', rawInput: { file_path: 'src/app.ts' } }),
|
||||
],
|
||||
}),
|
||||
]
|
||||
|
||||
const { calls } = decodeGrok({ records, context })
|
||||
const { sessions } = toObservations(
|
||||
{ sessionId: 'sess-1', projectPath: '/Users/test/project', calls },
|
||||
{ privacyKey: 'test-privacy-key', provider: 'grok' },
|
||||
)
|
||||
const envelope = {
|
||||
schemaVersion: OBSERVATION_SCHEMA_VERSION,
|
||||
generator: { name: '@codeburn/core', version: '0.0.0-test' },
|
||||
sessions,
|
||||
}
|
||||
expect(ObservationEnvelope.safeParse(envelope).success).toBe(true)
|
||||
// Grok does not capture per-tool file paths, so there are no resource refs.
|
||||
const reads = sessions.flatMap(s => s.calls.flatMap(c => c.resourceReads ?? []))
|
||||
expect(reads).toEqual([])
|
||||
})
|
||||
})
|
||||
176
packages/core/tests/providers/kimi-decode.test.ts
Normal file
176
packages/core/tests/providers/kimi-decode.test.ts
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { decodeKimi, toObservations } from '../../src/providers/kimi/index.js'
|
||||
import { ObservationEnvelope } from '../../src/observations.js'
|
||||
import { OBSERVATION_SCHEMA_VERSION } from '../../src/schema.js'
|
||||
import type { DecodeContext } from '../../src/contracts.js'
|
||||
import type { KimiSessionRecords } from '../../src/providers/kimi/types.js'
|
||||
|
||||
const context: DecodeContext = { privacyKey: 'k', providerId: 'kimi', sourceRef: 'ref' }
|
||||
|
||||
function record(timestamp: number, type: string, payload: Record<string, unknown>): string {
|
||||
return JSON.stringify({ timestamp, message: { type, payload } })
|
||||
}
|
||||
|
||||
function session(opts: Partial<KimiSessionRecords> = {}): KimiSessionRecords {
|
||||
return {
|
||||
lines: opts.lines ?? [],
|
||||
configuredModel: opts.configuredModel ?? 'kimi-auto',
|
||||
sessionName: opts.sessionName ?? 'sess-a',
|
||||
}
|
||||
}
|
||||
|
||||
describe('kimi rich decode (moved to @codeburn/core)', () => {
|
||||
it('decodes StatusUpdate usage, tools, and raw bash commands', () => {
|
||||
const records: KimiSessionRecords[] = [
|
||||
session({
|
||||
configuredModel: 'kimi-k2-thinking-turbo',
|
||||
lines: [
|
||||
record(1776162400, 'TurnBegin', { user_input: 'add status endpoint' }),
|
||||
record(1776162401, 'ToolCall', {
|
||||
type: 'function',
|
||||
id: 'call-shell',
|
||||
function: { name: 'Shell', arguments: JSON.stringify({ command: 'git status && npm test' }) },
|
||||
}),
|
||||
record(1776162402, 'ToolCall', {
|
||||
type: 'function',
|
||||
id: 'call-read',
|
||||
function: { name: 'ReadFile', arguments: JSON.stringify({ path: 'src/index.ts' }) },
|
||||
}),
|
||||
record(1776162403, 'StatusUpdate', {
|
||||
message_id: 'msg-1',
|
||||
token_usage: {
|
||||
input_other: 100,
|
||||
input_cache_read: 25,
|
||||
input_cache_creation: 10,
|
||||
output: 40,
|
||||
},
|
||||
}),
|
||||
],
|
||||
}),
|
||||
]
|
||||
|
||||
const { calls } = decodeKimi({ records, context })
|
||||
expect(calls).toHaveLength(1)
|
||||
|
||||
const call = calls[0]!
|
||||
expect(call.provider).toBe('kimi')
|
||||
expect(call.model).toBe('kimi-k2-thinking-turbo')
|
||||
expect(call.inputTokens).toBe(100)
|
||||
expect(call.outputTokens).toBe(40)
|
||||
expect(call.cacheReadInputTokens).toBe(25)
|
||||
expect(call.cacheCreationInputTokens).toBe(10)
|
||||
expect(call.cachedInputTokens).toBe(25)
|
||||
expect(call.tools).toEqual(['Bash', 'Read'])
|
||||
expect(call.rawBashCommands).toEqual(['git status && npm test'])
|
||||
expect(call.userMessage).toBe('add status endpoint')
|
||||
expect(call.deduplicationKey).toBe('kimi:sess-a:msg-1')
|
||||
expect(call.sessionId).toBe('sess-a')
|
||||
})
|
||||
|
||||
it('dedups tools per turn (Set), matching the legacy in-CLI decode', () => {
|
||||
const records: KimiSessionRecords[] = [
|
||||
session({
|
||||
lines: [
|
||||
record(1776162400, 'TurnBegin', { user_input: 'x' }),
|
||||
record(1776162401, 'ToolCall', {
|
||||
type: 'function',
|
||||
function: { name: 'ReadFile', arguments: JSON.stringify({ path: 'a.ts' }) },
|
||||
}),
|
||||
record(1776162402, 'ToolCall', {
|
||||
type: 'function',
|
||||
function: { name: 'ReadFile', arguments: JSON.stringify({ path: 'b.ts' }) },
|
||||
}),
|
||||
record(1776162403, 'ToolCall', {
|
||||
type: 'function',
|
||||
function: { name: 'Shell', arguments: JSON.stringify({ command: 'ls' }) },
|
||||
}),
|
||||
record(1776162404, 'StatusUpdate', {
|
||||
message_id: 'msg-dup',
|
||||
token_usage: { input_other: 5, output: 7 },
|
||||
}),
|
||||
],
|
||||
}),
|
||||
]
|
||||
|
||||
const { calls } = decodeKimi({ records, context })
|
||||
expect(calls).toHaveLength(1)
|
||||
// Two ReadFile calls collapse to one 'Read'; raw bash commands stay raw.
|
||||
expect(calls[0]!.tools).toEqual(['Read', 'Bash'])
|
||||
expect(calls[0]!.rawBashCommands).toEqual(['ls'])
|
||||
})
|
||||
|
||||
it('filters thought parts and joins content parts for the user message', () => {
|
||||
const records: KimiSessionRecords[] = [
|
||||
session({
|
||||
lines: [
|
||||
record(1776023300, 'TurnBegin', {
|
||||
user_input: [
|
||||
{ type: 'text', text: 'refactor parser' },
|
||||
{ type: 'image_url', image_url: { url: 'file://diagram.png' } },
|
||||
{ type: 'text', text: 'carefully' },
|
||||
],
|
||||
}),
|
||||
record(1776023301, 'ToolCallRequest', {
|
||||
id: 'call-write',
|
||||
name: 'WriteFile',
|
||||
arguments: JSON.stringify({ path: 'src/parser.ts', content: 'x' }),
|
||||
}),
|
||||
record(1776023302, 'StatusUpdate', {
|
||||
message_id: 'msg-2',
|
||||
model_name: 'kimi-k2.6',
|
||||
token_usage: { input_other: 5, output: 7 },
|
||||
}),
|
||||
],
|
||||
}),
|
||||
]
|
||||
|
||||
const { calls } = decodeKimi({ records, context })
|
||||
expect(calls).toHaveLength(1)
|
||||
expect(calls[0]!.userMessage).toBe('refactor parser carefully')
|
||||
expect(calls[0]!.model).toBe('kimi-k2.6')
|
||||
expect(calls[0]!.tools).toEqual(['Write'])
|
||||
})
|
||||
|
||||
it('deduplicates repeated message ids using the live seenKeys set', () => {
|
||||
const records: KimiSessionRecords[] = [
|
||||
session({
|
||||
lines: [
|
||||
record(1776023300, 'TurnBegin', { user_input: 'x' }),
|
||||
record(1776023301, 'StatusUpdate', { message_id: 'msg-3', token_usage: { input_other: 5, output: 7 } }),
|
||||
record(1776023302, 'StatusUpdate', { message_id: 'msg-3', token_usage: { input_other: 5, output: 7 } }),
|
||||
],
|
||||
}),
|
||||
]
|
||||
|
||||
const seen = new Set<string>()
|
||||
expect(decodeKimi({ records, context, seenKeys: seen }).calls).toHaveLength(1)
|
||||
expect(decodeKimi({ records, context, seenKeys: seen }).calls).toEqual([])
|
||||
})
|
||||
|
||||
it('toObservations produces a schema-valid, content-free envelope', () => {
|
||||
const records: KimiSessionRecords[] = [
|
||||
session({
|
||||
lines: [
|
||||
record(1776162400, 'TurnBegin', { user_input: 'add status endpoint' }),
|
||||
record(1776162403, 'StatusUpdate', {
|
||||
message_id: 'msg-1',
|
||||
token_usage: { input_other: 100, output: 40 },
|
||||
}),
|
||||
],
|
||||
}),
|
||||
]
|
||||
|
||||
const { calls } = decodeKimi({ records, context })
|
||||
const { sessions } = toObservations(
|
||||
{ sessionId: 'sess-a', projectPath: '', calls },
|
||||
{ privacyKey: 'test-privacy-key', provider: 'kimi' },
|
||||
)
|
||||
const envelope = {
|
||||
schemaVersion: OBSERVATION_SCHEMA_VERSION,
|
||||
generator: { name: '@codeburn/core', version: '0.0.0-test' },
|
||||
sessions,
|
||||
}
|
||||
expect(ObservationEnvelope.safeParse(envelope).success).toBe(true)
|
||||
})
|
||||
})
|
||||
|
|
@ -12,7 +12,10 @@ export default defineConfig({
|
|||
'src/contracts.ts',
|
||||
'src/detectors/index.ts',
|
||||
'src/providers/claude/index.ts',
|
||||
'src/providers/codewhale/index.ts',
|
||||
'src/providers/codex/index.ts',
|
||||
'src/providers/grok/index.ts',
|
||||
'src/providers/kimi/index.ts',
|
||||
'src/providers/qwen/index.ts',
|
||||
],
|
||||
format: ['esm'],
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue