mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-05-17 12:20:43 +00:00
Fix V8 OOM crash on 30-day period with Buffer-based line reader and large-line parser
Three-layer fix for V8 heap exhaustion when parsing heavy session data: 1. Buffer-based readSessionLines (fs-utils.ts): Replace readline with raw Buffer streaming using Buffer.indexOf(0x0a). Eliminates ConsString trees that caused OOM when regex-flattening 100MB+ lines. Two-state machine (ACCUMULATING/SCANNING) skips old lines at ~2KB cost instead of 200MB. 2. Large-line streaming parser (parser.ts): Hand-written JSON scanner for lines >32KB extracts only cost/token/tool fields without JSON.parse, avoiding full object graph allocation. Dual string/Buffer paths. 3. Dashboard memory management (dashboard.tsx): Disable auto-refresh for heavy periods (30d/month/all), clear old dataset before reload via nextTick to allow GC, prevent overlapping reloads with mutex, lazy optimize scanning on keypress instead of useEffect. Also fixes three race conditions in dashboard reload deduplication: - Early return after nextTick bypassing finally block (permanent mutex lock) - A->B->A period switching dropping final reload (stale pending) - Stale pendingReloadRef not cleared when in-flight matches request
This commit is contained in:
iamtoruk
parent
36e94169fb
commit
2fb078bdfb
14 changed files with 1420 additions and 80 deletions
|
|
@ -9,13 +9,12 @@ import { parseAllSessions, filterProjectsByName } from './parser.js'
|
|||
import { loadPricing } from './models.js'
|
||||
import { getAllProviders } from './providers/index.js'
|
||||
import { scanAndDetect, type WasteFinding, type WasteAction, type OptimizeResult } from './optimize.js'
|
||||
import { estimateContextBudget, discoverProjectCwd, type ContextBudget } from './context-budget.js'
|
||||
import { estimateContextBudget, type ContextBudget } from './context-budget.js'
|
||||
import { dateKey } from './day-aggregator.js'
|
||||
import { CompareView } from './compare.js'
|
||||
import { getPlanUsageOrNull, type PlanUsage } from './plan-usage.js'
|
||||
import { planDisplayName } from './plans.js'
|
||||
import { getDateRange, PERIODS, PERIOD_LABELS, type Period, formatDateRangeLabel } from './cli-date.js'
|
||||
import { join } from 'path'
|
||||
import { patchStdoutForWindows } from './ink-win.js'
|
||||
|
||||
type View = 'dashboard' | 'optimize' | 'compare'
|
||||
|
|
@ -25,6 +24,7 @@ const ORANGE = '#FF8C42'
|
|||
const DIM = '#555555'
|
||||
const GOLD = '#FFD700'
|
||||
const PLAN_BAR_WIDTH = 10
|
||||
const HEAVY_PERIODS = new Set<Period>(['30days', 'month', 'all'])
|
||||
|
||||
const LANG_DISPLAY_NAMES: Record<string, string> = {
|
||||
javascript: 'JavaScript', typescript: 'TypeScript', python: 'Python',
|
||||
|
|
@ -101,6 +101,14 @@ function getPeriodRange(period: Period): { start: Date; end: Date } {
|
|||
return getDateRange(period).range
|
||||
}
|
||||
|
||||
function isHeavyPeriod(period: Period): boolean {
|
||||
return HEAVY_PERIODS.has(period)
|
||||
}
|
||||
|
||||
function nextTick(): Promise<void> {
|
||||
return new Promise(resolve => setImmediate(resolve))
|
||||
}
|
||||
|
||||
type Layout = { dashWidth: number; wide: boolean; halfWidth: number; barWidth: number }
|
||||
|
||||
function getLayout(columns?: number): Layout {
|
||||
|
|
@ -659,8 +667,8 @@ function StatusBar({ width, showProvider, view, findingCount, optimizeAvailable,
|
|||
<Text color={ORANGE} bold>5</Text><Text dimColor> 6 months</Text>
|
||||
</>
|
||||
)}
|
||||
{!isOptimize && optimizeAvailable && findingCount != null && findingCount > 0 && (
|
||||
<><Text dimColor> </Text><Text color={ORANGE} bold>o</Text><Text dimColor> optimize</Text><Text color="#F55B5B"> ({findingCount})</Text></>
|
||||
{!isOptimize && optimizeAvailable && (
|
||||
<><Text dimColor> </Text><Text color={ORANGE} bold>o</Text><Text dimColor> optimize</Text>{findingCount != null && findingCount > 0 ? <Text color="#F55B5B"> ({findingCount})</Text> : null}</>
|
||||
)}
|
||||
{!isOptimize && compareAvailable && (
|
||||
<><Text dimColor> </Text><Text color={ORANGE} bold>c</Text><Text dimColor> compare</Text></>
|
||||
|
|
@ -716,6 +724,7 @@ function InteractiveDashboard({ initialProjects, initialPeriod, initialProvider,
|
|||
const [detectedProviders, setDetectedProviders] = useState<string[]>([])
|
||||
const [view, setView] = useState<View>('dashboard')
|
||||
const [optimizeResult, setOptimizeResult] = useState<OptimizeResult | null>(null)
|
||||
const [optimizeLoading, setOptimizeLoading] = useState(false)
|
||||
const [projectBudgets, setProjectBudgets] = useState<Map<string, ContextBudget>>(new Map())
|
||||
const [planUsage, setPlanUsage] = useState<PlanUsage | undefined>(initialPlanUsage)
|
||||
// Cursor for the OptimizeView's findings window. Reset whenever the user
|
||||
|
|
@ -726,13 +735,16 @@ function InteractiveDashboard({ initialProjects, initialPeriod, initialProvider,
|
|||
const { columns } = useWindowSize()
|
||||
const { dashWidth } = getLayout(columns)
|
||||
const multipleProviders = detectedProviders.length > 1
|
||||
const optimizeAvailable = activeProvider === 'all' || activeProvider === 'claude'
|
||||
const optimizeAvailable = !isCustomRange && (activeProvider === 'all' || activeProvider === 'claude')
|
||||
const modelCount = new Set(
|
||||
projects.flatMap(p => p.sessions.flatMap(s => Object.keys(s.modelBreakdown)))
|
||||
).size
|
||||
const compareAvailable = modelCount >= 2
|
||||
const debounceRef = useRef<ReturnType<typeof setTimeout> | null>(null)
|
||||
const reloadGenerationRef = useRef(0)
|
||||
const reloadInFlightRef = useRef(false)
|
||||
const currentReloadRef = useRef<{ period: Period; provider: string } | null>(null)
|
||||
const pendingReloadRef = useRef<{ period: Period; provider: string } | null>(null)
|
||||
const findingCount = optimizeResult?.findings.length ?? 0
|
||||
|
||||
useEffect(() => {
|
||||
|
|
@ -749,13 +761,11 @@ function InteractiveDashboard({ initialProjects, initialPeriod, initialProvider,
|
|||
useEffect(() => {
|
||||
let cancelled = false
|
||||
async function loadBudgets() {
|
||||
const claudeDir = join(homedir(), '.claude', 'projects')
|
||||
const budgets = new Map<string, ContextBudget>()
|
||||
for (const project of projects.slice(0, 8)) {
|
||||
if (cancelled) return
|
||||
const cwd = await discoverProjectCwd(join(claudeDir, project.project))
|
||||
if (!cwd) continue
|
||||
budgets.set(project.project, await estimateContextBudget(cwd))
|
||||
if (!project.projectPath.startsWith('/')) continue
|
||||
budgets.set(project.project, await estimateContextBudget(project.projectPath))
|
||||
}
|
||||
if (!cancelled) setProjectBudgets(budgets)
|
||||
}
|
||||
|
|
@ -763,23 +773,30 @@ function InteractiveDashboard({ initialProjects, initialPeriod, initialProvider,
|
|||
return () => { cancelled = true }
|
||||
}, [projects])
|
||||
|
||||
useEffect(() => {
|
||||
if (!optimizeAvailable) { setOptimizeResult(null); return }
|
||||
let cancelled = false
|
||||
async function scan() {
|
||||
if (projects.length === 0) { setOptimizeResult(null); return }
|
||||
const result = await scanAndDetect(projects, getPeriodRange(period))
|
||||
if (!cancelled) setOptimizeResult(result)
|
||||
}
|
||||
scan()
|
||||
return () => { cancelled = true }
|
||||
}, [projects, period, optimizeAvailable])
|
||||
|
||||
const reloadData = useCallback(async (p: Period, prov: string) => {
|
||||
if (reloadInFlightRef.current) {
|
||||
const current = currentReloadRef.current
|
||||
if (current?.period === p && current.provider === prov) {
|
||||
pendingReloadRef.current = null
|
||||
return
|
||||
}
|
||||
reloadGenerationRef.current++
|
||||
pendingReloadRef.current = { period: p, provider: prov }
|
||||
return
|
||||
}
|
||||
reloadInFlightRef.current = true
|
||||
currentReloadRef.current = { period: p, provider: prov }
|
||||
const generation = ++reloadGenerationRef.current
|
||||
setLoading(true)
|
||||
setOptimizeLoading(false)
|
||||
setOptimizeResult(null)
|
||||
try {
|
||||
if (isHeavyPeriod(p)) {
|
||||
setProjects([])
|
||||
setProjectBudgets(new Map())
|
||||
await nextTick()
|
||||
if (reloadGenerationRef.current !== generation) return
|
||||
}
|
||||
const range = getPeriodRange(p)
|
||||
const data = await parseAllSessions(range, prov)
|
||||
if (reloadGenerationRef.current !== generation) return
|
||||
|
|
@ -797,11 +814,37 @@ function InteractiveDashboard({ initialProjects, initialPeriod, initialProvider,
|
|||
if (reloadGenerationRef.current === generation) {
|
||||
setLoading(false)
|
||||
}
|
||||
reloadInFlightRef.current = false
|
||||
currentReloadRef.current = null
|
||||
const pending = pendingReloadRef.current
|
||||
pendingReloadRef.current = null
|
||||
if (pending) {
|
||||
void reloadData(pending.period, pending.provider)
|
||||
}
|
||||
}
|
||||
}, [projectFilter, excludeFilter])
|
||||
|
||||
const loadOptimizeResult = useCallback(async () => {
|
||||
if (!optimizeAvailable || projects.length === 0 || optimizeLoading) return
|
||||
setView('optimize')
|
||||
setFindingsCursor(0)
|
||||
if (optimizeResult) return
|
||||
|
||||
const generation = reloadGenerationRef.current
|
||||
setOptimizeLoading(true)
|
||||
try {
|
||||
const result = await scanAndDetect(projects, getPeriodRange(period))
|
||||
if (reloadGenerationRef.current === generation) setOptimizeResult(result)
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
} finally {
|
||||
if (reloadGenerationRef.current === generation) setOptimizeLoading(false)
|
||||
}
|
||||
}, [optimizeAvailable, projects, period, optimizeLoading, optimizeResult])
|
||||
|
||||
useEffect(() => {
|
||||
if (!refreshSeconds || refreshSeconds <= 0) return
|
||||
if (isHeavyPeriod(period)) return
|
||||
const id = setInterval(() => { reloadData(period, activeProvider) }, refreshSeconds * 1000)
|
||||
return () => clearInterval(id)
|
||||
}, [refreshSeconds, period, activeProvider, reloadData])
|
||||
|
|
@ -831,7 +874,7 @@ function InteractiveDashboard({ initialProjects, initialPeriod, initialProvider,
|
|||
|
||||
useInput((input, key) => {
|
||||
if (input === 'q') { exit(); return }
|
||||
if (input === 'o' && findingCount > 0 && view === 'dashboard' && optimizeAvailable) { setView('optimize'); return }
|
||||
if (input === 'o' && view === 'dashboard' && optimizeAvailable) { void loadOptimizeResult(); return }
|
||||
if ((input === 'b' || key.escape) && view === 'optimize') { setView('dashboard'); setFindingsCursor(0); return }
|
||||
if (view === 'optimize') {
|
||||
const total = optimizeResult?.findings.length ?? 0
|
||||
|
|
@ -869,7 +912,7 @@ function InteractiveDashboard({ initialProjects, initialPeriod, initialProvider,
|
|||
|
||||
const headerLabel = customRangeLabel ?? PERIOD_LABELS[period]
|
||||
|
||||
if (loading) {
|
||||
if (loading || optimizeLoading) {
|
||||
return (
|
||||
<Box flexDirection="column" width={dashWidth}>
|
||||
{!isCustomRange && <PeriodTabs active={period} providerName={activeProvider} showProvider={view !== 'compare' && multipleProviders} />}
|
||||
|
|
@ -882,7 +925,9 @@ function InteractiveDashboard({ initialProjects, initialPeriod, initialProvider,
|
|||
<Text dimColor>Loading {headerLabel} model data...</Text>
|
||||
</Box>
|
||||
</Box>
|
||||
: <Panel title="CodeBurn" color={ORANGE} width={dashWidth}><Text dimColor>Loading {headerLabel}...</Text></Panel>}
|
||||
: view === 'optimize'
|
||||
? <Panel title="CodeBurn Optimize" color={ORANGE} width={dashWidth}><Text dimColor>Scanning {headerLabel}...</Text></Panel>
|
||||
: <Panel title="CodeBurn" color={ORANGE} width={dashWidth}><Text dimColor>Loading {headerLabel}...</Text></Panel>}
|
||||
{view !== 'compare' && <StatusBar width={dashWidth} showProvider={multipleProviders} view={view} findingCount={0} optimizeAvailable={false} compareAvailable={false} customRange={isCustomRange} />}
|
||||
</Box>
|
||||
)
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
139
src/fs-utils.ts
139
src/fs-utils.ts
|
|
@ -1,12 +1,11 @@
|
|||
import { readFile, stat } from 'fs/promises'
|
||||
import { readFileSync, statSync, createReadStream } from 'fs'
|
||||
import { createInterface } from 'readline'
|
||||
|
||||
// Hard cap well below V8's 512 MB string limit even with split('\n') doubling.
|
||||
// Stream threshold chosen as empirical breakeven between readFile+split peak
|
||||
// memory and createReadStream+readline overhead for typical session files.
|
||||
// Hard cap well below V8's 512 MB string limit. Callers that need line-by-line
|
||||
// processing should use readSessionLines(), which avoids materializing the
|
||||
// whole file and can return large lines as Buffers.
|
||||
export const MAX_SESSION_FILE_BYTES = 128 * 1024 * 1024
|
||||
export const STREAM_THRESHOLD_BYTES = 8 * 1024 * 1024
|
||||
export const LARGE_STREAM_LINE_BYTES = 32 * 1024
|
||||
|
||||
// Line-by-line streaming has bounded memory (one line at a time) and is not
|
||||
// constrained by V8's string limit, so it can safely handle multi-GB session
|
||||
|
|
@ -23,14 +22,6 @@ function warn(msg: string): void {
|
|||
if (verbose()) process.stderr.write(`codeburn: ${msg}\n`)
|
||||
}
|
||||
|
||||
async function readViaStream(filePath: string): Promise<string> {
|
||||
const chunks: string[] = []
|
||||
const stream = createReadStream(filePath, { encoding: 'utf-8' })
|
||||
const rl = createInterface({ input: stream, crlfDelay: Infinity })
|
||||
for await (const line of rl) chunks.push(line)
|
||||
return chunks.join('\n')
|
||||
}
|
||||
|
||||
export async function readSessionFile(filePath: string): Promise<string | null> {
|
||||
let size: number
|
||||
try {
|
||||
|
|
@ -46,7 +37,6 @@ export async function readSessionFile(filePath: string): Promise<string | null>
|
|||
}
|
||||
|
||||
try {
|
||||
if (size >= STREAM_THRESHOLD_BYTES) return await readViaStream(filePath)
|
||||
return await readFile(filePath, 'utf-8')
|
||||
} catch (err) {
|
||||
warn(`read failed for ${filePath}: ${(err as NodeJS.ErrnoException).code ?? 'unknown'}`)
|
||||
|
|
@ -76,7 +66,27 @@ export function readSessionFileSync(filePath: string): string | null {
|
|||
}
|
||||
}
|
||||
|
||||
export async function* readSessionLines(filePath: string): AsyncGenerator<string> {
|
||||
export type SessionLine = string | Buffer
|
||||
|
||||
type ReadSessionLinesOptions = {
|
||||
largeLineAsBuffer?: boolean
|
||||
largeLineThresholdBytes?: number
|
||||
}
|
||||
|
||||
export function readSessionLines(
|
||||
filePath: string,
|
||||
shouldSkipHead?: (head: string) => boolean,
|
||||
): AsyncGenerator<string>
|
||||
export function readSessionLines(
|
||||
filePath: string,
|
||||
shouldSkipHead?: (head: string) => boolean,
|
||||
options?: ReadSessionLinesOptions & { largeLineAsBuffer: true },
|
||||
): AsyncGenerator<SessionLine>
|
||||
export async function* readSessionLines(
|
||||
filePath: string,
|
||||
shouldSkipHead?: (head: string) => boolean,
|
||||
options: ReadSessionLinesOptions = {},
|
||||
): AsyncGenerator<SessionLine> {
|
||||
let size: number
|
||||
try {
|
||||
size = (await stat(filePath)).size
|
||||
|
|
@ -92,10 +102,103 @@ export async function* readSessionLines(filePath: string): AsyncGenerator<string
|
|||
return
|
||||
}
|
||||
|
||||
const stream = createReadStream(filePath, { encoding: 'utf-8' })
|
||||
const rl = createInterface({ input: stream, crlfDelay: Infinity })
|
||||
// Raw Buffers — no encoding. This avoids readline's ConsString trees
|
||||
// which OOM on V8 when regex-flattening 100 MB+ lines.
|
||||
const stream = createReadStream(filePath)
|
||||
const SKIP_HEAD = 2048
|
||||
const largeLineThreshold = options.largeLineThresholdBytes ?? LARGE_STREAM_LINE_BYTES
|
||||
const formatLine = (buf: Buffer, lineLen: number, head?: string): SessionLine => {
|
||||
if (options.largeLineAsBuffer && lineLen > largeLineThreshold) return buf
|
||||
return head !== undefined && lineLen <= SKIP_HEAD ? head : buf.toString('utf-8')
|
||||
}
|
||||
let parts: Buffer[] = []
|
||||
let len = 0
|
||||
let skipping = false
|
||||
let headChecked = false
|
||||
|
||||
try {
|
||||
for await (const line of rl) yield line
|
||||
for await (const raw of stream) {
|
||||
const chunk = raw as Buffer
|
||||
let pos = 0
|
||||
|
||||
while (pos < chunk.length) {
|
||||
const nl = chunk.indexOf(0x0a, pos)
|
||||
|
||||
if (skipping) {
|
||||
if (nl === -1) {
|
||||
pos = chunk.length
|
||||
} else {
|
||||
skipping = false
|
||||
pos = nl + 1
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if (nl !== -1) {
|
||||
if (pos < nl) {
|
||||
parts.push(chunk.subarray(pos, nl))
|
||||
len += nl - pos
|
||||
}
|
||||
pos = nl + 1
|
||||
|
||||
if (len === 0) {
|
||||
parts = []
|
||||
headChecked = false
|
||||
continue
|
||||
}
|
||||
|
||||
const buf = parts.length === 1 ? parts[0]! : Buffer.concat(parts, len)
|
||||
const lineLen = len
|
||||
parts = []
|
||||
len = 0
|
||||
headChecked = false
|
||||
|
||||
if (shouldSkipHead) {
|
||||
const head = lineLen > SKIP_HEAD
|
||||
? buf.subarray(0, SKIP_HEAD).toString('utf-8')
|
||||
: buf.toString('utf-8')
|
||||
if (shouldSkipHead(head)) continue
|
||||
yield formatLine(buf, lineLen, head)
|
||||
} else {
|
||||
yield formatLine(buf, lineLen)
|
||||
}
|
||||
} else {
|
||||
const slice = chunk.subarray(pos)
|
||||
parts.push(slice)
|
||||
len += slice.length
|
||||
pos = chunk.length
|
||||
|
||||
// Mid-line skip: once we have enough bytes to check the head,
|
||||
// enter scanning mode — just look for \n without accumulating.
|
||||
if (shouldSkipHead && !headChecked && len >= SKIP_HEAD) {
|
||||
headChecked = true
|
||||
const headBuf = parts.length === 1
|
||||
? parts[0]!.subarray(0, SKIP_HEAD)
|
||||
: Buffer.concat(parts, len).subarray(0, SKIP_HEAD)
|
||||
if (shouldSkipHead(headBuf.toString('utf-8'))) {
|
||||
skipping = true
|
||||
parts = []
|
||||
len = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!skipping && len > 0) {
|
||||
const buf = parts.length === 1 ? parts[0]! : Buffer.concat(parts, len)
|
||||
const lineLen = len
|
||||
if (shouldSkipHead) {
|
||||
const head = lineLen > SKIP_HEAD
|
||||
? buf.subarray(0, SKIP_HEAD).toString('utf-8')
|
||||
: buf.toString('utf-8')
|
||||
if (!shouldSkipHead(head)) {
|
||||
yield formatLine(buf, lineLen, head)
|
||||
}
|
||||
} else {
|
||||
yield formatLine(buf, lineLen)
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
warn(`stream read failed for ${filePath}: ${(err as NodeJS.ErrnoException).code ?? 'unknown'}`)
|
||||
} finally {
|
||||
|
|
|
|||
22
src/main.ts
22
src/main.ts
|
|
@ -2,7 +2,7 @@ import { Command } from 'commander'
|
|||
import { installMenubarApp } from './menubar-installer.js'
|
||||
import { exportCsv, exportJson, type PeriodExport } from './export.js'
|
||||
import { loadPricing, setModelAliases } from './models.js'
|
||||
import { parseAllSessions, filterProjectsByName, clearSessionCache } from './parser.js'
|
||||
import { parseAllSessions, filterProjectsByName, filterProjectsByDateRange, clearSessionCache } from './parser.js'
|
||||
import { convertCost } from './currency.js'
|
||||
import { renderStatusBar } from './format.js'
|
||||
import { type PeriodData, type ProviderCost } from './menubar-json.js'
|
||||
|
|
@ -615,13 +615,19 @@ program
|
|||
process.exit(1)
|
||||
}
|
||||
|
||||
const periods: PeriodExport[] = customRange
|
||||
? [{ label: formatDateRangeLabel(opts.from, opts.to), projects: fp(await parseAllSessions(customRange, pf)) }]
|
||||
: [
|
||||
{ label: 'Today', projects: fp(await parseAllSessions(getDateRange('today').range, pf)) },
|
||||
{ label: '7 Days', projects: fp(await parseAllSessions(getDateRange('week').range, pf)) },
|
||||
{ label: '30 Days', projects: fp(await parseAllSessions(getDateRange('30days').range, pf)) },
|
||||
]
|
||||
let periods: PeriodExport[]
|
||||
if (customRange) {
|
||||
periods = [{ label: formatDateRangeLabel(opts.from, opts.to), projects: fp(await parseAllSessions(customRange, pf)) }]
|
||||
clearSessionCache()
|
||||
} else {
|
||||
const thirtyDayProjects = fp(await parseAllSessions(getDateRange('30days').range, pf))
|
||||
clearSessionCache()
|
||||
periods = [
|
||||
{ label: 'Today', projects: filterProjectsByDateRange(thirtyDayProjects, getDateRange('today').range) },
|
||||
{ label: '7 Days', projects: filterProjectsByDateRange(thirtyDayProjects, getDateRange('week').range) },
|
||||
{ label: '30 Days', projects: thirtyDayProjects },
|
||||
]
|
||||
}
|
||||
|
||||
if (periods.every(p => p.projects.length === 0)) {
|
||||
console.log('\n No usage data found.\n')
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ type Bucket = {
|
|||
}
|
||||
|
||||
type ModelKey = string
|
||||
type CategoryKey = string
|
||||
type CategoryKey = TaskCategory
|
||||
|
||||
function bucketKey(provider: string, model: string, category: TaskCategory | null): string {
|
||||
return `${provider} ${model} ${category ?? ''}`
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { homedir } from 'os'
|
|||
|
||||
import { readSessionLines, readSessionFileSync } from './fs-utils.js'
|
||||
import { discoverAllSessions } from './providers/index.js'
|
||||
import { parseJsonlLine, shouldSkipLine } from './parser.js'
|
||||
import type { DateRange, ProjectSummary } from './types.js'
|
||||
import { formatCost } from './currency.js'
|
||||
import { formatTokens } from './format.js'
|
||||
|
|
@ -141,6 +142,8 @@ const SHELL_PROFILES = ['.zshrc', '.bashrc', '.bash_profile', '.profile']
|
|||
const TOP_ITEMS_PREVIEW = 3
|
||||
const GHOST_NAMES_PREVIEW = 5
|
||||
const GHOST_CLEANUP_COMMANDS_LIMIT = 10
|
||||
const OPTIMIZE_TEXT_CAP = 2000
|
||||
const OPTIMIZE_FIELD_CAP = 500
|
||||
|
||||
// ============================================================================
|
||||
// Types
|
||||
|
|
@ -209,7 +212,33 @@ type ScanData = {
|
|||
// JSONL scanner
|
||||
// ============================================================================
|
||||
|
||||
const FILE_READ_CONCURRENCY = 16
|
||||
function cappedString(value: unknown, cap = OPTIMIZE_FIELD_CAP): string | undefined {
|
||||
return typeof value === 'string' ? value.slice(0, cap) : undefined
|
||||
}
|
||||
|
||||
function compactOptimizeInput(name: string, input: unknown): Record<string, unknown> {
|
||||
if (!input || typeof input !== 'object') return {}
|
||||
const raw = input as Record<string, unknown>
|
||||
if (isReadTool(name)) {
|
||||
const filePath = cappedString(raw['file_path'], OPTIMIZE_TEXT_CAP)
|
||||
return filePath ? { file_path: filePath } : {}
|
||||
}
|
||||
if (name === 'Agent' || name === 'Task') {
|
||||
const subagentType = cappedString(raw['subagent_type'])
|
||||
return subagentType ? { subagent_type: subagentType } : {}
|
||||
}
|
||||
if (name === 'Skill') {
|
||||
const skill = cappedString(raw['skill'])
|
||||
const skillName = cappedString(raw['name'])
|
||||
return {
|
||||
...(skill ? { skill } : {}),
|
||||
...(skillName ? { name: skillName } : {}),
|
||||
}
|
||||
}
|
||||
return {}
|
||||
}
|
||||
|
||||
const FILE_READ_CONCURRENCY = 4
|
||||
const RESULT_CACHE_TTL_MS = 60_000
|
||||
const RECENT_WINDOW_HOURS = 48
|
||||
const RECENT_WINDOW_MS = RECENT_WINDOW_HOURS * 60 * 60 * 1000
|
||||
|
|
@ -286,10 +315,19 @@ export async function scanJsonlFile(
|
|||
const sessionId = basename(filePath, '.jsonl')
|
||||
let lastVersion = ''
|
||||
|
||||
for await (const line of readSessionLines(filePath)) {
|
||||
if (!line.trim()) continue
|
||||
let entry: Record<string, unknown>
|
||||
try { entry = JSON.parse(line) } catch { continue }
|
||||
const skipThreshold = dateRange
|
||||
? new Date(dateRange.start.getTime() - 86_400_000).toISOString()
|
||||
: null
|
||||
const skipFn = dateRange
|
||||
? (head: string) => shouldSkipLine(head, skipThreshold!)
|
||||
: undefined
|
||||
const lines = readSessionLines(filePath, skipFn, { largeLineAsBuffer: true })
|
||||
for await (const line of lines) {
|
||||
if (typeof line === 'string' && !line.trim()) continue
|
||||
if (Buffer.isBuffer(line) && line.length === 0) continue
|
||||
const parsed = parseJsonlLine(line)
|
||||
if (!parsed) continue
|
||||
const entry = parsed as Record<string, unknown>
|
||||
|
||||
if (entry.version && typeof entry.version === 'string') lastVersion = entry.version
|
||||
|
||||
|
|
@ -304,11 +342,15 @@ export async function scanJsonlFile(
|
|||
const msg = entry.message as Record<string, unknown> | undefined
|
||||
const msgContent = msg?.content
|
||||
if (typeof msgContent === 'string') {
|
||||
userMessages.push(msgContent)
|
||||
userMessages.push(msgContent.slice(0, OPTIMIZE_TEXT_CAP))
|
||||
} else if (Array.isArray(msgContent)) {
|
||||
let remaining = OPTIMIZE_TEXT_CAP
|
||||
for (const block of msgContent) {
|
||||
if (remaining <= 0) break
|
||||
if (block && typeof block === 'object' && block.type === 'text' && typeof block.text === 'string') {
|
||||
userMessages.push(block.text)
|
||||
const text = block.text.slice(0, remaining)
|
||||
userMessages.push(text)
|
||||
remaining -= text.length
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -330,9 +372,10 @@ export async function scanJsonlFile(
|
|||
|
||||
for (const block of blocks) {
|
||||
if (block.type !== 'tool_use') continue
|
||||
const name = typeof block.name === 'string' ? block.name : ''
|
||||
calls.push({
|
||||
name: block.name as string,
|
||||
input: (block.input as Record<string, unknown>) ?? {},
|
||||
name,
|
||||
input: compactOptimizeInput(name, block.input),
|
||||
sessionId,
|
||||
project,
|
||||
recent,
|
||||
|
|
|
|||
776
src/parser.ts
776
src/parser.ts
|
|
@ -32,7 +32,18 @@ function normalizeProjectPathKey(projectPath: string): string {
|
|||
return (normalized.replace(/\/+$/, '') || normalized).toLowerCase()
|
||||
}
|
||||
|
||||
function parseJsonlLine(line: string): JournalEntry | null {
|
||||
const LARGE_JSONL_LINE_BYTES = 32 * 1024
|
||||
|
||||
export function parseJsonlLine(line: string | Buffer): JournalEntry | null {
|
||||
if (Buffer.isBuffer(line)) {
|
||||
if (line.length > LARGE_JSONL_LINE_BYTES) return parseLargeJsonlBuffer(line)
|
||||
try {
|
||||
return JSON.parse(line.toString('utf-8')) as JournalEntry
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
if (line.length > LARGE_JSONL_LINE_BYTES) return parseLargeJsonlLine(line)
|
||||
try {
|
||||
return JSON.parse(line) as JournalEntry
|
||||
} catch {
|
||||
|
|
@ -40,6 +51,721 @@ function parseJsonlLine(line: string): JournalEntry | null {
|
|||
}
|
||||
}
|
||||
|
||||
const RAW_HEAD_BYTES = 2048
|
||||
|
||||
type JsonValueBounds = {
|
||||
start: number
|
||||
end: number
|
||||
kind: 'string' | 'object' | 'array' | 'scalar'
|
||||
}
|
||||
|
||||
function findJsonStringEnd(source: string, start: number, limit = source.length): number {
|
||||
for (let i = start + 1; i < limit; i++) {
|
||||
const ch = source.charCodeAt(i)
|
||||
if (ch === 0x5c) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (ch === 0x22) return i
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
function findJsonContainerEnd(source: string, start: number, open: number, close: number, limit = source.length): number {
|
||||
let depth = 0
|
||||
let inString = false
|
||||
for (let i = start; i < limit; i++) {
|
||||
const ch = source.charCodeAt(i)
|
||||
if (inString) {
|
||||
if (ch === 0x5c) {
|
||||
i++
|
||||
} else if (ch === 0x22) {
|
||||
inString = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (ch === 0x22) {
|
||||
inString = true
|
||||
} else if (ch === open) {
|
||||
depth++
|
||||
} else if (ch === close) {
|
||||
depth--
|
||||
if (depth === 0) return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
function findJsonValueBounds(source: string, start: number, limit = source.length): JsonValueBounds | null {
|
||||
let i = start
|
||||
while (i < limit && /\s/.test(source[i]!)) i++
|
||||
if (i >= limit) return null
|
||||
const ch = source.charCodeAt(i)
|
||||
if (ch === 0x22) {
|
||||
const end = findJsonStringEnd(source, i, limit)
|
||||
return end === -1 ? null : { start: i, end: end + 1, kind: 'string' }
|
||||
}
|
||||
if (ch === 0x7b) {
|
||||
const end = findJsonContainerEnd(source, i, 0x7b, 0x7d, limit)
|
||||
return end === -1 ? null : { start: i, end: end + 1, kind: 'object' }
|
||||
}
|
||||
if (ch === 0x5b) {
|
||||
const end = findJsonContainerEnd(source, i, 0x5b, 0x5d, limit)
|
||||
return end === -1 ? null : { start: i, end: end + 1, kind: 'array' }
|
||||
}
|
||||
let end = i
|
||||
while (end < limit) {
|
||||
const c = source.charCodeAt(end)
|
||||
if (c === 0x2c || c === 0x7d || c === 0x5d || /\s/.test(source[end]!)) break
|
||||
end++
|
||||
}
|
||||
return { start: i, end, kind: 'scalar' }
|
||||
}
|
||||
|
||||
function findObjectFieldValue(source: string, objectStart: number, objectEnd: number, field: string): JsonValueBounds | null {
|
||||
if (source.charCodeAt(objectStart) !== 0x7b) return null
|
||||
let i = objectStart + 1
|
||||
while (i < objectEnd - 1) {
|
||||
while (i < objectEnd && /\s/.test(source[i]!)) i++
|
||||
if (source.charCodeAt(i) === 0x2c) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (source.charCodeAt(i) !== 0x22) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
const keyEnd = findJsonStringEnd(source, i, objectEnd)
|
||||
if (keyEnd === -1) return null
|
||||
const key = source.slice(i + 1, keyEnd)
|
||||
i = keyEnd + 1
|
||||
while (i < objectEnd && /\s/.test(source[i]!)) i++
|
||||
if (source.charCodeAt(i) !== 0x3a) continue
|
||||
const value = findJsonValueBounds(source, i + 1, objectEnd)
|
||||
if (!value) return null
|
||||
if (key === field) return value
|
||||
i = value.end
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function readJsonString(source: string, bounds: JsonValueBounds | null, cap = Number.POSITIVE_INFINITY): string | undefined {
|
||||
if (!bounds || bounds.kind !== 'string') return undefined
|
||||
let out = ''
|
||||
for (let i = bounds.start + 1; i < bounds.end - 1 && out.length < cap; i++) {
|
||||
const ch = source[i]!
|
||||
if (ch !== '\\') {
|
||||
out += ch
|
||||
continue
|
||||
}
|
||||
const next = source[++i]
|
||||
if (!next) break
|
||||
if (next === 'n') out += '\n'
|
||||
else if (next === 'r') out += '\r'
|
||||
else if (next === 't') out += '\t'
|
||||
else if (next === 'b') out += '\b'
|
||||
else if (next === 'f') out += '\f'
|
||||
else if (next === 'u' && i + 4 < bounds.end) {
|
||||
const hex = source.slice(i + 1, i + 5)
|
||||
const code = Number.parseInt(hex, 16)
|
||||
if (Number.isFinite(code)) out += String.fromCharCode(code)
|
||||
i += 4
|
||||
} else {
|
||||
out += next
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
function readJsonNumberField(source: string, objectBounds: JsonValueBounds | null, field: string): number | undefined {
|
||||
if (!objectBounds || objectBounds.kind !== 'object') return undefined
|
||||
const bounds = findObjectFieldValue(source, objectBounds.start, objectBounds.end, field)
|
||||
if (!bounds) return undefined
|
||||
const value = Number(source.slice(bounds.start, bounds.end))
|
||||
return Number.isFinite(value) ? value : undefined
|
||||
}
|
||||
|
||||
function parseLargeUsage(source: string, usageBounds: JsonValueBounds | null) {
|
||||
const usage: AssistantMessageContent['usage'] = {
|
||||
input_tokens: readJsonNumberField(source, usageBounds, 'input_tokens') ?? 0,
|
||||
output_tokens: readJsonNumberField(source, usageBounds, 'output_tokens') ?? 0,
|
||||
cache_creation_input_tokens: readJsonNumberField(source, usageBounds, 'cache_creation_input_tokens'),
|
||||
cache_read_input_tokens: readJsonNumberField(source, usageBounds, 'cache_read_input_tokens'),
|
||||
}
|
||||
|
||||
if (usageBounds?.kind === 'object') {
|
||||
const cacheCreation = findObjectFieldValue(source, usageBounds.start, usageBounds.end, 'cache_creation')
|
||||
const ephemeral5m = readJsonNumberField(source, cacheCreation, 'ephemeral_5m_input_tokens')
|
||||
const ephemeral1h = readJsonNumberField(source, cacheCreation, 'ephemeral_1h_input_tokens')
|
||||
if (ephemeral5m !== undefined || ephemeral1h !== undefined) {
|
||||
;(usage as AssistantMessageContent['usage']).cache_creation = {
|
||||
...(ephemeral5m !== undefined ? { ephemeral_5m_input_tokens: ephemeral5m } : {}),
|
||||
...(ephemeral1h !== undefined ? { ephemeral_1h_input_tokens: ephemeral1h } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
const serverToolUse = findObjectFieldValue(source, usageBounds.start, usageBounds.end, 'server_tool_use')
|
||||
const webSearch = readJsonNumberField(source, serverToolUse, 'web_search_requests')
|
||||
const webFetch = readJsonNumberField(source, serverToolUse, 'web_fetch_requests')
|
||||
if (webSearch !== undefined || webFetch !== undefined) {
|
||||
;(usage as AssistantMessageContent['usage']).server_tool_use = {
|
||||
...(webSearch !== undefined ? { web_search_requests: webSearch } : {}),
|
||||
...(webFetch !== undefined ? { web_fetch_requests: webFetch } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
const speed = readJsonString(source, findObjectFieldValue(source, usageBounds.start, usageBounds.end, 'speed'))
|
||||
if (speed === 'standard' || speed === 'fast') usage.speed = speed
|
||||
}
|
||||
|
||||
return usage
|
||||
}
|
||||
|
||||
function extractLargeToolBlocks(source: string, contentBounds: JsonValueBounds | null): ToolUseBlock[] {
|
||||
if (!contentBounds || contentBounds.kind !== 'array') return []
|
||||
const tools: ToolUseBlock[] = []
|
||||
let i = contentBounds.start + 1
|
||||
while (i < contentBounds.end - 1 && tools.length < MAX_TOOL_BLOCKS) {
|
||||
while (i < contentBounds.end && /\s/.test(source[i]!)) i++
|
||||
if (source.charCodeAt(i) === 0x2c) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (source.charCodeAt(i) !== 0x7b) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
const objectEnd = findJsonContainerEnd(source, i, 0x7b, 0x7d, contentBounds.end)
|
||||
if (objectEnd === -1) break
|
||||
const objectBounds = { start: i, end: objectEnd + 1, kind: 'object' as const }
|
||||
const blockType = readJsonString(source, findObjectFieldValue(source, objectBounds.start, objectBounds.end, 'type'))
|
||||
if (blockType === 'tool_use') {
|
||||
const name = readJsonString(source, findObjectFieldValue(source, objectBounds.start, objectBounds.end, 'name')) ?? ''
|
||||
const id = readJsonString(source, findObjectFieldValue(source, objectBounds.start, objectBounds.end, 'id')) ?? ''
|
||||
const inputBounds = findObjectFieldValue(source, objectBounds.start, objectBounds.end, 'input')
|
||||
const input: Record<string, unknown> = {}
|
||||
if (inputBounds?.kind === 'object') {
|
||||
if (name === 'Skill') {
|
||||
const skill = readJsonString(source, findObjectFieldValue(source, inputBounds.start, inputBounds.end, 'skill'), 200)
|
||||
const skillName = readJsonString(source, findObjectFieldValue(source, inputBounds.start, inputBounds.end, 'name'), 200)
|
||||
if (skill !== undefined) input['skill'] = skill
|
||||
if (skillName !== undefined) input['name'] = skillName
|
||||
} else if (name === 'Read' || name === 'FileReadTool') {
|
||||
const filePath = readJsonString(source, findObjectFieldValue(source, inputBounds.start, inputBounds.end, 'file_path'), BASH_COMMAND_CAP)
|
||||
if (filePath !== undefined) input['file_path'] = filePath
|
||||
} else if (name === 'Agent' || name === 'Task') {
|
||||
const subagentType = readJsonString(source, findObjectFieldValue(source, inputBounds.start, inputBounds.end, 'subagent_type'), 200)
|
||||
if (subagentType !== undefined) input['subagent_type'] = subagentType
|
||||
} else if (BASH_TOOLS.has(name)) {
|
||||
const command = readJsonString(source, findObjectFieldValue(source, inputBounds.start, inputBounds.end, 'command'), BASH_COMMAND_CAP)
|
||||
if (command !== undefined) input['command'] = command
|
||||
}
|
||||
}
|
||||
tools.push({ type: 'tool_use', id, name, input })
|
||||
}
|
||||
i = objectEnd + 1
|
||||
}
|
||||
return tools
|
||||
}
|
||||
|
||||
function extractLargeUserText(source: string, contentBounds: JsonValueBounds | null): string | undefined {
|
||||
if (!contentBounds) return undefined
|
||||
if (contentBounds.kind === 'string') return readJsonString(source, contentBounds, USER_TEXT_CAP)
|
||||
if (contentBounds.kind !== 'array') return undefined
|
||||
|
||||
let text = ''
|
||||
let i = contentBounds.start + 1
|
||||
while (i < contentBounds.end - 1 && text.length < USER_TEXT_CAP) {
|
||||
while (i < contentBounds.end && /\s/.test(source[i]!)) i++
|
||||
if (source.charCodeAt(i) === 0x2c) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (source.charCodeAt(i) !== 0x7b) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
const objectEnd = findJsonContainerEnd(source, i, 0x7b, 0x7d, contentBounds.end)
|
||||
if (objectEnd === -1) break
|
||||
const objectBounds = { start: i, end: objectEnd + 1, kind: 'object' as const }
|
||||
const type = readJsonString(source, findObjectFieldValue(source, objectBounds.start, objectBounds.end, 'type'))
|
||||
if (type === 'text' || type === 'input_text') {
|
||||
const part = readJsonString(
|
||||
source,
|
||||
findObjectFieldValue(source, objectBounds.start, objectBounds.end, 'text'),
|
||||
USER_TEXT_CAP - text.length,
|
||||
)
|
||||
if (part) text += (text ? ' ' : '') + part
|
||||
}
|
||||
i = objectEnd + 1
|
||||
}
|
||||
return text || undefined
|
||||
}
|
||||
|
||||
function extractLargeAddedNames(source: string, attachmentBounds: JsonValueBounds | null): string[] {
|
||||
if (!attachmentBounds || attachmentBounds.kind !== 'object') return []
|
||||
const attachmentType = readJsonString(source, findObjectFieldValue(source, attachmentBounds.start, attachmentBounds.end, 'type'))
|
||||
if (attachmentType !== 'deferred_tools_delta') return []
|
||||
const addedNames = findObjectFieldValue(source, attachmentBounds.start, attachmentBounds.end, 'addedNames')
|
||||
if (!addedNames || addedNames.kind !== 'array') return []
|
||||
const names: string[] = []
|
||||
let i = addedNames.start + 1
|
||||
while (i < addedNames.end - 1 && names.length < MAX_ADDED_NAMES) {
|
||||
while (i < addedNames.end && /\s/.test(source[i]!)) i++
|
||||
if (source.charCodeAt(i) === 0x2c) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (source.charCodeAt(i) !== 0x22) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
const end = findJsonStringEnd(source, i, addedNames.end)
|
||||
if (end === -1) break
|
||||
const name = readJsonString(source, { start: i, end: end + 1, kind: 'string' }, 500)
|
||||
if (name) names.push(name)
|
||||
i = end + 1
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
function parseLargeJsonlLine(line: string): JournalEntry | null {
|
||||
const rootEnd = findJsonContainerEnd(line, 0, 0x7b, 0x7d)
|
||||
if (rootEnd === -1) return null
|
||||
const rootStart = 0
|
||||
const rootLimit = rootEnd + 1
|
||||
const type = readJsonString(line, findObjectFieldValue(line, rootStart, rootLimit, 'type'))
|
||||
if (!type) return null
|
||||
|
||||
const entry: JournalEntry = { type }
|
||||
const timestamp = readJsonString(line, findObjectFieldValue(line, rootStart, rootLimit, 'timestamp'))
|
||||
const sessionId = readJsonString(line, findObjectFieldValue(line, rootStart, rootLimit, 'sessionId'))
|
||||
const cwd = readJsonString(line, findObjectFieldValue(line, rootStart, rootLimit, 'cwd'))
|
||||
if (timestamp !== undefined) entry.timestamp = timestamp
|
||||
if (sessionId !== undefined) entry.sessionId = sessionId
|
||||
if (cwd !== undefined) entry.cwd = cwd
|
||||
const addedNames = extractLargeAddedNames(line, findObjectFieldValue(line, rootStart, rootLimit, 'attachment'))
|
||||
if (addedNames.length > 0) {
|
||||
;(entry as Record<string, unknown>)['attachment'] = { type: 'deferred_tools_delta', addedNames }
|
||||
}
|
||||
|
||||
if (type === 'user') {
|
||||
const message = findObjectFieldValue(line, rootStart, rootLimit, 'message')
|
||||
if (message?.kind === 'object') {
|
||||
const content = findObjectFieldValue(line, message.start, message.end, 'content')
|
||||
const text = extractLargeUserText(line, content)
|
||||
if (text !== undefined) entry.message = { role: 'user', content: text }
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
if (type !== 'assistant') return entry
|
||||
const message = findObjectFieldValue(line, rootStart, rootLimit, 'message')
|
||||
if (message?.kind !== 'object') return entry
|
||||
const model = readJsonString(line, findObjectFieldValue(line, message.start, message.end, 'model'))
|
||||
const usageBounds = findObjectFieldValue(line, message.start, message.end, 'usage')
|
||||
if (!model || usageBounds?.kind !== 'object') return entry
|
||||
const id = readJsonString(line, findObjectFieldValue(line, message.start, message.end, 'id'))
|
||||
const contentBounds = findObjectFieldValue(line, message.start, message.end, 'content')
|
||||
|
||||
entry.message = {
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
model,
|
||||
...(id !== undefined ? { id } : {}),
|
||||
content: extractLargeToolBlocks(line, contentBounds),
|
||||
usage: parseLargeUsage(line, usageBounds),
|
||||
}
|
||||
|
||||
return entry
|
||||
}
|
||||
|
||||
type BufferJsonValueBounds = {
|
||||
start: number
|
||||
end: number
|
||||
kind: 'string' | 'object' | 'array' | 'scalar'
|
||||
}
|
||||
|
||||
function isJsonWhitespaceByte(ch: number | undefined): boolean {
|
||||
return ch === 0x20 || ch === 0x0a || ch === 0x0d || ch === 0x09
|
||||
}
|
||||
|
||||
function findJsonStringEndBuffer(source: Buffer, start: number, limit = source.length): number {
|
||||
for (let i = start + 1; i < limit; i++) {
|
||||
const ch = source[i]
|
||||
if (ch === 0x5c) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (ch === 0x22) return i
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
function findJsonContainerEndBuffer(source: Buffer, start: number, open: number, close: number, limit = source.length): number {
|
||||
let depth = 0
|
||||
let inString = false
|
||||
for (let i = start; i < limit; i++) {
|
||||
const ch = source[i]
|
||||
if (inString) {
|
||||
if (ch === 0x5c) {
|
||||
i++
|
||||
} else if (ch === 0x22) {
|
||||
inString = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
if (ch === 0x22) {
|
||||
inString = true
|
||||
} else if (ch === open) {
|
||||
depth++
|
||||
} else if (ch === close) {
|
||||
depth--
|
||||
if (depth === 0) return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
function findJsonValueBoundsBuffer(source: Buffer, start: number, limit = source.length): BufferJsonValueBounds | null {
|
||||
let i = start
|
||||
while (i < limit && isJsonWhitespaceByte(source[i])) i++
|
||||
if (i >= limit) return null
|
||||
const ch = source[i]
|
||||
if (ch === 0x22) {
|
||||
const end = findJsonStringEndBuffer(source, i, limit)
|
||||
return end === -1 ? null : { start: i, end: end + 1, kind: 'string' }
|
||||
}
|
||||
if (ch === 0x7b) {
|
||||
const end = findJsonContainerEndBuffer(source, i, 0x7b, 0x7d, limit)
|
||||
return end === -1 ? null : { start: i, end: end + 1, kind: 'object' }
|
||||
}
|
||||
if (ch === 0x5b) {
|
||||
const end = findJsonContainerEndBuffer(source, i, 0x5b, 0x5d, limit)
|
||||
return end === -1 ? null : { start: i, end: end + 1, kind: 'array' }
|
||||
}
|
||||
let end = i
|
||||
while (end < limit) {
|
||||
const c = source[end]
|
||||
if (c === 0x2c || c === 0x7d || c === 0x5d || isJsonWhitespaceByte(c)) break
|
||||
end++
|
||||
}
|
||||
return { start: i, end, kind: 'scalar' }
|
||||
}
|
||||
|
||||
function bufferKeyEquals(source: Buffer, keyStart: number, keyEnd: number, field: string): boolean {
|
||||
if (keyEnd - keyStart !== field.length) return false
|
||||
return source.subarray(keyStart, keyEnd).equals(Buffer.from(field))
|
||||
}
|
||||
|
||||
function findObjectFieldValueBuffer(source: Buffer, objectStart: number, objectEnd: number, field: string): BufferJsonValueBounds | null {
|
||||
if (source[objectStart] !== 0x7b) return null
|
||||
let i = objectStart + 1
|
||||
while (i < objectEnd - 1) {
|
||||
while (i < objectEnd && isJsonWhitespaceByte(source[i])) i++
|
||||
if (source[i] === 0x2c) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (source[i] !== 0x22) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
const keyEnd = findJsonStringEndBuffer(source, i, objectEnd)
|
||||
if (keyEnd === -1) return null
|
||||
const keyStart = i + 1
|
||||
i = keyEnd + 1
|
||||
while (i < objectEnd && isJsonWhitespaceByte(source[i])) i++
|
||||
if (source[i] !== 0x3a) continue
|
||||
const value = findJsonValueBoundsBuffer(source, i + 1, objectEnd)
|
||||
if (!value) return null
|
||||
if (bufferKeyEquals(source, keyStart, keyEnd, field)) return value
|
||||
i = value.end
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function appendBufferJsonSegment(source: Buffer, start: number, end: number, current: string, cap: number): string {
|
||||
if (start >= end || current.length >= cap) return current
|
||||
const remaining = cap - current.length
|
||||
const cappedEnd = Number.isFinite(cap) ? Math.min(end, start + remaining * 4) : end
|
||||
return current + source.subarray(start, cappedEnd).toString('utf-8').slice(0, remaining)
|
||||
}
|
||||
|
||||
function readJsonStringBuffer(source: Buffer, bounds: BufferJsonValueBounds | null, cap = Number.POSITIVE_INFINITY): string | undefined {
|
||||
if (!bounds || bounds.kind !== 'string') return undefined
|
||||
let out = ''
|
||||
let segmentStart = bounds.start + 1
|
||||
for (let i = bounds.start + 1; i < bounds.end - 1 && out.length < cap; i++) {
|
||||
const ch = source[i]
|
||||
if (ch !== 0x5c) continue
|
||||
|
||||
out = appendBufferJsonSegment(source, segmentStart, i, out, cap)
|
||||
if (out.length >= cap) break
|
||||
const next = source[++i]
|
||||
if (next === undefined) break
|
||||
if (next === 0x6e) out += '\n'
|
||||
else if (next === 0x72) out += '\r'
|
||||
else if (next === 0x74) out += '\t'
|
||||
else if (next === 0x62) out += '\b'
|
||||
else if (next === 0x66) out += '\f'
|
||||
else if (next === 0x75 && i + 4 < bounds.end) {
|
||||
const hex = source.subarray(i + 1, i + 5).toString('ascii')
|
||||
const code = Number.parseInt(hex, 16)
|
||||
if (Number.isFinite(code)) out += String.fromCharCode(code)
|
||||
i += 4
|
||||
} else {
|
||||
out += String.fromCharCode(next)
|
||||
}
|
||||
segmentStart = i + 1
|
||||
}
|
||||
return appendBufferJsonSegment(source, segmentStart, bounds.end - 1, out, cap)
|
||||
}
|
||||
|
||||
function readJsonNumberFieldBuffer(source: Buffer, objectBounds: BufferJsonValueBounds | null, field: string): number | undefined {
|
||||
if (!objectBounds || objectBounds.kind !== 'object') return undefined
|
||||
const bounds = findObjectFieldValueBuffer(source, objectBounds.start, objectBounds.end, field)
|
||||
if (!bounds) return undefined
|
||||
const value = Number(source.subarray(bounds.start, bounds.end).toString('ascii'))
|
||||
return Number.isFinite(value) ? value : undefined
|
||||
}
|
||||
|
||||
function parseLargeUsageBuffer(source: Buffer, usageBounds: BufferJsonValueBounds | null) {
|
||||
const usage: AssistantMessageContent['usage'] = {
|
||||
input_tokens: readJsonNumberFieldBuffer(source, usageBounds, 'input_tokens') ?? 0,
|
||||
output_tokens: readJsonNumberFieldBuffer(source, usageBounds, 'output_tokens') ?? 0,
|
||||
cache_creation_input_tokens: readJsonNumberFieldBuffer(source, usageBounds, 'cache_creation_input_tokens'),
|
||||
cache_read_input_tokens: readJsonNumberFieldBuffer(source, usageBounds, 'cache_read_input_tokens'),
|
||||
}
|
||||
|
||||
if (usageBounds?.kind === 'object') {
|
||||
const cacheCreation = findObjectFieldValueBuffer(source, usageBounds.start, usageBounds.end, 'cache_creation')
|
||||
const ephemeral5m = readJsonNumberFieldBuffer(source, cacheCreation, 'ephemeral_5m_input_tokens')
|
||||
const ephemeral1h = readJsonNumberFieldBuffer(source, cacheCreation, 'ephemeral_1h_input_tokens')
|
||||
if (ephemeral5m !== undefined || ephemeral1h !== undefined) {
|
||||
;(usage as AssistantMessageContent['usage']).cache_creation = {
|
||||
...(ephemeral5m !== undefined ? { ephemeral_5m_input_tokens: ephemeral5m } : {}),
|
||||
...(ephemeral1h !== undefined ? { ephemeral_1h_input_tokens: ephemeral1h } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
const serverToolUse = findObjectFieldValueBuffer(source, usageBounds.start, usageBounds.end, 'server_tool_use')
|
||||
const webSearch = readJsonNumberFieldBuffer(source, serverToolUse, 'web_search_requests')
|
||||
const webFetch = readJsonNumberFieldBuffer(source, serverToolUse, 'web_fetch_requests')
|
||||
if (webSearch !== undefined || webFetch !== undefined) {
|
||||
;(usage as AssistantMessageContent['usage']).server_tool_use = {
|
||||
...(webSearch !== undefined ? { web_search_requests: webSearch } : {}),
|
||||
...(webFetch !== undefined ? { web_fetch_requests: webFetch } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
const speed = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, usageBounds.start, usageBounds.end, 'speed'))
|
||||
if (speed === 'standard' || speed === 'fast') usage.speed = speed
|
||||
}
|
||||
|
||||
return usage
|
||||
}
|
||||
|
||||
function extractLargeToolBlocksBuffer(source: Buffer, contentBounds: BufferJsonValueBounds | null): ToolUseBlock[] {
|
||||
if (!contentBounds || contentBounds.kind !== 'array') return []
|
||||
const tools: ToolUseBlock[] = []
|
||||
let i = contentBounds.start + 1
|
||||
while (i < contentBounds.end - 1 && tools.length < MAX_TOOL_BLOCKS) {
|
||||
while (i < contentBounds.end && isJsonWhitespaceByte(source[i])) i++
|
||||
if (source[i] === 0x2c) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (source[i] !== 0x7b) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
const objectEnd = findJsonContainerEndBuffer(source, i, 0x7b, 0x7d, contentBounds.end)
|
||||
if (objectEnd === -1) break
|
||||
const objectBounds = { start: i, end: objectEnd + 1, kind: 'object' as const }
|
||||
const blockType = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, objectBounds.start, objectBounds.end, 'type'))
|
||||
if (blockType === 'tool_use') {
|
||||
const name = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, objectBounds.start, objectBounds.end, 'name')) ?? ''
|
||||
const id = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, objectBounds.start, objectBounds.end, 'id')) ?? ''
|
||||
const inputBounds = findObjectFieldValueBuffer(source, objectBounds.start, objectBounds.end, 'input')
|
||||
const input: Record<string, unknown> = {}
|
||||
if (inputBounds?.kind === 'object') {
|
||||
if (name === 'Skill') {
|
||||
const skill = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, inputBounds.start, inputBounds.end, 'skill'), 200)
|
||||
const skillName = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, inputBounds.start, inputBounds.end, 'name'), 200)
|
||||
if (skill !== undefined) input['skill'] = skill
|
||||
if (skillName !== undefined) input['name'] = skillName
|
||||
} else if (name === 'Read' || name === 'FileReadTool') {
|
||||
const filePath = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, inputBounds.start, inputBounds.end, 'file_path'), BASH_COMMAND_CAP)
|
||||
if (filePath !== undefined) input['file_path'] = filePath
|
||||
} else if (name === 'Agent' || name === 'Task') {
|
||||
const subagentType = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, inputBounds.start, inputBounds.end, 'subagent_type'), 200)
|
||||
if (subagentType !== undefined) input['subagent_type'] = subagentType
|
||||
} else if (BASH_TOOLS.has(name)) {
|
||||
const command = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, inputBounds.start, inputBounds.end, 'command'), BASH_COMMAND_CAP)
|
||||
if (command !== undefined) input['command'] = command
|
||||
}
|
||||
}
|
||||
tools.push({ type: 'tool_use', id, name, input })
|
||||
}
|
||||
i = objectEnd + 1
|
||||
}
|
||||
return tools
|
||||
}
|
||||
|
||||
function extractLargeUserTextBuffer(source: Buffer, contentBounds: BufferJsonValueBounds | null): string | undefined {
|
||||
if (!contentBounds) return undefined
|
||||
if (contentBounds.kind === 'string') return readJsonStringBuffer(source, contentBounds, USER_TEXT_CAP)
|
||||
if (contentBounds.kind !== 'array') return undefined
|
||||
|
||||
let text = ''
|
||||
let i = contentBounds.start + 1
|
||||
while (i < contentBounds.end - 1 && text.length < USER_TEXT_CAP) {
|
||||
while (i < contentBounds.end && isJsonWhitespaceByte(source[i])) i++
|
||||
if (source[i] === 0x2c) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (source[i] !== 0x7b) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
const objectEnd = findJsonContainerEndBuffer(source, i, 0x7b, 0x7d, contentBounds.end)
|
||||
if (objectEnd === -1) break
|
||||
const objectBounds = { start: i, end: objectEnd + 1, kind: 'object' as const }
|
||||
const type = readJsonStringBuffer(source, findObjectFieldValueBuffer(source, objectBounds.start, objectBounds.end, 'type'))
|
||||
if (type === 'text' || type === 'input_text') {
|
||||
const part = readJsonStringBuffer(
|
||||
source,
|
||||
findObjectFieldValueBuffer(source, objectBounds.start, objectBounds.end, 'text'),
|
||||
USER_TEXT_CAP - text.length,
|
||||
)
|
||||
if (part) text += (text ? ' ' : '') + part
|
||||
}
|
||||
i = objectEnd + 1
|
||||
}
|
||||
return text || undefined
|
||||
}
|
||||
|
||||
function extractLargeAddedNamesBuffer(source: Buffer, attachmentBounds: BufferJsonValueBounds | null): string[] {
|
||||
if (!attachmentBounds || attachmentBounds.kind !== 'object') return []
|
||||
const attachmentType = readJsonStringBuffer(
|
||||
source,
|
||||
findObjectFieldValueBuffer(source, attachmentBounds.start, attachmentBounds.end, 'type'),
|
||||
)
|
||||
if (attachmentType !== 'deferred_tools_delta') return []
|
||||
const addedNames = findObjectFieldValueBuffer(source, attachmentBounds.start, attachmentBounds.end, 'addedNames')
|
||||
if (!addedNames || addedNames.kind !== 'array') return []
|
||||
const names: string[] = []
|
||||
let i = addedNames.start + 1
|
||||
while (i < addedNames.end - 1 && names.length < MAX_ADDED_NAMES) {
|
||||
while (i < addedNames.end && isJsonWhitespaceByte(source[i])) i++
|
||||
if (source[i] === 0x2c) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (source[i] !== 0x22) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
const end = findJsonStringEndBuffer(source, i, addedNames.end)
|
||||
if (end === -1) break
|
||||
const name = readJsonStringBuffer(source, { start: i, end: end + 1, kind: 'string' }, 500)
|
||||
if (name) names.push(name)
|
||||
i = end + 1
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
function parseLargeJsonlBuffer(line: Buffer): JournalEntry | null {
|
||||
let rootStart = 0
|
||||
while (rootStart < line.length && isJsonWhitespaceByte(line[rootStart])) rootStart++
|
||||
if (line[rootStart] !== 0x7b) return null
|
||||
const rootEnd = findJsonContainerEndBuffer(line, rootStart, 0x7b, 0x7d)
|
||||
if (rootEnd === -1) return null
|
||||
const rootLimit = rootEnd + 1
|
||||
const type = readJsonStringBuffer(line, findObjectFieldValueBuffer(line, rootStart, rootLimit, 'type'))
|
||||
if (!type) return null
|
||||
|
||||
const entry: JournalEntry = { type }
|
||||
const timestamp = readJsonStringBuffer(line, findObjectFieldValueBuffer(line, rootStart, rootLimit, 'timestamp'))
|
||||
const sessionId = readJsonStringBuffer(line, findObjectFieldValueBuffer(line, rootStart, rootLimit, 'sessionId'))
|
||||
const cwd = readJsonStringBuffer(line, findObjectFieldValueBuffer(line, rootStart, rootLimit, 'cwd'))
|
||||
if (timestamp !== undefined) entry.timestamp = timestamp
|
||||
if (sessionId !== undefined) entry.sessionId = sessionId
|
||||
if (cwd !== undefined) entry.cwd = cwd
|
||||
const addedNames = extractLargeAddedNamesBuffer(line, findObjectFieldValueBuffer(line, rootStart, rootLimit, 'attachment'))
|
||||
if (addedNames.length > 0) {
|
||||
;(entry as Record<string, unknown>)['attachment'] = { type: 'deferred_tools_delta', addedNames }
|
||||
}
|
||||
|
||||
if (type === 'user') {
|
||||
const message = findObjectFieldValueBuffer(line, rootStart, rootLimit, 'message')
|
||||
if (message?.kind === 'object') {
|
||||
const content = findObjectFieldValueBuffer(line, message.start, message.end, 'content')
|
||||
const text = extractLargeUserTextBuffer(line, content)
|
||||
if (text !== undefined) entry.message = { role: 'user', content: text }
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
if (type !== 'assistant') return entry
|
||||
const message = findObjectFieldValueBuffer(line, rootStart, rootLimit, 'message')
|
||||
if (message?.kind !== 'object') return entry
|
||||
const model = readJsonStringBuffer(line, findObjectFieldValueBuffer(line, message.start, message.end, 'model'))
|
||||
const usageBounds = findObjectFieldValueBuffer(line, message.start, message.end, 'usage')
|
||||
if (!model || usageBounds?.kind !== 'object') return entry
|
||||
const id = readJsonStringBuffer(line, findObjectFieldValueBuffer(line, message.start, message.end, 'id'))
|
||||
const contentBounds = findObjectFieldValueBuffer(line, message.start, message.end, 'content')
|
||||
|
||||
entry.message = {
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
model,
|
||||
...(id !== undefined ? { id } : {}),
|
||||
content: extractLargeToolBlocksBuffer(line, contentBounds),
|
||||
usage: parseLargeUsageBuffer(line, usageBounds),
|
||||
}
|
||||
|
||||
return entry
|
||||
}
|
||||
|
||||
function getTopLevelRawJsonStringField(head: string, field: string): string | null {
|
||||
let i = 0
|
||||
while (i < head.length && /\s/.test(head[i]!)) i++
|
||||
if (head.charCodeAt(i) !== 0x7b) return null
|
||||
i++
|
||||
while (i < head.length) {
|
||||
while (i < head.length && /\s/.test(head[i]!)) i++
|
||||
if (head.charCodeAt(i) === 0x2c) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (head.charCodeAt(i) === 0x7d) return null
|
||||
if (head.charCodeAt(i) !== 0x22) return null
|
||||
const keyEnd = findJsonStringEnd(head, i)
|
||||
if (keyEnd === -1) return null
|
||||
const key = head.slice(i + 1, keyEnd)
|
||||
i = keyEnd + 1
|
||||
while (i < head.length && /\s/.test(head[i]!)) i++
|
||||
if (head.charCodeAt(i) !== 0x3a) return null
|
||||
const value = findJsonValueBounds(head, i + 1)
|
||||
if (!value) return null
|
||||
if (key === field) return readJsonString(head, value) ?? null
|
||||
i = value.end
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
export function shouldSkipLine(line: string, threshold: string): boolean {
|
||||
const head = line.length > RAW_HEAD_BYTES ? line.slice(0, RAW_HEAD_BYTES) : line
|
||||
const type = getTopLevelRawJsonStringField(head, 'type')
|
||||
if (type !== 'user' && type !== 'assistant') return false
|
||||
const ts = getTopLevelRawJsonStringField(head, 'timestamp')
|
||||
if (!ts || ts.length < 10) return false
|
||||
return ts < threshold
|
||||
}
|
||||
|
||||
const USER_TEXT_CAP = 2000
|
||||
const BASH_COMMAND_CAP = 2000
|
||||
const MAX_TOOL_BLOCKS = 500
|
||||
|
|
@ -100,6 +826,12 @@ export function compactEntry(raw: JournalEntry): JournalEntry {
|
|||
const ri = (tb.input ?? {}) as Record<string, unknown>
|
||||
if (typeof ri['skill'] === 'string') input['skill'] = (ri['skill'] as string).slice(0, 200)
|
||||
if (typeof ri['name'] === 'string') input['name'] = (ri['name'] as string).slice(0, 200)
|
||||
} else if (tb.name === 'Read' || tb.name === 'FileReadTool') {
|
||||
const ri = (tb.input ?? {}) as Record<string, unknown>
|
||||
if (typeof ri['file_path'] === 'string') input['file_path'] = (ri['file_path'] as string).slice(0, BASH_COMMAND_CAP)
|
||||
} else if (tb.name === 'Agent' || tb.name === 'Task') {
|
||||
const ri = (tb.input ?? {}) as Record<string, unknown>
|
||||
if (typeof ri['subagent_type'] === 'string') input['subagent_type'] = (ri['subagent_type'] as string).slice(0, 200)
|
||||
} else if (BASH_TOOLS.has(tb.name)) {
|
||||
const ri = (tb.input ?? {}) as Record<string, unknown>
|
||||
if (typeof ri['command'] === 'string') {
|
||||
|
|
@ -518,7 +1250,18 @@ async function parseSessionFile(
|
|||
const entries: JournalEntry[] = []
|
||||
let hasLines = false
|
||||
|
||||
for await (const line of readSessionLines(filePath)) {
|
||||
// When a dateRange is given, skip user/assistant lines whose timestamp
|
||||
// is older than range.start - 24h without calling JSON.parse. Huge lines
|
||||
// that cannot be skipped are yielded as Buffers and compact-parsed without
|
||||
// converting the whole line into a V8 string.
|
||||
const earlySkipThreshold = dateRange
|
||||
? new Date(dateRange.start.getTime() - 86_400_000).toISOString()
|
||||
: null
|
||||
const skipFn = earlySkipThreshold
|
||||
? (head: string) => shouldSkipLine(head, earlySkipThreshold)
|
||||
: undefined
|
||||
|
||||
for await (const line of readSessionLines(filePath, skipFn, { largeLineAsBuffer: true })) {
|
||||
hasLines = true
|
||||
const entry = parseJsonlLine(line)
|
||||
if (entry) entries.push(compactEntry(entry))
|
||||
|
|
@ -825,6 +1568,35 @@ export function filterProjectsByName(
|
|||
return result
|
||||
}
|
||||
|
||||
function turnIsInDateRange(turn: ClassifiedTurn, dateRange: DateRange): boolean {
|
||||
if (turn.assistantCalls.length === 0) return false
|
||||
const firstCallTs = turn.assistantCalls[0]!.timestamp
|
||||
if (!firstCallTs) return false
|
||||
const ts = new Date(firstCallTs)
|
||||
return ts >= dateRange.start && ts <= dateRange.end
|
||||
}
|
||||
|
||||
export function filterProjectsByDateRange(projects: ProjectSummary[], dateRange: DateRange): ProjectSummary[] {
|
||||
const filtered: ProjectSummary[] = []
|
||||
for (const project of projects) {
|
||||
const sessions: SessionSummary[] = []
|
||||
for (const session of project.sessions) {
|
||||
const turns = session.turns.filter(turn => turnIsInDateRange(turn, dateRange))
|
||||
if (turns.length === 0) continue
|
||||
sessions.push(buildSessionSummary(session.sessionId, session.project, turns, session.mcpInventory))
|
||||
}
|
||||
if (sessions.length === 0) continue
|
||||
filtered.push({
|
||||
project: project.project,
|
||||
projectPath: project.projectPath,
|
||||
sessions,
|
||||
totalCostUSD: sessions.reduce((s, sess) => s + sess.totalCostUSD, 0),
|
||||
totalApiCalls: sessions.reduce((s, sess) => s + sess.apiCalls, 0),
|
||||
})
|
||||
}
|
||||
return filtered.sort((a, b) => b.totalCostUSD - a.totalCostUSD)
|
||||
}
|
||||
|
||||
export async function parseAllSessions(dateRange?: DateRange, providerFilter?: string): Promise<ProjectSummary[]> {
|
||||
const key = cacheKey(dateRange, providerFilter)
|
||||
const cached = sessionCache.get(key)
|
||||
|
|
|
|||
|
|
@ -65,6 +65,8 @@ type CodexTokenUsage = {
|
|||
}
|
||||
|
||||
const CHARS_PER_TOKEN = 4
|
||||
const RAW_HEAD_BYTES = 64 * 1024
|
||||
const LARGE_TEXT_CAP = 2000
|
||||
|
||||
function getCodexDir(override?: string): string {
|
||||
return override ?? process.env['CODEX_HOME'] ?? join(homedir(), '.codex')
|
||||
|
|
@ -126,6 +128,116 @@ async function isValidCodexSession(filePath: string): Promise<{ valid: boolean;
|
|||
return { valid, meta: valid ? entry : undefined }
|
||||
}
|
||||
|
||||
function getRawJsonStringField(head: string, field: string): string | undefined {
|
||||
const re = new RegExp(`"${field}"\\s*:\\s*"((?:\\\\.|[^"\\\\])*)"`)
|
||||
const match = re.exec(head)
|
||||
if (!match) return undefined
|
||||
try {
|
||||
return JSON.parse(`"${match[1]}"`) as string
|
||||
} catch {
|
||||
return match[1]
|
||||
}
|
||||
}
|
||||
|
||||
function payloadHead(head: string): string {
|
||||
const idx = head.indexOf('"payload"')
|
||||
return idx === -1 ? head : head.slice(idx)
|
||||
}
|
||||
|
||||
function countJsonStringBytes(source: Buffer, valueStart: number): number {
|
||||
let count = 0
|
||||
for (let i = valueStart; i < source.length; i++) {
|
||||
const ch = source[i]
|
||||
if (ch === 0x5c) {
|
||||
i++
|
||||
count++
|
||||
continue
|
||||
}
|
||||
if (ch === 0x22) return count
|
||||
count++
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
function extractFirstJsonText(source: Buffer, cap = LARGE_TEXT_CAP): string {
|
||||
const key = Buffer.from('"text"')
|
||||
const idx = source.indexOf(key)
|
||||
if (idx === -1) return ''
|
||||
const colon = source.indexOf(0x3a, idx + key.length)
|
||||
if (colon === -1) return ''
|
||||
const qStart = source.indexOf(0x22, colon + 1)
|
||||
if (qStart === -1) return ''
|
||||
const chunks: number[] = []
|
||||
for (let i = qStart + 1; i < source.length && chunks.length < cap; i++) {
|
||||
const ch = source[i]
|
||||
if (ch === 0x5c) {
|
||||
const next = source[++i]
|
||||
if (next === 0x6e) chunks.push(0x0a)
|
||||
else if (next === 0x72) chunks.push(0x0d)
|
||||
else if (next === 0x74) chunks.push(0x09)
|
||||
else if (next !== undefined) chunks.push(next)
|
||||
continue
|
||||
}
|
||||
if (ch === 0x22) break
|
||||
chunks.push(ch)
|
||||
}
|
||||
return Buffer.from(chunks).toString('utf-8')
|
||||
}
|
||||
|
||||
function countFirstJsonText(source: Buffer): number {
|
||||
const key = Buffer.from('"text"')
|
||||
const idx = source.indexOf(key)
|
||||
if (idx === -1) return 0
|
||||
const colon = source.indexOf(0x3a, idx + key.length)
|
||||
if (colon === -1) return 0
|
||||
const qStart = source.indexOf(0x22, colon + 1)
|
||||
if (qStart === -1) return 0
|
||||
return countJsonStringBytes(source, qStart + 1)
|
||||
}
|
||||
|
||||
function parseCodexLine(line: string | Buffer): CodexEntry | null {
|
||||
if (typeof line === 'string') {
|
||||
const trimmed = line.trim()
|
||||
if (!trimmed) return null
|
||||
try {
|
||||
return JSON.parse(trimmed) as CodexEntry
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
if (line.length === 0) return null
|
||||
const head = line.subarray(0, RAW_HEAD_BYTES).toString('utf-8')
|
||||
const type = getRawJsonStringField(head, 'type')
|
||||
if (!type) return null
|
||||
const pHead = payloadHead(head)
|
||||
const payloadType = getRawJsonStringField(pHead, 'type')
|
||||
const role = getRawJsonStringField(pHead, 'role')
|
||||
|
||||
const entry: CodexEntry = {
|
||||
type,
|
||||
timestamp: getRawJsonStringField(head, 'timestamp'),
|
||||
payload: {
|
||||
type: payloadType,
|
||||
role,
|
||||
cwd: getRawJsonStringField(pHead, 'cwd'),
|
||||
model_provider: getRawJsonStringField(pHead, 'model_provider'),
|
||||
originator: getRawJsonStringField(pHead, 'originator'),
|
||||
session_id: getRawJsonStringField(pHead, 'session_id'),
|
||||
model: getRawJsonStringField(pHead, 'model'),
|
||||
name: getRawJsonStringField(pHead, 'name'),
|
||||
},
|
||||
}
|
||||
|
||||
if (type === 'response_item' && payloadType === 'message' && role === 'user') {
|
||||
entry.payload!.content = [{ type: 'input_text', text: extractFirstJsonText(line) }]
|
||||
} else if (type === 'response_item' && payloadType === 'message' && role === 'assistant') {
|
||||
entry.payload!.content = [{ type: 'output_text', text: 'x'.repeat(Math.min(countFirstJsonText(line), LARGE_TEXT_CAP)) }]
|
||||
}
|
||||
|
||||
return entry
|
||||
}
|
||||
|
||||
async function discoverSessionsInDir(codexDir: string): Promise<SessionSource[]> {
|
||||
const sessionsDir = join(codexDir, 'sessions')
|
||||
const sources: SessionSource[] = []
|
||||
|
|
@ -224,18 +336,12 @@ function createParser(source: SessionSource, seenKeys: Set<string>): SessionPars
|
|||
// Stream the session file line by line. Heavy Codex sessions can exceed
|
||||
// 250 MB on disk; reading the entire file into a string would either hit
|
||||
// the readSessionFile cap or push V8 toward its 512 MB string limit
|
||||
// after split('\n'). readSessionLines streams via readline so memory
|
||||
// stays bounded to the longest line.
|
||||
for await (const rawLine of readSessionLines(source.path)) {
|
||||
// after split('\n'). readSessionLines streams raw buffers and hands
|
||||
// huge lines to the compact parser without full string conversion.
|
||||
for await (const rawLine of readSessionLines(source.path, undefined, { largeLineAsBuffer: true })) {
|
||||
sawAnyLine = true
|
||||
const line = rawLine.trim()
|
||||
if (!line) continue
|
||||
let entry: CodexEntry
|
||||
try {
|
||||
entry = JSON.parse(line) as CodexEntry
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
const entry = parseCodexLine(rawLine)
|
||||
if (!entry) continue
|
||||
|
||||
if (entry.type === 'session_meta') {
|
||||
sessionId = entry.payload?.session_id ?? basename(source.path, '.jsonl')
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import { join } from 'path'
|
|||
|
||||
import {
|
||||
MAX_SESSION_FILE_BYTES,
|
||||
STREAM_THRESHOLD_BYTES,
|
||||
readSessionFile,
|
||||
readSessionLines,
|
||||
} from '../src/fs-utils.js'
|
||||
|
|
@ -34,11 +33,12 @@ describe('readSessionFile', () => {
|
|||
expect(await readSessionFile(p)).toBe('hello\nworld\n')
|
||||
})
|
||||
|
||||
it('returns content for files at the stream threshold via stream path', async () => {
|
||||
const p = await tmpPath(Buffer.alloc(STREAM_THRESHOLD_BYTES, 'a'))
|
||||
it('returns content for large files under the full-file cap', async () => {
|
||||
const size = 8 * 1024 * 1024
|
||||
const p = await tmpPath(Buffer.alloc(size, 'a'))
|
||||
const got = await readSessionFile(p)
|
||||
expect(got).not.toBeNull()
|
||||
expect(got!.length).toBe(STREAM_THRESHOLD_BYTES)
|
||||
expect(got!.length).toBe(size)
|
||||
})
|
||||
|
||||
it('returns null and skips files over the cap', async () => {
|
||||
|
|
@ -88,6 +88,28 @@ describe('readSessionLines', () => {
|
|||
expect(lines).toEqual(['line1', 'line2', 'line3'])
|
||||
})
|
||||
|
||||
it('skips old large lines before materializing the full line', async () => {
|
||||
const oldLine = `{"type":"assistant","timestamp":"2026-01-01T00:00:00Z","payload":"${'x'.repeat(100_000)}"}`
|
||||
const newLine = '{"type":"assistant","timestamp":"2026-05-01T00:00:00Z"}'
|
||||
const p = await tmpPath(`${oldLine}\n${newLine}\n`)
|
||||
const lines: string[] = []
|
||||
for await (const line of readSessionLines(p, head => head.includes('2026-01-01'))) {
|
||||
lines.push(line)
|
||||
}
|
||||
expect(lines).toEqual([newLine])
|
||||
})
|
||||
|
||||
it('yields large lines as Buffers when requested', async () => {
|
||||
const largeLine = `{"type":"assistant","timestamp":"2026-05-01T00:00:00Z","payload":"${'x'.repeat(100_000)}"}`
|
||||
const p = await tmpPath(`${largeLine}\nsmall\n`)
|
||||
const lines: Array<string | Buffer> = []
|
||||
for await (const line of readSessionLines(p, undefined, { largeLineAsBuffer: true })) {
|
||||
lines.push(line)
|
||||
}
|
||||
expect(Buffer.isBuffer(lines[0])).toBe(true)
|
||||
expect(lines[1]).toBe('small')
|
||||
})
|
||||
|
||||
it('does not leak file descriptors when generator is abandoned early', async () => {
|
||||
const content = Array.from({ length: 1000 }, (_, i) => `line-${i}`).join('\n')
|
||||
const p = await tmpPath(content)
|
||||
|
|
|
|||
|
|
@ -325,7 +325,7 @@ describe('scanJsonlFile', () => {
|
|||
message: { content: [{ type: 'tool_use', name: 'Bash', input: {} }] },
|
||||
}))
|
||||
await scanJsonlFile(filePath, 'p1', undefined)
|
||||
expect(readSessionLinesSpy).toHaveBeenCalledWith(filePath)
|
||||
expect(readSessionLinesSpy).toHaveBeenCalledWith(filePath, undefined, { largeLineAsBuffer: true })
|
||||
expect(readSessionFileSpy).not.toHaveBeenCalled()
|
||||
readSessionLinesSpy.mockRestore()
|
||||
readSessionFileSpy.mockRestore()
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ describe('compactEntry', () => {
|
|||
expect(msg.content).toHaveLength(2)
|
||||
expect(msg.content[0]!.name).toBe('Read')
|
||||
expect(msg.content[0]!.id).toBe('tu1')
|
||||
expect(msg.content[0]!.input).toEqual({})
|
||||
expect(msg.content[0]!.input).toEqual({ file_path: '/foo' })
|
||||
expect(msg.content[1]!.name).toBe('Edit')
|
||||
expect(msg.content[1]!.id).toBe('tu2')
|
||||
expect(msg.content[1]!.input).toEqual({})
|
||||
|
|
@ -290,6 +290,44 @@ describe('compactEntry', () => {
|
|||
expect(msg.content[0]!.input['description']).toBeUndefined()
|
||||
})
|
||||
|
||||
it('keeps Read file_path capped and drops unrelated input fields', () => {
|
||||
const raw = entry({
|
||||
type: 'assistant',
|
||||
message: {
|
||||
type: 'message' as const,
|
||||
role: 'assistant' as const,
|
||||
model: 'claude-opus-4-6',
|
||||
usage: { input_tokens: 10, output_tokens: 10 },
|
||||
content: [
|
||||
{ type: 'tool_use', id: 'tu', name: 'Read', input: { file_path: '/tmp/' + 'x'.repeat(3000), content: 'big' } },
|
||||
],
|
||||
},
|
||||
})
|
||||
const c = compactEntry(raw)
|
||||
const msg = c.message as { content: Array<{ input: Record<string, unknown> }> }
|
||||
expect((msg.content[0]!.input['file_path'] as string).length).toBe(2000)
|
||||
expect(msg.content[0]!.input['content']).toBeUndefined()
|
||||
})
|
||||
|
||||
it('keeps Agent subagent_type capped and drops prompt text', () => {
|
||||
const raw = entry({
|
||||
type: 'assistant',
|
||||
message: {
|
||||
type: 'message' as const,
|
||||
role: 'assistant' as const,
|
||||
model: 'claude-opus-4-6',
|
||||
usage: { input_tokens: 10, output_tokens: 10 },
|
||||
content: [
|
||||
{ type: 'tool_use', id: 'tu', name: 'Agent', input: { subagent_type: 'reviewer'.repeat(50), prompt: 'big' } },
|
||||
],
|
||||
},
|
||||
})
|
||||
const c = compactEntry(raw)
|
||||
const msg = c.message as { content: Array<{ input: Record<string, unknown> }> }
|
||||
expect((msg.content[0]!.input['subagent_type'] as string).length).toBe(200)
|
||||
expect(msg.content[0]!.input['prompt']).toBeUndefined()
|
||||
})
|
||||
|
||||
it('handles entry with no message field', () => {
|
||||
const raw = entry({ type: 'system', timestamp: 't1', cwd: '/x' })
|
||||
const c = compactEntry(raw)
|
||||
|
|
|
|||
87
tests/parser-large-json-scanner.test.ts
Normal file
87
tests/parser-large-json-scanner.test.ts
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { parseJsonlLine } from '../src/parser.js'
|
||||
|
||||
function largeUserLine(): string {
|
||||
return JSON.stringify({
|
||||
type: 'user',
|
||||
sessionId: 's1',
|
||||
timestamp: '2026-05-01T00:00:00Z',
|
||||
cwd: '/repo',
|
||||
message: {
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'image', source: { data: 'x'.repeat(40_000) } },
|
||||
{ type: 'text', text: 'hello ' + 'a'.repeat(3000) },
|
||||
],
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
function largeAssistantLine(): string {
|
||||
return JSON.stringify({
|
||||
type: 'assistant',
|
||||
sessionId: 's1',
|
||||
timestamp: '2026-05-01T00:00:01Z',
|
||||
cwd: '/repo',
|
||||
message: {
|
||||
id: 'm1',
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
model: 'claude-sonnet-4-5',
|
||||
content: [
|
||||
{ type: 'text', text: 'x'.repeat(40_000) },
|
||||
{ type: 'tool_use', id: 'read1', name: 'Read', input: { file_path: '/tmp/file.ts', content: 'drop me' } },
|
||||
{ type: 'tool_use', id: 'agent1', name: 'Agent', input: { subagent_type: 'reviewer', prompt: 'drop me' } },
|
||||
],
|
||||
usage: {
|
||||
input_tokens: 100,
|
||||
output_tokens: 20,
|
||||
cache_read_input_tokens: 300,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
describe('large JSONL compact scanner', () => {
|
||||
it('extracts user text from array content without full JSON.parse', () => {
|
||||
const parsed = parseJsonlLine(largeUserLine())
|
||||
expect(parsed?.type).toBe('user')
|
||||
const content = parsed?.message?.role === 'user' ? parsed.message.content : ''
|
||||
expect(content).toBeTypeOf('string')
|
||||
expect((content as string).startsWith('hello ')).toBe(true)
|
||||
expect((content as string).length).toBe(2000)
|
||||
})
|
||||
|
||||
it('extracts capped tool inputs needed by optimize', () => {
|
||||
const parsed = parseJsonlLine(Buffer.from(largeAssistantLine()))
|
||||
const msg = parsed?.message
|
||||
expect(msg?.role).toBe('assistant')
|
||||
if (msg?.role !== 'assistant') return
|
||||
expect(msg.usage.input_tokens).toBe(100)
|
||||
expect(msg.usage.output_tokens).toBe(20)
|
||||
expect(msg.usage.cache_read_input_tokens).toBe(300)
|
||||
expect(msg.content).toEqual([
|
||||
{ type: 'tool_use', id: 'read1', name: 'Read', input: { file_path: '/tmp/file.ts' } },
|
||||
{ type: 'tool_use', id: 'agent1', name: 'Agent', input: { subagent_type: 'reviewer' } },
|
||||
])
|
||||
})
|
||||
|
||||
it('extracts deferred MCP inventory from large attachment lines', () => {
|
||||
const line = JSON.stringify({
|
||||
type: 'attachment',
|
||||
sessionId: 's1',
|
||||
timestamp: '2026-05-01T00:00:02Z',
|
||||
padding: 'x'.repeat(40_000),
|
||||
attachment: {
|
||||
type: 'deferred_tools_delta',
|
||||
addedNames: ['Bash', 'mcp__svc__tool'],
|
||||
},
|
||||
})
|
||||
const parsed = parseJsonlLine(Buffer.from(line)) as Record<string, unknown>
|
||||
expect(parsed['attachment']).toEqual({
|
||||
type: 'deferred_tools_delta',
|
||||
addedNames: ['Bash', 'mcp__svc__tool'],
|
||||
})
|
||||
})
|
||||
})
|
||||
|
|
@ -64,6 +64,11 @@ function assistantLine(sessionId: string, timestamp: string, messageId: string,
|
|||
})
|
||||
}
|
||||
|
||||
function messageFirstLargeAssistantLine(sessionId: string, timestamp: string, messageId: string): string {
|
||||
const hugeText = 'y'.repeat(3_000_000)
|
||||
return `{"parentUuid":"u1","isSidechain":false,"message":{"model":"claude-sonnet-4-5","id":"${messageId}","type":"message","role":"assistant","content":[{"type":"text","text":"${hugeText}"},{"type":"tool_use","id":"tu-large","name":"Edit","input":{"file_path":"/tmp/x","old_string":"a","new_string":"b"}}],"usage":{"input_tokens":1000,"output_tokens":100,"cache_read_input_tokens":5000}},"uuid":"a1","timestamp":"${timestamp}","type":"assistant","sessionId":"${sessionId}","cwd":"/projects/app"}`
|
||||
}
|
||||
|
||||
function attachmentLine(sessionId: string, timestamp: string): string {
|
||||
return JSON.stringify({
|
||||
type: 'attachment',
|
||||
|
|
@ -145,4 +150,31 @@ describe('parseAllSessions with large Claude fixture', () => {
|
|||
const sess = projects[0]!.sessions[0]!
|
||||
expect(sess.apiCalls).toBeGreaterThanOrEqual(1)
|
||||
})
|
||||
|
||||
it('parses huge message-first assistant lines without full JSON.parse expansion', async () => {
|
||||
const projectDir = join(home, '.claude', 'projects', 'messagefirst')
|
||||
await mkdir(projectDir, { recursive: true })
|
||||
|
||||
const lines = [
|
||||
userLine('s1', '2026-04-10T10:00:00Z', 100),
|
||||
messageFirstLargeAssistantLine('s1', '2026-04-10T10:00:01Z', 'msg-large'),
|
||||
]
|
||||
|
||||
await writeFile(join(projectDir, 'session.jsonl'), lines.join('\n'))
|
||||
|
||||
const range: DateRange = {
|
||||
start: new Date('2026-04-10T00:00:00Z'),
|
||||
end: new Date('2026-04-10T23:59:59Z'),
|
||||
}
|
||||
|
||||
const projects = await parseAllSessions(range, 'claude')
|
||||
expect(projects.length).toBeGreaterThan(0)
|
||||
|
||||
const sess = projects[0]!.sessions[0]!
|
||||
expect(sess.apiCalls).toBe(1)
|
||||
expect(sess.totalInputTokens).toBe(1000)
|
||||
expect(sess.totalOutputTokens).toBe(100)
|
||||
expect(sess.totalCacheReadTokens).toBe(5000)
|
||||
expect(sess.toolBreakdown['Edit']?.calls).toBe(1)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
86
tests/parser-skip-line.test.ts
Normal file
86
tests/parser-skip-line.test.ts
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
import { describe, it, expect } from 'vitest'
|
||||
|
||||
import { shouldSkipLine } from '../src/parser.js'
|
||||
|
||||
const threshold = '2026-04-01T00:00:00.000Z'
|
||||
|
||||
function makeLine(type: string, timestamp: string, payloadSize = 0): string {
|
||||
const payload = payloadSize > 0 ? `,"content":"${'x'.repeat(payloadSize)}"` : ''
|
||||
return `{"type":"${type}","sessionId":"s1","timestamp":"${timestamp}"${payload}}`
|
||||
}
|
||||
|
||||
function makeLineWithLongCwd(type: string, timestamp: string, cwdLength: number): string {
|
||||
const cwd = '/projects/' + 'a'.repeat(cwdLength)
|
||||
return `{"type":"${type}","sessionId":"s1","cwd":"${cwd}","timestamp":"${timestamp}","message":{"role":"user","content":"hi"}}`
|
||||
}
|
||||
|
||||
describe('shouldSkipLine', () => {
|
||||
it('skips old user lines', () => {
|
||||
expect(shouldSkipLine(makeLine('user', '2026-03-01T10:00:00Z'), threshold)).toBe(true)
|
||||
})
|
||||
|
||||
it('skips old assistant lines', () => {
|
||||
expect(shouldSkipLine(makeLine('assistant', '2026-03-15T10:00:00Z'), threshold)).toBe(true)
|
||||
})
|
||||
|
||||
it('does not skip in-range user lines', () => {
|
||||
expect(shouldSkipLine(makeLine('user', '2026-04-05T10:00:00Z'), threshold)).toBe(false)
|
||||
})
|
||||
|
||||
it('does not skip in-range assistant lines', () => {
|
||||
expect(shouldSkipLine(makeLine('assistant', '2026-04-10T10:00:00Z'), threshold)).toBe(false)
|
||||
})
|
||||
|
||||
it('never skips attachment lines regardless of timestamp', () => {
|
||||
expect(shouldSkipLine(makeLine('attachment', '2026-01-01T00:00:00Z'), threshold)).toBe(false)
|
||||
})
|
||||
|
||||
it('never skips system lines regardless of timestamp', () => {
|
||||
expect(shouldSkipLine(makeLine('system', '2026-01-01T00:00:00Z'), threshold)).toBe(false)
|
||||
})
|
||||
|
||||
it('never skips summary lines regardless of timestamp', () => {
|
||||
expect(shouldSkipLine(makeLine('summary', '2026-01-01T00:00:00Z'), threshold)).toBe(false)
|
||||
})
|
||||
|
||||
it('does not skip lines with no timestamp field', () => {
|
||||
expect(shouldSkipLine('{"type":"user","sessionId":"s1"}', threshold)).toBe(false)
|
||||
})
|
||||
|
||||
it('does not skip lines with unparseable timestamp', () => {
|
||||
expect(shouldSkipLine('{"type":"user","timestamp":"bad"}', threshold)).toBe(false)
|
||||
})
|
||||
|
||||
it('does not skip malformed JSON', () => {
|
||||
expect(shouldSkipLine('not json at all', threshold)).toBe(false)
|
||||
})
|
||||
|
||||
it('only reads top-level type and timestamp fields', () => {
|
||||
const line = '{"message":{"type":"assistant","timestamp":"2026-03-01T10:00:00Z"},"type":"user","timestamp":"2026-04-05T10:00:00Z"}'
|
||||
expect(shouldSkipLine(line, threshold)).toBe(false)
|
||||
})
|
||||
|
||||
it('handles timestamp pushed past 200 chars by long cwd', () => {
|
||||
const line = makeLineWithLongCwd('user', '2026-03-01T10:00:00Z', 300)
|
||||
expect(line.indexOf('"timestamp"')).toBeGreaterThan(200)
|
||||
expect(shouldSkipLine(line, threshold)).toBe(true)
|
||||
})
|
||||
|
||||
it('handles timestamp at the edge of the 2048 head window', () => {
|
||||
const line = makeLineWithLongCwd('user', '2026-03-01T10:00:00Z', 1900)
|
||||
expect(line.indexOf('"timestamp"')).toBeGreaterThan(1900)
|
||||
expect(shouldSkipLine(line, threshold)).toBe(true)
|
||||
})
|
||||
|
||||
it('returns false when timestamp is beyond the head window', () => {
|
||||
const line = makeLineWithLongCwd('user', '2026-03-01T10:00:00Z', 2100)
|
||||
expect(line.indexOf('"timestamp"')).toBeGreaterThan(2048)
|
||||
expect(shouldSkipLine(line, threshold)).toBe(false)
|
||||
})
|
||||
|
||||
it('skips old assistant line with large payload without parsing it', () => {
|
||||
const line = makeLine('assistant', '2026-02-01T10:00:00Z', 50_000_000)
|
||||
expect(line.length).toBeGreaterThan(50_000_000)
|
||||
expect(shouldSkipLine(line, threshold)).toBe(true)
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue