mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-05-17 12:20:43 +00:00
Three-layer fix for V8 heap exhaustion when parsing heavy session data: 1. Buffer-based readSessionLines (fs-utils.ts): Replace readline with raw Buffer streaming using Buffer.indexOf(0x0a). Eliminates ConsString trees that caused OOM when regex-flattening 100MB+ lines. Two-state machine (ACCUMULATING/SCANNING) skips old lines at ~2KB cost instead of 200MB. 2. Large-line streaming parser (parser.ts): Hand-written JSON scanner for lines >32KB extracts only cost/token/tool fields without JSON.parse, avoiding full object graph allocation. Dual string/Buffer paths. 3. Dashboard memory management (dashboard.tsx): Disable auto-refresh for heavy periods (30d/month/all), clear old dataset before reload via nextTick to allow GC, prevent overlapping reloads with mutex, lazy optimize scanning on keypress instead of useEffect. Also fixes three race conditions in dashboard reload deduplication: - Early return after nextTick bypassing finally block (permanent mutex lock) - A->B->A period switching dropping final reload (stale pending) - Stale pendingReloadRef not cleared when in-flight matches request
120 lines
4 KiB
TypeScript
120 lines
4 KiB
TypeScript
import { describe, it, expect, afterEach, vi } from 'vitest'
|
|
import { mkdtemp, writeFile, rm } from 'fs/promises'
|
|
import { tmpdir } from 'os'
|
|
import { join } from 'path'
|
|
|
|
import {
|
|
MAX_SESSION_FILE_BYTES,
|
|
readSessionFile,
|
|
readSessionLines,
|
|
} from '../src/fs-utils.js'
|
|
|
|
describe('readSessionFile', () => {
|
|
const tmpDirs: string[] = []
|
|
|
|
afterEach(async () => {
|
|
delete process.env.CODEBURN_VERBOSE
|
|
while (tmpDirs.length > 0) {
|
|
const d = tmpDirs.pop()
|
|
if (d) await rm(d, { recursive: true, force: true })
|
|
}
|
|
})
|
|
|
|
async function tmpPath(content: string | Buffer): Promise<string> {
|
|
const base = await mkdtemp(join(tmpdir(), 'codeburn-fs-'))
|
|
tmpDirs.push(base)
|
|
const p = join(base, 'x.jsonl')
|
|
await writeFile(p, content)
|
|
return p
|
|
}
|
|
|
|
it('returns content for small files via readFile fast path', async () => {
|
|
const p = await tmpPath('hello\nworld\n')
|
|
expect(await readSessionFile(p)).toBe('hello\nworld\n')
|
|
})
|
|
|
|
it('returns content for large files under the full-file cap', async () => {
|
|
const size = 8 * 1024 * 1024
|
|
const p = await tmpPath(Buffer.alloc(size, 'a'))
|
|
const got = await readSessionFile(p)
|
|
expect(got).not.toBeNull()
|
|
expect(got!.length).toBe(size)
|
|
})
|
|
|
|
it('returns null and skips files over the cap', async () => {
|
|
const p = await tmpPath(Buffer.alloc(MAX_SESSION_FILE_BYTES + 1, 'b'))
|
|
expect(await readSessionFile(p)).toBeNull()
|
|
})
|
|
|
|
it('emits stderr warning under CODEBURN_VERBOSE=1 for skipped file', async () => {
|
|
process.env.CODEBURN_VERBOSE = '1'
|
|
const p = await tmpPath(Buffer.alloc(MAX_SESSION_FILE_BYTES + 1, 'c'))
|
|
const spy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true)
|
|
await readSessionFile(p)
|
|
expect(spy).toHaveBeenCalled()
|
|
const msg = (spy.mock.calls[0][0] as string)
|
|
expect(msg).toContain('codeburn')
|
|
expect(msg).toContain('oversize')
|
|
spy.mockRestore()
|
|
})
|
|
|
|
it('returns null on stat failure without throwing', async () => {
|
|
expect(await readSessionFile('/nonexistent/path/x.jsonl')).toBeNull()
|
|
})
|
|
})
|
|
|
|
describe('readSessionLines', () => {
|
|
const tmpDirs: string[] = []
|
|
|
|
afterEach(async () => {
|
|
while (tmpDirs.length > 0) {
|
|
const d = tmpDirs.pop()
|
|
if (d) await rm(d, { recursive: true, force: true })
|
|
}
|
|
})
|
|
|
|
async function tmpPath(content: string): Promise<string> {
|
|
const base = await mkdtemp(join(tmpdir(), 'codeburn-lines-'))
|
|
tmpDirs.push(base)
|
|
const p = join(base, 'session.jsonl')
|
|
await writeFile(p, content)
|
|
return p
|
|
}
|
|
|
|
it('yields all lines from a file', async () => {
|
|
const p = await tmpPath('line1\nline2\nline3\n')
|
|
const lines: string[] = []
|
|
for await (const line of readSessionLines(p)) lines.push(line)
|
|
expect(lines).toEqual(['line1', 'line2', 'line3'])
|
|
})
|
|
|
|
it('skips old large lines before materializing the full line', async () => {
|
|
const oldLine = `{"type":"assistant","timestamp":"2026-01-01T00:00:00Z","payload":"${'x'.repeat(100_000)}"}`
|
|
const newLine = '{"type":"assistant","timestamp":"2026-05-01T00:00:00Z"}'
|
|
const p = await tmpPath(`${oldLine}\n${newLine}\n`)
|
|
const lines: string[] = []
|
|
for await (const line of readSessionLines(p, head => head.includes('2026-01-01'))) {
|
|
lines.push(line)
|
|
}
|
|
expect(lines).toEqual([newLine])
|
|
})
|
|
|
|
it('yields large lines as Buffers when requested', async () => {
|
|
const largeLine = `{"type":"assistant","timestamp":"2026-05-01T00:00:00Z","payload":"${'x'.repeat(100_000)}"}`
|
|
const p = await tmpPath(`${largeLine}\nsmall\n`)
|
|
const lines: Array<string | Buffer> = []
|
|
for await (const line of readSessionLines(p, undefined, { largeLineAsBuffer: true })) {
|
|
lines.push(line)
|
|
}
|
|
expect(Buffer.isBuffer(lines[0])).toBe(true)
|
|
expect(lines[1]).toBe('small')
|
|
})
|
|
|
|
it('does not leak file descriptors when generator is abandoned early', async () => {
|
|
const content = Array.from({ length: 1000 }, (_, i) => `line-${i}`).join('\n')
|
|
const p = await tmpPath(content)
|
|
const gen = readSessionLines(p)
|
|
await gen.next()
|
|
await gen.return(undefined)
|
|
})
|
|
})
|