mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-08-10 17:14:48 +00:00
observe() classified a stable unparseable session-refresh.lock body as the terminal 'unavailable'. parseAllSessions routes that to a read-only parse, so a zero-byte or truncated lock froze warm-cache ingestion permanently across every later run while each command still exited successfully. A corrupt body is now a recoverable observation carrying a real mtime, and is recovered only through the UNMODIFIED staleness gate — tryTakeover and the age check are byte-identical to main. sameObservation gains an explicit null/non-null boundary and a sha1 of the raw bytes, because two corrupt bodies have no tokens to compare and mtime granularity is coarse on some filesystems. The heartbeat deliberately does NOT rewrite a body it cannot prove is its own. An owner that cannot prove ownership ends its ownership: mtime stops advancing, the publication fence refuses, and a successor recovers the lock one staleMs later. Losing that parse is the price of never having two owners.
448 lines
18 KiB
TypeScript
448 lines
18 KiB
TypeScript
import { afterEach, describe, expect, it } from 'vitest'
|
|
import { chmod, mkdir, mkdtemp, readFile, rm, stat, unlink, utimes, writeFile } from 'fs/promises'
|
|
import { tmpdir } from 'os'
|
|
import { join } from 'path'
|
|
|
|
import {
|
|
acquireCacheRefreshLock,
|
|
type RefreshLockClock,
|
|
} from '../src/cache-refresh-lock.js'
|
|
import { clearSessionCache, parseAllSessions } from '../src/parser.js'
|
|
import { emptyCache, loadCache, saveCache, sessionCachePath } from '../src/session-cache.js'
|
|
|
|
const dirs: string[] = []
|
|
|
|
async function tempDir(): Promise<string> {
|
|
const dir = await mkdtemp(join(tmpdir(), 'cb-refresh-lock-'))
|
|
dirs.push(dir)
|
|
return dir
|
|
}
|
|
|
|
function lockPath(dir: string): string {
|
|
return join(dir, 'session-refresh.lock')
|
|
}
|
|
|
|
function fakeClock(start = 1_000): RefreshLockClock & { advance: (ms: number) => void } {
|
|
let wall = start
|
|
let monotonic = start
|
|
return {
|
|
wallNow: () => wall,
|
|
monotonicNow: () => monotonic,
|
|
advance: ms => { wall += ms; monotonic += ms },
|
|
}
|
|
}
|
|
|
|
afterEach(async () => {
|
|
delete process.env['CODEBURN_CACHE_DIR']
|
|
await Promise.all(dirs.splice(0).map(dir => rm(dir, { recursive: true, force: true })))
|
|
})
|
|
|
|
describe('warm session-cache refresh lock', () => {
|
|
it('returns acquired and releases its own token', async () => {
|
|
const dir = await tempDir()
|
|
const result = await acquireCacheRefreshLock({ cacheDir: dir })
|
|
expect(result.outcome).toBe('acquired')
|
|
if (result.outcome !== 'acquired') return
|
|
|
|
const record = JSON.parse(await readFile(lockPath(dir), 'utf-8'))
|
|
expect(record).toMatchObject({ pid: process.pid, token: result.handle.token })
|
|
expect(typeof record.at).toBe('number')
|
|
await result.handle.release()
|
|
await expect(stat(lockPath(dir))).rejects.toMatchObject({ code: 'ENOENT' })
|
|
})
|
|
|
|
it('reports completed-by-other after a clean release', async () => {
|
|
const dir = await tempDir()
|
|
const path = lockPath(dir)
|
|
await writeFile(path, JSON.stringify({ pid: 1, token: 'holder', at: Date.now() }))
|
|
let polls = 0
|
|
const result = await acquireCacheRefreshLock({
|
|
cacheDir: dir,
|
|
waitMs: 100,
|
|
pollMs: 1,
|
|
sleep: async () => {
|
|
if (++polls === 1) await unlink(path)
|
|
},
|
|
})
|
|
expect(result).toEqual({ outcome: 'completed-by-other' })
|
|
})
|
|
|
|
it('uses a monotonic deadline and times out without invalidating the holder', async () => {
|
|
const dir = await tempDir()
|
|
const clock = fakeClock()
|
|
const path = lockPath(dir)
|
|
await writeFile(path, JSON.stringify({ pid: 1, token: 'holder', at: clock.wallNow() }))
|
|
const now = new Date(clock.wallNow())
|
|
await utimes(path, now, now)
|
|
|
|
const result = await acquireCacheRefreshLock({
|
|
cacheDir: dir,
|
|
clock,
|
|
waitMs: 30,
|
|
staleMs: 90,
|
|
pollMs: 10,
|
|
sleep: async ms => { clock.advance(ms) },
|
|
})
|
|
expect(result).toEqual({ outcome: 'timed-out' })
|
|
expect(JSON.parse(await readFile(path, 'utf-8')).token).toBe('holder')
|
|
})
|
|
|
|
it('reports unavailable when lock infrastructure is unusable', async () => {
|
|
const dir = await tempDir()
|
|
const notDirectory = join(dir, 'file')
|
|
await writeFile(notDirectory, 'x')
|
|
expect(await acquireCacheRefreshLock({ cacheDir: notDirectory })).toEqual({ outcome: 'unavailable' })
|
|
})
|
|
|
|
it('serializes same-process acquisitions before touching the filesystem lock', async () => {
|
|
const dir = await tempDir()
|
|
const first = await acquireCacheRefreshLock({ cacheDir: dir })
|
|
expect(first.outcome).toBe('acquired')
|
|
if (first.outcome !== 'acquired') return
|
|
|
|
let settled = false
|
|
const secondPromise = acquireCacheRefreshLock({ cacheDir: dir }).then(result => {
|
|
settled = true
|
|
return result
|
|
})
|
|
await new Promise(resolve => { setTimeout(resolve, 20) })
|
|
expect(settled).toBe(false)
|
|
|
|
await first.handle.release()
|
|
const second = await secondPromise
|
|
expect(second.outcome).toBe('acquired')
|
|
if (second.outcome === 'acquired') {
|
|
expect(second.handle.token).not.toBe(first.handle.token)
|
|
await second.handle.release()
|
|
}
|
|
})
|
|
|
|
it('does not take over a heartbeating owner', async () => {
|
|
const dir = await tempDir()
|
|
const path = lockPath(dir)
|
|
await writeFile(path, JSON.stringify({ pid: 1, token: 'holder', at: Date.now() }))
|
|
const heartbeat = setInterval(() => {
|
|
const now = new Date()
|
|
void utimes(path, now, now)
|
|
}, 5)
|
|
try {
|
|
const result = await acquireCacheRefreshLock({ cacheDir: dir, staleMs: 20, waitMs: 60, pollMs: 5 })
|
|
expect(result).toEqual({ outcome: 'timed-out' })
|
|
expect(JSON.parse(await readFile(path, 'utf-8')).token).toBe('holder')
|
|
} finally {
|
|
clearInterval(heartbeat)
|
|
}
|
|
})
|
|
|
|
it('heartbeats its own lock body and mtime with the injected clock', async () => {
|
|
const dir = await tempDir()
|
|
const clock = fakeClock(10_000)
|
|
const result = await acquireCacheRefreshLock({ cacheDir: dir, clock, heartbeatMs: 5 })
|
|
expect(result.outcome).toBe('acquired')
|
|
if (result.outcome !== 'acquired') return
|
|
try {
|
|
const before = (await stat(lockPath(dir))).mtimeMs
|
|
clock.advance(1_000)
|
|
await new Promise(resolve => { setTimeout(resolve, 100) })
|
|
const record = JSON.parse(await readFile(lockPath(dir), 'utf-8'))
|
|
expect(record.at).toBe(clock.wallNow())
|
|
expect((await stat(lockPath(dir))).mtimeMs).not.toBe(before)
|
|
} finally {
|
|
await result.handle.release()
|
|
}
|
|
})
|
|
|
|
it('takes over only after re-verifying a stale token and mtime', async () => {
|
|
const dir = await tempDir()
|
|
const clock = fakeClock(100_000)
|
|
const path = lockPath(dir)
|
|
await writeFile(path, JSON.stringify({ pid: 1, token: 'stale', at: 1 }))
|
|
const old = new Date(1)
|
|
await utimes(path, old, old)
|
|
|
|
const result = await acquireCacheRefreshLock({ cacheDir: dir, clock, staleMs: 90, waitMs: 100 })
|
|
expect(result.outcome).toBe('acquired')
|
|
if (result.outcome !== 'acquired') return
|
|
expect(JSON.parse(await readFile(path, 'utf-8')).token).toBe(result.handle.token)
|
|
await result.handle.release()
|
|
await expect(stat(join(dir, 'session-refresh.lock.takeover'))).rejects.toMatchObject({ code: 'ENOENT' })
|
|
})
|
|
|
|
it('reclaims an abandoned stale takeover guard', async () => {
|
|
const dir = await tempDir()
|
|
const clock = fakeClock(100_000)
|
|
await writeFile(lockPath(dir), JSON.stringify({ pid: 1, token: 'stale', at: 1 }))
|
|
await writeFile(join(dir, 'session-refresh.lock.takeover'), JSON.stringify({ pid: 2, token: 'stale-guard', at: 1 }))
|
|
const old = new Date(1)
|
|
await utimes(lockPath(dir), old, old)
|
|
await utimes(join(dir, 'session-refresh.lock.takeover'), old, old)
|
|
|
|
const result = await acquireCacheRefreshLock({ cacheDir: dir, clock, staleMs: 90, waitMs: 100 })
|
|
expect(result.outcome).toBe('acquired')
|
|
if (result.outcome === 'acquired') await result.handle.release()
|
|
})
|
|
|
|
it('fences publication and release removes only its own token', async () => {
|
|
const dir = await tempDir()
|
|
process.env['CODEBURN_CACHE_DIR'] = dir
|
|
const original = emptyCache()
|
|
original.complete = true
|
|
await saveCache(original)
|
|
|
|
const result = await acquireCacheRefreshLock({ cacheDir: dir, heartbeatMs: 60_000 })
|
|
expect(result.outcome).toBe('acquired')
|
|
if (result.outcome !== 'acquired') return
|
|
|
|
await writeFile(lockPath(dir), JSON.stringify({ pid: 999, token: 'successor', at: Date.now() }))
|
|
const changed = emptyCache()
|
|
changed.complete = true
|
|
changed.providers['claude'] = { parseVersion: 'test', envFingerprint: 'test', files: {} }
|
|
expect(await saveCache(changed, result.handle.verifyStillOwner)).toBe(false)
|
|
expect((await loadCache()).providers['claude']).toBeUndefined()
|
|
|
|
await result.handle.release()
|
|
expect(JSON.parse(await readFile(lockPath(dir), 'utf-8')).token).toBe('successor')
|
|
expect(sessionCachePath()).toContain(dir)
|
|
})
|
|
|
|
// retry shields environmental fd/CPU starvation in a saturated full-suite
|
|
// run (fs 'unavailable' makes the fence fail CLOSED, which is correct but
|
|
// not what this test measures); the actual race fails ~6% per verify, so a
|
|
// mutated build cannot pass any attempt.
|
|
it('the fence never loses to its own heartbeat (in-process serialization)', { retry: 5 }, async () => {
|
|
// Regression: verifyStillOwner and the heartbeat tick both take the
|
|
// takeover guard; without in-process serialization the fence could observe
|
|
// its own heartbeat's guard file and abort a legitimate publication.
|
|
// At a 1ms heartbeat this raced ~6% of the time before the fix.
|
|
const dir = await mkdtemp(join(tmpdir(), 'refresh-lock-'))
|
|
const result = await acquireCacheRefreshLock({ cacheDir: dir, heartbeatMs: 1 })
|
|
if (result.outcome !== 'acquired') throw new Error(`expected acquired, got ${result.outcome}`)
|
|
for (let i = 0; i < 120; i++) {
|
|
expect(await result.handle.verifyStillOwner()).toBe(true)
|
|
}
|
|
await result.handle.release()
|
|
await rm(dir, { recursive: true, force: true })
|
|
})
|
|
})
|
|
|
|
// A lock body that never parses into a record is a corrupt leftover, not an
|
|
// unusable filesystem: classifying it as 'unavailable' routed every subsequent
|
|
// refresh to the read-only path and froze ingestion permanently.
|
|
describe('warm session-cache refresh lock: corrupt lock recovery', () => {
|
|
it('takes over a stale zero-byte lock', async () => {
|
|
const dir = await tempDir()
|
|
const clock = fakeClock(100_000)
|
|
const path = lockPath(dir)
|
|
await writeFile(path, '')
|
|
const old = new Date(1)
|
|
await utimes(path, old, old)
|
|
|
|
const result = await acquireCacheRefreshLock({ cacheDir: dir, clock, staleMs: 90, waitMs: 100, pollMs: 1 })
|
|
expect(result.outcome).toBe('acquired')
|
|
if (result.outcome !== 'acquired') return
|
|
expect(JSON.parse(await readFile(path, 'utf-8')).token).toBe(result.handle.token)
|
|
await result.handle.release()
|
|
await expect(stat(join(dir, 'session-refresh.lock.takeover'))).rejects.toMatchObject({ code: 'ENOENT' })
|
|
})
|
|
|
|
it('takes over a stale malformed-JSON lock', async () => {
|
|
const dir = await tempDir()
|
|
const clock = fakeClock(100_000)
|
|
const path = lockPath(dir)
|
|
await writeFile(path, '{"pid":1,"token":')
|
|
const old = new Date(1)
|
|
await utimes(path, old, old)
|
|
|
|
const result = await acquireCacheRefreshLock({ cacheDir: dir, clock, staleMs: 90, waitMs: 100, pollMs: 1 })
|
|
expect(result.outcome).toBe('acquired')
|
|
if (result.outcome !== 'acquired') return
|
|
expect(JSON.parse(await readFile(path, 'utf-8')).token).toBe(result.handle.token)
|
|
await result.handle.release()
|
|
})
|
|
|
|
it('takes over a stale lock whose body parses but has the wrong shape', async () => {
|
|
const dir = await tempDir()
|
|
const clock = fakeClock(100_000)
|
|
const path = lockPath(dir)
|
|
await writeFile(path, JSON.stringify({ pid: 'one', token: 7 }))
|
|
const old = new Date(1)
|
|
await utimes(path, old, old)
|
|
|
|
const result = await acquireCacheRefreshLock({ cacheDir: dir, clock, staleMs: 90, waitMs: 100, pollMs: 1 })
|
|
expect(result.outcome).toBe('acquired')
|
|
if (result.outcome === 'acquired') await result.handle.release()
|
|
})
|
|
|
|
// Staleness is never waived for a corrupt body, so a FRESH one is waited out
|
|
// and left alone: it may belong to a live owner whose heartbeat is about to
|
|
// repair it. The resulting freeze is bounded by staleMs rather than
|
|
// permanent, which is the whole of the reported defect.
|
|
it('waits out a fresh malformed lock, then recovers it once it ages past staleMs', async () => {
|
|
const dir = await tempDir()
|
|
const clock = fakeClock(100_000)
|
|
const path = lockPath(dir)
|
|
await writeFile(path, '')
|
|
const now = new Date(clock.wallNow())
|
|
await utimes(path, now, now)
|
|
|
|
let polls = 0
|
|
const fresh = await acquireCacheRefreshLock({
|
|
cacheDir: dir,
|
|
clock,
|
|
staleMs: 90_000,
|
|
waitMs: 50,
|
|
pollMs: 10,
|
|
sleep: async ms => { polls++; clock.advance(ms) },
|
|
})
|
|
expect(polls).toBeGreaterThan(0)
|
|
expect(fresh).toEqual({ outcome: 'timed-out' })
|
|
expect(await readFile(path, 'utf-8')).toBe('')
|
|
|
|
// Nothing rewrote the body, so its mtime is still frozen. One stale window
|
|
// later the very next run recovers it through the unmodified age gate.
|
|
clock.advance(90_001)
|
|
const later = await acquireCacheRefreshLock({ cacheDir: dir, clock, staleMs: 90_000, waitMs: 50, pollMs: 10 })
|
|
expect(later.outcome).toBe('acquired')
|
|
if (later.outcome !== 'acquired') return
|
|
expect(JSON.parse(await readFile(path, 'utf-8')).token).toBe(later.handle.token)
|
|
await later.handle.release()
|
|
})
|
|
|
|
it('never steals a fresh malformed lock from an owner that repairs it mid-wait', async () => {
|
|
const dir = await tempDir()
|
|
const clock = fakeClock(100_000)
|
|
const path = lockPath(dir)
|
|
await writeFile(path, '')
|
|
const now = new Date(clock.wallNow())
|
|
await utimes(path, now, now)
|
|
|
|
let polls = 0
|
|
const result = await acquireCacheRefreshLock({
|
|
cacheDir: dir,
|
|
clock,
|
|
staleMs: 90_000,
|
|
waitMs: 50,
|
|
pollMs: 10,
|
|
sleep: async ms => {
|
|
if (++polls === 1) {
|
|
await writeFile(path, JSON.stringify({ pid: 1, token: 'holder', at: clock.wallNow() }))
|
|
const t = new Date(clock.wallNow())
|
|
await utimes(path, t, t)
|
|
}
|
|
clock.advance(ms)
|
|
},
|
|
})
|
|
expect(result).toEqual({ outcome: 'timed-out' })
|
|
expect(JSON.parse(await readFile(path, 'utf-8')).token).toBe('holder')
|
|
})
|
|
|
|
it('treats a body truncated mid-heartbeat as contention, not corruption', async () => {
|
|
const dir = await tempDir()
|
|
const path = lockPath(dir)
|
|
const record = (): string => JSON.stringify({ pid: 1, token: 'holder', at: Date.now() })
|
|
await writeFile(path, record())
|
|
// A real heartbeat rewrite exposes a zero-length body for an instant. The
|
|
// waiter must keep polling and leave the live owner alone.
|
|
const heartbeat = setInterval(() => {
|
|
void (async () => {
|
|
try {
|
|
await writeFile(path, '')
|
|
await writeFile(path, record())
|
|
} catch { /* the winner may have replaced the file */ }
|
|
})()
|
|
}, 1)
|
|
let result
|
|
try {
|
|
result = await acquireCacheRefreshLock({ cacheDir: dir, staleMs: 30, waitMs: 120, pollMs: 5 })
|
|
} finally {
|
|
clearInterval(heartbeat)
|
|
}
|
|
await new Promise(resolve => { setTimeout(resolve, 20) })
|
|
expect(result).toEqual({ outcome: 'timed-out' })
|
|
expect(JSON.parse(await readFile(path, 'utf-8')).token).toBe('holder')
|
|
})
|
|
|
|
// The sidecar is created by the same createExclusive as the lock, so it can be
|
|
// left 0-byte by exactly the same crash. Before observe() separated corrupt
|
|
// from unreadable this returned 'unavailable' from acquireTakeoverGuard and
|
|
// froze recovery even when the primary lock was perfectly fine.
|
|
it('reclaims a stale zero-byte takeover sidecar', async () => {
|
|
const dir = await tempDir()
|
|
const path = lockPath(dir)
|
|
const sidecar = join(dir, 'session-refresh.lock.takeover')
|
|
await writeFile(path, '')
|
|
await writeFile(sidecar, '')
|
|
const old = new Date(1)
|
|
await utimes(path, old, old)
|
|
await utimes(sidecar, old, old)
|
|
|
|
const result = await acquireCacheRefreshLock({ cacheDir: dir, staleMs: 90, waitMs: 200, pollMs: 5 })
|
|
expect(result.outcome).toBe('acquired')
|
|
if (result.outcome !== 'acquired') return
|
|
expect(JSON.parse(await readFile(path, 'utf-8')).token).toBe(result.handle.token)
|
|
await result.handle.release()
|
|
await expect(stat(sidecar)).rejects.toMatchObject({ code: 'ENOENT' })
|
|
})
|
|
|
|
it('still reports unavailable when the lock body cannot be read', async () => {
|
|
if (process.getuid?.() === 0) return
|
|
const dir = await tempDir()
|
|
const path = lockPath(dir)
|
|
await writeFile(path, '')
|
|
await chmod(path, 0o000)
|
|
const old = new Date(1)
|
|
await utimes(path, old, old)
|
|
try {
|
|
const result = await acquireCacheRefreshLock({ cacheDir: dir, staleMs: 1, waitMs: 50, pollMs: 5 })
|
|
expect(result).toEqual({ outcome: 'unavailable' })
|
|
} finally {
|
|
await chmod(path, 0o600)
|
|
}
|
|
})
|
|
|
|
it('resumes ingesting changed sources after a corrupt lock froze the refresh', async () => {
|
|
const root = await tempDir()
|
|
const cacheDir = join(root, 'cache')
|
|
const config = join(root, 'claude')
|
|
const projectDir = join(config, 'projects', 'frozen-proj')
|
|
await mkdir(cacheDir, { recursive: true })
|
|
await mkdir(projectDir, { recursive: true })
|
|
process.env['CODEBURN_CACHE_DIR'] = cacheDir
|
|
process.env['CLAUDE_CONFIG_DIR'] = config
|
|
process.env['CODEBURN_DESKTOP_SESSIONS_DIR'] = join(root, 'desktop-sessions')
|
|
|
|
const session = (id: string, ts: string): string => [
|
|
JSON.stringify({ type: 'user', sessionId: id, timestamp: ts, cwd: '/tmp/frozen-proj', message: { role: 'user', content: 'hi' } }),
|
|
JSON.stringify({
|
|
type: 'assistant', sessionId: id, timestamp: ts, cwd: '/tmp/frozen-proj',
|
|
message: { id: `msg-${id}`, type: 'message', role: 'assistant', model: 'claude-sonnet-4-5', content: [], usage: { input_tokens: 100, output_tokens: 20 } },
|
|
}),
|
|
].join('\n') + '\n'
|
|
|
|
await writeFile(join(projectDir, 'sess-1.jsonl'), session('sess-1', '2026-05-01T10:00:00.000Z'))
|
|
clearSessionCache()
|
|
const warm = await parseAllSessions()
|
|
expect(warm[0]?.sessions.map(s => s.sessionId)).toEqual(['sess-1'])
|
|
|
|
// The field state: a 0-byte lock AND the takeover sidecar of the dead owner
|
|
// that was mid-recovery when it died, both older than the stale window.
|
|
// Nothing else on the machine ever repairs either file, so on every later
|
|
// run the lock read as 'unavailable', the parser fell back to a read-only
|
|
// re-parse, and sess-2 was never ingested.
|
|
const old = new Date(1)
|
|
for (const name of ['session-refresh.lock', 'session-refresh.lock.takeover']) {
|
|
await writeFile(join(cacheDir, name), name.endsWith('.takeover') ? JSON.stringify({ pid: 999_999, token: 'dead-owner', at: 1 }) : '')
|
|
await utimes(join(cacheDir, name), old, old)
|
|
}
|
|
|
|
await writeFile(join(projectDir, 'sess-2.jsonl'), session('sess-2', '2026-05-01T11:00:00.000Z'))
|
|
clearSessionCache()
|
|
const after = await parseAllSessions()
|
|
expect(after[0]?.sessions.map(s => s.sessionId).sort()).toEqual(['sess-1', 'sess-2'])
|
|
expect(Object.keys((await loadCache()).providers['claude']?.files ?? {}).length).toBe(2)
|
|
|
|
clearSessionCache()
|
|
delete process.env['CLAUDE_CONFIG_DIR']
|
|
delete process.env['CODEBURN_DESKTOP_SESSIONS_DIR']
|
|
})
|
|
})
|