codeburn/tests/cache-refresh-lock.test.ts
Resham Joshi d96149b60f
test: widen the fence stress test's flake shield (#775)
The 1ms-heartbeat stress test fails under saturated full-suite runs when
filesystem starvation makes guard operations report unavailable, which
the fence correctly treats as fail-closed. Isolation runs, the process
tests, and mutation checks all hold. Retries go 2 to 5 and iterations
300 to 120: a mutated build still fails every attempt (six percent per
verify compounds to a per-attempt pass chance under a tenth of a
percent), while transient starvation now has five chances to clear.

Co-authored-by: reviewer <review@local>
2026-07-20 09:39:05 -07:00

225 lines
8.7 KiB
TypeScript

import { afterEach, describe, expect, it } from 'vitest'
import { mkdir, mkdtemp, readFile, rm, stat, unlink, utimes, writeFile } from 'fs/promises'
import { tmpdir } from 'os'
import { join } from 'path'
import {
acquireCacheRefreshLock,
type RefreshLockClock,
} from '../src/cache-refresh-lock.js'
import { emptyCache, loadCache, saveCache, sessionCachePath } from '../src/session-cache.js'
const dirs: string[] = []
async function tempDir(): Promise<string> {
const dir = await mkdtemp(join(tmpdir(), 'cb-refresh-lock-'))
dirs.push(dir)
return dir
}
function lockPath(dir: string): string {
return join(dir, 'session-refresh.lock')
}
function fakeClock(start = 1_000): RefreshLockClock & { advance: (ms: number) => void } {
let wall = start
let monotonic = start
return {
wallNow: () => wall,
monotonicNow: () => monotonic,
advance: ms => { wall += ms; monotonic += ms },
}
}
afterEach(async () => {
delete process.env['CODEBURN_CACHE_DIR']
await Promise.all(dirs.splice(0).map(dir => rm(dir, { recursive: true, force: true })))
})
describe('warm session-cache refresh lock', () => {
it('returns acquired and releases its own token', async () => {
const dir = await tempDir()
const result = await acquireCacheRefreshLock({ cacheDir: dir })
expect(result.outcome).toBe('acquired')
if (result.outcome !== 'acquired') return
const record = JSON.parse(await readFile(lockPath(dir), 'utf-8'))
expect(record).toMatchObject({ pid: process.pid, token: result.handle.token })
expect(typeof record.at).toBe('number')
await result.handle.release()
await expect(stat(lockPath(dir))).rejects.toMatchObject({ code: 'ENOENT' })
})
it('reports completed-by-other after a clean release', async () => {
const dir = await tempDir()
const path = lockPath(dir)
await writeFile(path, JSON.stringify({ pid: 1, token: 'holder', at: Date.now() }))
let polls = 0
const result = await acquireCacheRefreshLock({
cacheDir: dir,
waitMs: 100,
pollMs: 1,
sleep: async () => {
if (++polls === 1) await unlink(path)
},
})
expect(result).toEqual({ outcome: 'completed-by-other' })
})
it('uses a monotonic deadline and times out without invalidating the holder', async () => {
const dir = await tempDir()
const clock = fakeClock()
const path = lockPath(dir)
await writeFile(path, JSON.stringify({ pid: 1, token: 'holder', at: clock.wallNow() }))
const now = new Date(clock.wallNow())
await utimes(path, now, now)
const result = await acquireCacheRefreshLock({
cacheDir: dir,
clock,
waitMs: 30,
staleMs: 90,
pollMs: 10,
sleep: async ms => { clock.advance(ms) },
})
expect(result).toEqual({ outcome: 'timed-out' })
expect(JSON.parse(await readFile(path, 'utf-8')).token).toBe('holder')
})
it('reports unavailable when lock infrastructure is unusable', async () => {
const dir = await tempDir()
const notDirectory = join(dir, 'file')
await writeFile(notDirectory, 'x')
expect(await acquireCacheRefreshLock({ cacheDir: notDirectory })).toEqual({ outcome: 'unavailable' })
})
it('serializes same-process acquisitions before touching the filesystem lock', async () => {
const dir = await tempDir()
const first = await acquireCacheRefreshLock({ cacheDir: dir })
expect(first.outcome).toBe('acquired')
if (first.outcome !== 'acquired') return
let settled = false
const secondPromise = acquireCacheRefreshLock({ cacheDir: dir }).then(result => {
settled = true
return result
})
await new Promise(resolve => { setTimeout(resolve, 20) })
expect(settled).toBe(false)
await first.handle.release()
const second = await secondPromise
expect(second.outcome).toBe('acquired')
if (second.outcome === 'acquired') {
expect(second.handle.token).not.toBe(first.handle.token)
await second.handle.release()
}
})
it('does not take over a heartbeating owner', async () => {
const dir = await tempDir()
const path = lockPath(dir)
await writeFile(path, JSON.stringify({ pid: 1, token: 'holder', at: Date.now() }))
const heartbeat = setInterval(() => {
const now = new Date()
void utimes(path, now, now)
}, 5)
try {
const result = await acquireCacheRefreshLock({ cacheDir: dir, staleMs: 20, waitMs: 60, pollMs: 5 })
expect(result).toEqual({ outcome: 'timed-out' })
expect(JSON.parse(await readFile(path, 'utf-8')).token).toBe('holder')
} finally {
clearInterval(heartbeat)
}
})
it('heartbeats its own lock body and mtime with the injected clock', async () => {
const dir = await tempDir()
const clock = fakeClock(10_000)
const result = await acquireCacheRefreshLock({ cacheDir: dir, clock, heartbeatMs: 5 })
expect(result.outcome).toBe('acquired')
if (result.outcome !== 'acquired') return
try {
const before = (await stat(lockPath(dir))).mtimeMs
clock.advance(1_000)
await new Promise(resolve => { setTimeout(resolve, 100) })
const record = JSON.parse(await readFile(lockPath(dir), 'utf-8'))
expect(record.at).toBe(clock.wallNow())
expect((await stat(lockPath(dir))).mtimeMs).not.toBe(before)
} finally {
await result.handle.release()
}
})
it('takes over only after re-verifying a stale token and mtime', async () => {
const dir = await tempDir()
const clock = fakeClock(100_000)
const path = lockPath(dir)
await writeFile(path, JSON.stringify({ pid: 1, token: 'stale', at: 1 }))
const old = new Date(1)
await utimes(path, old, old)
const result = await acquireCacheRefreshLock({ cacheDir: dir, clock, staleMs: 90, waitMs: 100 })
expect(result.outcome).toBe('acquired')
if (result.outcome !== 'acquired') return
expect(JSON.parse(await readFile(path, 'utf-8')).token).toBe(result.handle.token)
await result.handle.release()
await expect(stat(join(dir, 'session-refresh.lock.takeover'))).rejects.toMatchObject({ code: 'ENOENT' })
})
it('reclaims an abandoned stale takeover guard', async () => {
const dir = await tempDir()
const clock = fakeClock(100_000)
await writeFile(lockPath(dir), JSON.stringify({ pid: 1, token: 'stale', at: 1 }))
await writeFile(join(dir, 'session-refresh.lock.takeover'), JSON.stringify({ pid: 2, token: 'stale-guard', at: 1 }))
const old = new Date(1)
await utimes(lockPath(dir), old, old)
await utimes(join(dir, 'session-refresh.lock.takeover'), old, old)
const result = await acquireCacheRefreshLock({ cacheDir: dir, clock, staleMs: 90, waitMs: 100 })
expect(result.outcome).toBe('acquired')
if (result.outcome === 'acquired') await result.handle.release()
})
it('fences publication and release removes only its own token', async () => {
const dir = await tempDir()
process.env['CODEBURN_CACHE_DIR'] = dir
const original = emptyCache()
original.complete = true
await saveCache(original)
const result = await acquireCacheRefreshLock({ cacheDir: dir, heartbeatMs: 60_000 })
expect(result.outcome).toBe('acquired')
if (result.outcome !== 'acquired') return
await writeFile(lockPath(dir), JSON.stringify({ pid: 999, token: 'successor', at: Date.now() }))
const changed = emptyCache()
changed.complete = true
changed.providers['claude'] = { parseVersion: 'test', envFingerprint: 'test', files: {} }
expect(await saveCache(changed, result.handle.verifyStillOwner)).toBe(false)
expect((await loadCache()).providers['claude']).toBeUndefined()
await result.handle.release()
expect(JSON.parse(await readFile(lockPath(dir), 'utf-8')).token).toBe('successor')
expect(sessionCachePath()).toContain(dir)
})
// retry shields environmental fd/CPU starvation in a saturated full-suite
// run (fs 'unavailable' makes the fence fail CLOSED, which is correct but
// not what this test measures); the actual race fails ~6% per verify, so a
// mutated build cannot pass any attempt.
it('the fence never loses to its own heartbeat (in-process serialization)', { retry: 5 }, async () => {
// Regression: verifyStillOwner and the heartbeat tick both take the
// takeover guard; without in-process serialization the fence could observe
// its own heartbeat's guard file and abort a legitimate publication.
// At a 1ms heartbeat this raced ~6% of the time before the fix.
const dir = await mkdtemp(join(tmpdir(), 'refresh-lock-'))
const result = await acquireCacheRefreshLock({ cacheDir: dir, heartbeatMs: 1 })
if (result.outcome !== 'acquired') throw new Error(`expected acquired, got ${result.outcome}`)
for (let i = 0; i < 120; i++) {
expect(await result.handle.verifyStillOwner()).toBe(true)
}
await result.handle.release()
await rm(dir, { recursive: true, force: true })
})
})