From 5bc78b8e9e0d482ab8fb81c1d3bca2a5978fa9d0 Mon Sep 17 00:00:00 2001 From: iamtoruk Date: Mon, 17 Aug 2026 02:50:30 -0700 Subject: [PATCH] perf(parse-workers): gate on pending bytes, size the per-worker budget per parse Three fixes from review, all measured on this box. The workload gate was files OR bytes. The files arm is wrong: 250 pending files holding 117 KB between them spawned 5 threads and ran ~5% SLOWER than serial, and a file count only starts paying for itself around 400. Gate on bytes alone; the count still takes max(files / 50, bytes / 200 MB), so a few hundred huge rollouts keep their threads. The flat 256 MB per-worker memory budget was contradicted by the Codex workload: a 260 MB rollout peaks near 430 MB in its worker, linearly across the pool. It is now derived per parse as clamp(256 MB, 2 x average pending file + 128 MB, 1 GB), which leaves a corpus of small Claude transcripts where it was and stops over-subscribing on rollouts. The parent's buffer of up to pool.size finished results is part of that peak and is named in the comment. The worker/file pairing at both install sites was positional, guarded only by position (Claude) or a path membership check (Codex). Each worker now echoes its path and the parent asserts it, outside the per-file try: a misalignment would install one session's turns under another's path -- a wrong number nobody would ever notice -- so it fails the run rather than being swallowed as a parse failure. On the Claude side that meant hoisting the whole worker-result block above the try, which is safe because an append never consumes a result in either its shortcut or its straddled-fallthrough case. --- CHANGELOG.md | 4 +- docs/architecture.md | 16 +++++- src/parse-worker.ts | 4 +- src/parse-workers.ts | 35 +++++++----- src/parser.ts | 110 +++++++++++++++++++++--------------- tests/parse-workers.test.ts | 29 ++++++---- 6 files changed, 118 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0b03ec2..9242c070 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,8 @@ ## Unreleased ### Changed -- **Codex rollouts parse across worker threads too, and the workload gate now takes bytes or files.** Codex is the bigger half of a real cold parse — a 4 GB rollout corpus against 1.8 GB of Claude sessions — and it was still decoding one file at a time. A whole-file rollout decode now runs on the same pool, against an empty dedup set, and comes back with the calls, the dedup keys it claimed, and the codex-cache entry it would have written; the parent installs all three in the serial loop's order, so `codex-results.json` and every payload come out byte-identical to a serial run. Cross-file state stays where it was: a forked rollout replaying its parent's token_count history collides on the parent's keys and is re-parsed in-process, and no worker ever touches the cache module's per-directory state. Files the Codex cache can serve exactly or resume into from a byte offset never reach a worker — they read a few KB and the resume state belongs to the parent. The workload gate is now files OR bytes rather than both (200 whole-file parses or 200 MB behind them), and the count takes `max(pendingFiles / 50, pendingBytes / 200 MB)`, so a corpus of a few hundred huge rollouts parallelizes instead of falling back to one thread. The decision is per provider, and at most one pool is alive at a time. -- **A large cold Claude parse now runs across worker threads.** Reading, decoding and line-parsing a session JSONL is per-file work that never touches anything shared, so it moves onto `worker_threads`; each worker ships its parsed turns back as a JSON string and the parent installs them in the exact order the serial loop would. Everything with cross-file state — the streaming-message dedup, canonical project paths, spawn links, PR correlation, progress saves — stays on the main thread, and a file whose message ids were already claimed by an earlier file (or whose worker failed) is simply re-parsed in-process, so the session cache and every payload are identical either way. On a 6 GB corpus a cold `status` drops from 27.5s to 14.8s with peak RSS up 2.27 GB → 2.52 GB. Threads only engage for a genuinely large cold parse: never with fewer than 200 pending whole-file re-parses and under 200 MB behind them, 2 or fewer cores, or under 4 GB of available memory — so warm and incremental runs are untouched and spawn nothing. Otherwise the count is `min(cores - 1, min(0.25 × available, 2 GB) / 256 MB, pendingFiles / 50)`, where available is `process.availableMemory()` (cgroup-aware in containers) rather than free memory, which on macOS reports free pages and would switch the feature on and off between runs. `CODEBURN_PARSE_WORKERS=0` forces the serial parse and `CODEBURN_PARSE_WORKERS=N` forces N (capped at the core count), both bypassing every gate; `CODEBURN_VERBOSE=1` prints the resolved count and why. +- **Codex rollouts parse across worker threads too, and the workload gate now takes bytes or files.** Codex is the bigger half of a real cold parse — a 4 GB rollout corpus against 1.8 GB of Claude sessions — and it was still decoding one file at a time. A whole-file rollout decode now runs on the same pool, against an empty dedup set, and comes back with the calls, the dedup keys it claimed, and the codex-cache entry it would have written; the parent installs all three in the serial loop's order, so `codex-results.json` and every payload come out byte-identical to a serial run. Cross-file state stays where it was: a forked rollout replaying its parent's token_count history collides on the parent's keys and is re-parsed in-process, and no worker ever touches the cache module's per-directory state. Files the Codex cache can serve exactly or resume into from a byte offset never reach a worker — they read a few KB and the resume state belongs to the parent. The workload gate is now pending BYTES alone (200 MB), not file count: 250 pending files holding under a megabyte between them spawned threads that made the run ~5% slower, while a few hundred huge rollouts were being turned away. The count takes `max(pendingFiles / 50, pendingBytes / 200 MB)`, and the per-thread memory budget is derived per parse as `clamp(256 MB, 2 × average pending file + 128 MB, 1 GB)` rather than a flat 256 MB — a 260 MB rollout peaks near 430 MB in its worker and scales linearly with the pool, so the flat figure over-subscribed exactly the workload this adds. The decision is per provider, and at most one pool is alive at a time. +- **A large cold Claude parse now runs across worker threads.** Reading, decoding and line-parsing a session JSONL is per-file work that never touches anything shared, so it moves onto `worker_threads`; each worker ships its parsed turns back as a JSON string and the parent installs them in the exact order the serial loop would. Everything with cross-file state — the streaming-message dedup, canonical project paths, spawn links, PR correlation, progress saves — stays on the main thread, and a file whose message ids were already claimed by an earlier file (or whose worker failed) is simply re-parsed in-process, so the session cache and every payload are identical either way. On a 6 GB corpus a cold `status` drops from 27.5s to 14.8s with peak RSS up 2.27 GB → 2.52 GB. Threads only engage for a genuinely large cold parse: never with under 200 MB behind the pending whole-file re-parses, 2 or fewer cores, or under 4 GB of available memory — so warm and incremental runs are untouched and spawn nothing. Otherwise the count is `min(cores - 1, min(0.25 × available, 2 GB) / 256 MB, pendingFiles / 50)`, where available is `process.availableMemory()` (cgroup-aware in containers) rather than free memory, which on macOS reports free pages and would switch the feature on and off between runs. `CODEBURN_PARSE_WORKERS=0` forces the serial parse and `CODEBURN_PARSE_WORKERS=N` forces N (capped at the core count), both bypassing every gate; `CODEBURN_VERBOSE=1` prints the resolved count and why. - **A warm launch rewrites only the month that changed, and a ranged query reads only the months it can report on.** Per-provider shards still meant one appended session republished that provider's entire history — 95 MB for Claude on a 6 GB corpus. Each provider's shard is now split again by the UTC month of the cached session's FIRST turn, a bucket that never moves as a session grows, so an append rewrites one month. Every shard records the newest month it holds, which lets `--period today/week` skip the shards that cannot contribute a turn to the range; the skipped months stay on disk untouched across the save, and providers whose cache is the only surviving record (durable) or whose parse fingerprint moved are always read in full. Remaining shards are read concurrently. Existing v8 and v7 caches are re-laid-out losslessly on first load and the old layout removed once the new one is published: nothing re-parses. - **A warm launch rewrites only the provider that changed.** The session cache was a single blob, so any provider appending a few KB republished the whole thing — 147 MB of stringify + fsync on a 6 GB corpus, ~18% of a warm run. It is now a version-suffixed directory holding one shard per provider plus a small envelope, written per provider and published by a single envelope rename. An existing v7 cache is re-laid-out losslessly on first load and the old file removed once the new layout is on disk: nothing re-parses. One unreadable shard now costs that provider a re-parse instead of discarding every provider's history, and partial saves during a cold parse are triggered every 2000 files rather than every 5 seconds, so a slow cold parse no longer rewrites the growing cache on a wall clock. - **An appended Codex rollout parses only its tail.** Rollout files are append-only and the active ones run to hundreds of MB, but the Codex result cache keyed on mtime + size alone, so any growth re-read the file from byte 0. Each entry now records a restart point at the last task boundary — byte offset plus the state the single-pass decode carries across it — and a grown file with the same inode resumes there, producing output identical to a full re-parse. An entry without a usable restart point simply re-parses in full once and gains one. diff --git a/docs/architecture.md b/docs/architecture.md index 226bcebe..2f7277b1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -100,14 +100,24 @@ The pool is off by default for anything that is not a large cold parse: | Gate | Serial when | |---|---| -| Pending workload | fewer than 200 whole-file parses AND under 200 MB behind them (either one on its own qualifies) | +| Pending bytes | under 200 MB behind the pending whole-file parses | | Cores | `availableParallelism() <= 2` | | Memory | under 4 GB available | Otherwise the worker count is -`min(cores - 1, min(0.25 * available, 2 GB) / 256 MB, max(pendingFiles / 50, pendingBytes / 200 MB))`. +`min(cores - 1, min(0.25 * available, 2 GB) / perWorker, max(pendingFiles / 50, pendingBytes / 200 MB))`. Files and bytes each earn threads on their own, so a few hundred multi-hundred-MB -Codex rollouts parallelize as well as a few thousand small Claude transcripts. +Codex rollouts parallelize as well as a few thousand small Claude transcripts. The +gate is bytes only, deliberately: 250 pending files holding under a megabyte +between them spawn threads that make the run ~5% slower, and a file count only +starts paying for itself around 400. + +`perWorker` is the per-thread memory budget, derived per parse as +`clamp(256 MB, 2 x (pendingBytes / pendingFiles) + 128 MB, 1 GB)`. A flat figure +was wrong in both directions: small Claude transcripts peak well under 256 MB, +while a 260 MB Codex rollout peaks near 430 MB in its worker and scales linearly +with the pool. The budget also covers the parent, which buffers up to `pool.size` +finished results while it installs one. "Available" is `process.availableMemory()`, falling back to `os.totalmem()`. It is deliberately not `os.freemem()`: on macOS that counts free pages rather than diff --git a/src/parse-worker.ts b/src/parse-worker.ts index 3329ebd7..df3d9cbf 100644 --- a/src/parse-worker.ts +++ b/src/parse-worker.ts @@ -22,11 +22,11 @@ port.on('message', (msg: ParseJob) => { const seen = new Set() if (msg.kind === 'codex') { const parsed = await parseCodexFileFull(msg.source, seen) - port.postMessage({ json: JSON.stringify({ ...parsed, keys: [...seen] }) }) + port.postMessage({ json: JSON.stringify({ ...parsed, keys: [...seen], path: msg.source.path }) }) return } const parsed = await parseClaudeFileFull(msg.filePath, seen) - port.postMessage({ json: parsed === null ? null : JSON.stringify({ ...parsed, msgIds: [...seen] }) }) + port.postMessage({ json: parsed === null ? null : JSON.stringify({ ...parsed, msgIds: [...seen], path: msg.filePath }) }) } catch (err) { port.postMessage({ error: err instanceof Error ? err.message : String(err) }) } diff --git a/src/parse-workers.ts b/src/parse-workers.ts index 842592ae..25639687 100644 --- a/src/parse-workers.ts +++ b/src/parse-workers.ts @@ -4,20 +4,23 @@ import { snapshotPricingState } from './models.js' import type { ClaudeFileParse } from './parser.js' import type { SessionSource } from './providers/types.js' -// Each worker holds one file's entry list plus its serialized result. 256 MB is -// the observed high-water mark for the largest real session files; going over the -// budget is what turns a parallel parse into a swapping one. -const PER_WORKER_RSS_BYTES = 256 * 1024 * 1024 +// A worker holds one file's entries plus its serialized result, and the parent +// buffers up to `pool.size` finished results while it installs one — both are in +// this budget. A flat 256 MB was measured wrong on Codex: a 260 MB rollout peaks +// near 430 MB per worker and scales linearly with the pool. So derive it from the +// average pending file instead, floored at the small-transcript figure and capped +// at 1 GB. Going over the budget is what turns a parallel parse into a swapping one. +const MIN_PER_WORKER_RSS_BYTES = 256 * 1024 * 1024 +const MAX_PER_WORKER_RSS_BYTES = 1024 * 1024 * 1024 +const PER_WORKER_RSS_OVERHEAD_BYTES = 128 * 1024 * 1024 const MEMORY_BUDGET_CAP_BYTES = 2 * 1024 * 1024 * 1024 const MIN_AVAILABLE_BYTES = 4 * 1024 * 1024 * 1024 const MIN_FILES_PER_WORKER = 50 const MIN_BYTES_PER_WORKER = 200 * 1024 * 1024 -// Below BOTH of these, a parse is warm/incremental and the thread startup + -// result transfer costs more than the parallelism buys. Either one on its own -// qualifies: a Codex corpus is a few hundred rollouts of which a handful carry -// most of the gigabytes, so a file-count-only gate leaves the biggest workload -// there is (150 huge rollouts) parsing serially. -const MIN_PENDING_FILES = 200 +// Below this a parse is warm/incremental and the thread startup + result transfer +// costs more than the parallelism buys. Bytes, not file count: 250 pending files +// holding under a megabyte between them spawn threads that make the run ~5% +// SLOWER, and the file count only starts paying for itself around 400. const MIN_PENDING_BYTES = 200 * 1024 * 1024 export type ParseWorkerDecision = { workers: number; reason: string } @@ -58,19 +61,21 @@ export function decideParseWorkers( // Workload gates first, so a warm run's log line says "warm", not whatever the // machine happened to look like at that moment. - if (pending.files < MIN_PENDING_FILES && pending.bytes < MIN_PENDING_BYTES) { - return { workers: 0, reason: `below ${MIN_PENDING_FILES} pending files and ${Math.round(MIN_PENDING_BYTES / 1e6)} MB pending; ${inputs}` } - } + if (pending.bytes < MIN_PENDING_BYTES) return { workers: 0, reason: `below ${Math.round(MIN_PENDING_BYTES / 1e6)} MB pending; ${inputs}` } if (sys.cores <= 2) return { workers: 0, reason: `too few cores; ${inputs}` } if (sys.availableBytes < MIN_AVAILABLE_BYTES) return { workers: 0, reason: `below ${Math.round(MIN_AVAILABLE_BYTES / 1e9)} GB available memory; ${inputs}` } const memoryBudget = Math.min(0.25 * sys.availableBytes, MEMORY_BUDGET_CAP_BYTES) + const perWorker = Math.min( + MAX_PER_WORKER_RSS_BYTES, + Math.max(MIN_PER_WORKER_RSS_BYTES, 2 * (pending.bytes / Math.max(1, pending.files)) + PER_WORKER_RSS_OVERHEAD_BYTES), + ) // Files and bytes each earn threads on their own: a few hundred huge rollouts // are as parallelisable as a few thousand small transcripts, and gating the // count on files alone would hand a 6 GB / 60-file workload a single thread. const workers = Math.min( sys.cores - 1, - Math.floor(memoryBudget / PER_WORKER_RSS_BYTES), + Math.floor(memoryBudget / perWorker), Math.max( Math.floor(pending.files / MIN_FILES_PER_WORKER), Math.floor(pending.bytes / MIN_BYTES_PER_WORKER), @@ -112,7 +117,7 @@ export type ParseWorkerResult = | { ok: true; parsed: T | null } | { ok: false; error: string } -export type ClaudeWorkerParse = ClaudeFileParse & { msgIds: string[] } +export type ClaudeWorkerParse = ClaudeFileParse & { msgIds: string[]; path: string } type Task = { job: ParseJob; resolve: (r: ParseWorkerResult) => void } diff --git a/src/parser.ts b/src/parser.ts index f7bf7167..7eec184c 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -2071,6 +2071,39 @@ async function scanProjectDirs( delete section.files[filePath] markCacheDirty(diskCache, 'claude', filePath) + // Off-thread results arrive in this order (parseFilesInOrder), so the Nth + // full re-parse here is the Nth yielded result — appends never consume one, + // in either the shortcut or the straddled-fallthrough case. A worker parses + // against an EMPTY dedup set, so an EMPTY id intersection is the proof that a + // serial parse would have dropped nothing either — that, and only that, makes + // the result installable. On any overlap the WHOLE file is discarded and + // re-parsed in-process. Never patch the overlapping turns out of a worker + // result instead: a drop is not local to its own turn, because + // parsedTurnsToCachedTurns delta-encodes gitBranch across turns, so removing + // one turn changes whether a LATER turn carries a gitBranch key. + // Deliberately OUTSIDE the per-file try below: the pairing is positional, and + // a misalignment would install one session's turns under another's path — a + // wrong number nobody would ever notice, so it fails the run instead of being + // caught as a parse failure. + let parsed: ClaudeFileParse | null | undefined + if (offThread && !append) { + const result = (await offThread.next()).value + if (result?.ok && result.parsed) { + if (result.parsed.path !== filePath) { + throw new Error(`claude parse worker result out of order: got ${result.parsed.path}, expected ${filePath}`) + } + if (result.parsed.msgIds.some(id => seenMsgIds.has(id))) { + workerDiscards++ + parsed = undefined + } else { + for (const id of result.parsed.msgIds) seenMsgIds.add(id) + parsed = result.parsed + } + } else if (result?.ok) { + parsed = null + } + } + try { if (append) { // Append-only growth: parse ONLY the bytes past the cached resume offset @@ -2186,30 +2219,6 @@ async function scanProjectDirs( // Straddled: fall through to the full re-parse below. } - // Off-thread results arrive in this order (parseFilesInOrder), so the Nth - // full re-parse here is the Nth yielded result. A worker parses against an - // EMPTY dedup set, so an EMPTY id intersection is the proof that a serial - // parse would have dropped nothing either — that, and only that, makes the - // result installable. On any overlap the WHOLE file is discarded and - // re-parsed in-process. Never patch the overlapping turns out of a worker - // result instead: a drop is not local to its own turn, because - // parsedTurnsToCachedTurns delta-encodes gitBranch across turns, so removing - // one turn changes whether a LATER turn carries a gitBranch key. - let parsed: ClaudeFileParse | null | undefined - if (offThread && !append) { - const result = (await offThread.next()).value - if (result?.ok && result.parsed) { - if (result.parsed.msgIds.some(id => seenMsgIds.has(id))) { - workerDiscards++ - parsed = undefined - } else { - for (const id of result.parsed.msgIds) seenMsgIds.add(id) - parsed = result.parsed - } - } else if (result?.ok) { - parsed = null - } - } if (parsed === undefined) parsed = await parseClaudeFileFull(filePath, seenMsgIds) if (!parsed) { filesDone++; await parseProgress.tick(filesDone); continue } @@ -3118,7 +3127,7 @@ async function parseProviderSources( process.stderr.write(`codeburn: parse workers unavailable, parsing serially (${err instanceof Error ? err.message : String(err)})\n`) } } - const offThread = pool ? parseFilesInOrder(pool, workerJobs) : null + const offThread = pool ? parseFilesInOrder(pool, workerJobs) : null // Parse changed files, update cache let didParse = false @@ -3143,30 +3152,37 @@ async function parseProviderSources( clearedPaths.add(source.path) } - try { - // Off-thread results arrive in this order, so the Nth eligible file here - // is the Nth yielded result. A worker decodes against an EMPTY dedup set, - // so an EMPTY key intersection is the proof that a serial parse would - // have dropped nothing either — that, and only that, makes the result - // installable. On any overlap (a forked rollout replaying its parent's - // token_count history is exactly this) the WHOLE file is discarded and - // re-parsed in-process against the real dedup set. - let providerCalls: ParsedProviderCall[] | undefined - if (offThread && workerPaths.has(source.path)) { - const result = (await offThread.next()).value - if (result?.ok && result.parsed) { - if (result.parsed.keys.some(k => parserDedup.has(k))) { - workerDiscards++ - } else { - for (const k of result.parsed.keys) parserDedup.add(k) - providerCalls = result.parsed.calls - // The worker never touches the codex cache; publish its entry here, - // in install order, so flushCodexCache writes what serial would. - const write = result.parsed.write - if (write) await writeCachedCodexResults(source.path, write.project, providerCalls, write.fingerprint, write.resume) - } + // Off-thread results arrive in this order, so the Nth eligible file here is + // the Nth yielded result. A worker decodes against an EMPTY dedup set, so an + // EMPTY key intersection is the proof that a serial parse would have dropped + // nothing either — that, and only that, makes the result installable. On any + // overlap (a forked rollout replaying its parent's token_count history is + // exactly this) the WHOLE file is discarded and re-parsed in-process against + // the real dedup set. Deliberately OUTSIDE the per-file try below: the + // pairing is positional, and a misalignment would install one rollout's + // calls under another's path — a wrong number nobody would ever notice, so + // it fails the run instead of being caught as a parse failure. + let providerCalls: ParsedProviderCall[] | undefined + if (offThread && workerPaths.has(source.path)) { + const result = (await offThread.next()).value + if (result?.ok && result.parsed) { + if (result.parsed.path !== source.path) { + throw new Error(`codex parse worker result out of order: got ${result.parsed.path}, expected ${source.path}`) + } + if (result.parsed.keys.some(k => parserDedup.has(k))) { + workerDiscards++ + } else { + for (const k of result.parsed.keys) parserDedup.add(k) + providerCalls = result.parsed.calls + // The worker never touches the codex cache; publish its entry here, + // in install order, so flushCodexCache writes what serial would. + const write = result.parsed.write + if (write) await writeCachedCodexResults(source.path, write.project, providerCalls, write.fingerprint, write.resume) } } + } + + try { if (!providerCalls) { const parser = provider.createSessionParser(source, parserDedup, dateRange) providerCalls = [] diff --git a/tests/parse-workers.test.ts b/tests/parse-workers.test.ts index fd44c98d..4185be59 100644 --- a/tests/parse-workers.test.ts +++ b/tests/parse-workers.test.ts @@ -31,6 +31,9 @@ describe('decideParseWorkers', () => { expect(decideParseWorkers(BIG_PENDING, { cores: 16, availableBytes: 6 * 1024 ** 3 }, NO_ENV).workers).toBe(6) // The smallest machine that clears every gate still only earns 2 threads expect(decideParseWorkers({ files: 200, bytes: 300 * 1024 ** 2 }, { cores: 3, availableBytes: 4 * 1024 ** 3 }, NO_ENV).workers).toBe(2) + // Big average file: the per-worker budget scales with it (2 x 260 MB + 128 MB), + // so the same 2 GB buys 3 threads instead of the 8 a flat 256 MB would. + expect(decideParseWorkers({ files: 250, bytes: 65 * 1024 ** 3 }, BIG_SYSTEM, NO_ENV).workers).toBe(3) // 300 files earn 6, but the 6 GB behind them earn 30 — bytes win, then the // memory budget caps it. A Codex corpus is exactly this shape. expect(decideParseWorkers({ files: 300, bytes: 6 * 1024 ** 3 }, BIG_SYSTEM, NO_ENV).workers).toBe(8) @@ -42,19 +45,19 @@ describe('decideParseWorkers', () => { expect(decideParseWorkers(BIG_PENDING, { cores: 2, availableBytes: 32 * 1024 ** 3 }, NO_ENV).workers).toBe(0) // A 4 GB box: availableMemory() always reads a little under the nominal size expect(decideParseWorkers(BIG_PENDING, { cores: 16, availableBytes: 3.9 * 1024 ** 3 }, NO_ENV).workers).toBe(0) - // Warm/incremental: neither gate reached + // Warm/incremental: the byte gate is not reached expect(decideParseWorkers({ files: 12, bytes: 10 * 1024 ** 2 }, BIG_SYSTEM, NO_ENV).workers).toBe(0) }) - it('takes files OR bytes, so a few huge rollouts still parallelise', () => { - // 150 rollouts of 4 GB: under the file gate, far over the byte gate + it('gates on bytes alone, so a thin corpus never spawns threads it cannot pay for', () => { + // 250 files holding under a megabyte between them: threads made this ~5% slower + expect(decideParseWorkers({ files: 250, bytes: 917 * 1024 }, BIG_SYSTEM, NO_ENV).workers).toBe(0) + expect(decideParseWorkers({ files: 5000, bytes: 10 * 1024 ** 2 }, BIG_SYSTEM, NO_ENV).workers).toBe(0) + expect(decideParseWorkers({ files: 250, bytes: 917 * 1024 }, BIG_SYSTEM, NO_ENV).reason) + .toContain('below 210 MB pending') + // 150 rollouts over the byte gate: far under any file-count threshold, and the + // biggest workload there is expect(decideParseWorkers({ files: 150, bytes: 4 * 1024 ** 3 }, BIG_SYSTEM, NO_ENV).workers).toBe(8) - // Many small files: under the byte gate, over the file gate - expect(decideParseWorkers({ files: 5000, bytes: 10 * 1024 ** 2 }, BIG_SYSTEM, NO_ENV).workers).toBe(8) - // Neither: still serial - expect(decideParseWorkers({ files: 150, bytes: 10 * 1024 ** 2 }, BIG_SYSTEM, NO_ENV).workers).toBe(0) - expect(decideParseWorkers({ files: 150, bytes: 10 * 1024 ** 2 }, BIG_SYSTEM, NO_ENV).reason) - .toContain('below 200 pending files and 210 MB pending') }) it('honours CODEBURN_PARSE_WORKERS, which also bypasses the auto gates', () => { @@ -431,8 +434,10 @@ describe('ParseWorkerPool', () => { const serial = await parseClaudeFileFull(files[0]!, new Set()) expect(fromWorker.ok).toBe(true) if (!fromWorker.ok || !fromWorker.parsed) throw new Error('expected a parsed result') - const { msgIds, ...worker } = fromWorker.parsed + const { msgIds, path, ...worker } = fromWorker.parsed expect(msgIds.length).toBeGreaterThan(0) + // Echoed back so the parent can assert the positional worker/file pairing. + expect(path).toBe(files[0]) expect(worker).toEqual(JSON.parse(JSON.stringify(serial))) }) @@ -453,9 +458,11 @@ describe('ParseWorkerPool', () => { const seen = new Set() const serial = await parseCodexFileFull(codexSource, seen) if (!fromWorker.ok || !fromWorker.parsed) throw new Error('expected a parsed result') - const { keys, ...worker } = fromWorker.parsed + const { keys, path, ...worker } = fromWorker.parsed expect(keys.length).toBeGreaterThan(0) expect(new Set(keys)).toEqual(seen) + // Echoed back so the parent can assert the positional worker/file pairing. + expect(path).toBe(codexPath) expect(worker).toEqual(JSON.parse(JSON.stringify(serial))) // The decode itself must never have touched the codex cache file. expect(await codexResults(join(home, '.cache', 'codeburn'))).toBeNull()