From 966454d7741bdeda9d21c098766b77666b23111a Mon Sep 17 00:00:00 2001 From: iamtoruk Date: Mon, 17 Aug 2026 00:09:38 -0700 Subject: [PATCH] perf(parser): classify only the turns that survive the date slice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scanProjectDirs ran every cached turn through cachedTurnToClassified — per-call reconstruction plus the turn classifier's category / retries / edit regexes — and only then applied the date slice, so a week view paid to classify all of history to keep a few percent of it. The keep/drop decision now runs on the raw CachedTurn (calls map 1:1 onto assistantCalls, so callsInRange sees the same survivors as the classified slicer) and only survivors are classified, still from their complete call list. The branch and PR-set carries still walk the full ordered turn list, and buildSpawnPrSets still reads the pre-slice turns. Real corpus, warm one-shot status --format menubar-json --no-optimize, identical payloads: today 1561ms -> 1318ms, week 1560ms -> 1410ms, month 1788ms -> 1675ms. --- CHANGELOG.md | 1 + src/parser.ts | 36 +++---- tests/parser-classify-after-slice.test.ts | 111 ++++++++++++++++++++++ 3 files changed, 131 insertions(+), 17 deletions(-) create mode 100644 tests/parser-classify-after-slice.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index fad4cf06..e18aabf9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Changed - **A warm launch rewrites only the provider that changed.** The session cache was a single blob, so any provider appending a few KB republished the whole thing — 147 MB of stringify + fsync on a 6 GB corpus, ~18% of a warm run. It is now a version-suffixed directory holding one shard per provider plus a small envelope, written per provider and published by a single envelope rename. An existing v7 cache is re-laid-out losslessly on first load and the old file removed once the new layout is on disk: nothing re-parses. One unreadable shard now costs that provider a re-parse instead of discarding every provider's history, and partial saves during a cold parse are triggered every 2000 files rather than every 5 seconds, so a slow cold parse no longer rewrites the growing cache on a wall clock. - **An appended Codex rollout parses only its tail.** Rollout files are append-only and the active ones run to hundreds of MB, but the Codex result cache keyed on mtime + size alone, so any growth re-read the file from byte 0. Each entry now records a restart point at the last task boundary — byte offset plus the state the single-pass decode carries across it — and a grown file with the same inode resumes there, producing output identical to a full re-parse. An entry without a usable restart point simply re-parses in full once and gains one. +- **A date-ranged report classifies only the turns it keeps.** Every cached turn went through the turn classifier — category, retries, edit detection, and a full reconstruction of its API calls — before the date slice discarded most of them, so a week view paid to classify all of history to keep a few percent of it. The keep/drop decision is now taken on the raw cached turn and only the survivors are classified, still from their complete call list, with the branch and pull-request carries still walking the full ordered turn list. Output is byte-identical. - **One rule for every cache file.** `CODEBURN_CACHE_DIR` when set, otherwise `~/.cache/codeburn`. `XDG_CACHE_HOME` is no longer consulted; the sync ledger, the only file that ever honored it, is merged into the canonical location on first read and the legacy copy is retired, so nothing is re-uploaded after the move. (#972) ### Fixed (Desktop & Menubar) diff --git a/src/parser.ts b/src/parser.ts index 03ea22f9..89f6e57d 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -2213,7 +2213,13 @@ async function scanProjectDirs( let carriedPrRefs: string[] | undefined let prRefsAtRangeStart: string[] | undefined let frozePrRefs = !dateRange - let classifiedTurns = cachedFile.turns.map(turn => { + // The keep/drop decision is taken on the RAW turn, before classifying it: + // `cachedTurnToClassified` maps `calls` 1:1 onto `assistantCalls`, so a turn + // with no call in range is dropped whole by the slicer below and classifying + // it is pure waste (on a week view that is nearly all of history). The + // carries above still run over the FULL ordered turn list. + const classifiedTurns: ClassifiedTurn[] = [] + for (const turn of cachedFile.turns) { if (turn.gitBranch) carriedBranch = turn.gitBranch if (dateRange && !frozePrRefs) { const firstTs = turn.calls[0]?.timestamp @@ -2223,10 +2229,16 @@ async function scanProjectDirs( } } if (turn.prRefs?.length) carriedPrRefs = turn.prRefs - return cachedTurnToClassified(turn, carriedBranch) - }) - // Captured from the FULL turn list, before the date slice below can drop the - // turn a branch was first seen on. Lets the by-branch report keep this + if (dateRange && !callsInRange(turn.calls, dateRange)) continue + const classified = cachedTurnToClassified(turn, carriedBranch) + // Slice rather than drop: a turn spanning local midnight would otherwise + // lose every call that lands in the requested day (issue #852). Only + // `assistantCalls`/`timestamp` are touched — see classifiedTurnSlicedToRange. + const sliced = dateRange ? classifiedTurnSlicedToRange(classified, dateRange) : classified + if (sliced) classifiedTurns.push(sliced) + } + // Captured from the FULL turn list, which the date slice above can strip of + // the turn a branch was first seen on. Lets the by-branch report keep this // session's in-range unbranched spend as `null` instead of discarding it. const everHadBranch = carriedBranch !== undefined @@ -2236,16 +2248,6 @@ async function scanProjectDirs( // sessions that both spawned subagents and referenced a PR. const spawnPrSets = cachedFile.prLinks?.length ? buildSpawnPrSets(cachedFile.turns) : {} - if (dateRange) { - // Slice rather than drop: a turn spanning local midnight would otherwise - // lose every call that lands in the requested day (issue #852). Only - // `assistantCalls`/`timestamp` are touched — see classifiedTurnSlicedToRange. - classifiedTurns = classifiedTurns.flatMap(turn => { - const sliced = classifiedTurnSlicedToRange(turn, dateRange) - return sliced ? [sliced] : [] - }) - } - // A PR-linked parent that spawned subagents is kept even when its OWN turns all // fall out of range, as a 0-cost fold ANCHOR: an in-range child (an async agent // that outlived the parent's last in-range turn) still needs the parent's @@ -2864,8 +2866,8 @@ function turnSlicedToRange(turn: CachedTurn, dateRange: DateRange): CachedTurn | return { ...turn, calls: inRangeCalls, timestamp: inRangeCalls[0]!.timestamp } } -// Same slice, applied post-classification (scanProjectDirs classifies every -// turn from its FULL call list up front, before date filtering — see the +// Same slice, applied post-classification (scanProjectDirs classifies each +// surviving turn from its FULL call list, before date filtering — see the // carriedBranch/carriedPrRefs comments in scanProjectDirs — so this only // trims `assistantCalls` and re-anchors `timestamp`; `category`/`subCategory`/ // `retries`/`hasEdits` stay exactly as classified from the complete turn. diff --git a/tests/parser-classify-after-slice.test.ts b/tests/parser-classify-after-slice.test.ts new file mode 100644 index 00000000..be48e0e7 --- /dev/null +++ b/tests/parser-classify-after-slice.test.ts @@ -0,0 +1,111 @@ +import { afterEach, beforeEach, expect, it } from 'vitest' +import { mkdtemp, mkdir, writeFile, rm } from 'fs/promises' +import { join } from 'path' +import { tmpdir } from 'os' + +import { parseAllSessions, filterProjectsByDateRange, clearSessionCache } from '../src/parser.js' +import { loadPricing } from '../src/models.js' +import type { ClassifiedTurn, DateRange } from '../src/types.js' + +// scanProjectDirs decides the date slice on the RAW cached turn and classifies +// only survivors. The classification itself must still see each surviving +// turn's COMPLETE call list, and the branch/PR carries must still run over the +// full ordered turn list — so this fixture puts the branch anchor and the PR +// reference before the range, and straddles the range start with a turn whose +// only Edit lands on the out-of-range side. + +const SESSION = '22222222-2222-4222-8222-222222222222' +const CWD = '/tmp/slice-proj' +const BRANCH = 'feat/carry' +const PR = 'https://github.com/o/r/pull/42' +const RANGE: DateRange = { + start: new Date('2026-07-20T00:00:00.000Z'), + end: new Date('2026-07-20T23:59:59.999Z'), +} + +let tmpDir: string + +beforeEach(async () => { + clearSessionCache() + tmpDir = await mkdtemp(join(tmpdir(), 'slice-')) + process.env['CLAUDE_CONFIG_DIR'] = join(tmpDir, 'claude') + process.env['CODEBURN_CACHE_DIR'] = join(tmpDir, 'cache') +}) + +afterEach(async () => { + clearSessionCache() + delete process.env['CLAUDE_CONFIG_DIR'] + delete process.env['CODEBURN_CACHE_DIR'] + await rm(tmpDir, { recursive: true, force: true }) +}) + +function user(ts: string, content: string): string { + return JSON.stringify({ type: 'user', sessionId: SESSION, timestamp: ts, cwd: CWD, gitBranch: BRANCH, message: { role: 'user', content } }) +} + +function assistant(ts: string, id: string, tools: string[]): string { + return JSON.stringify({ + type: 'assistant', sessionId: SESSION, timestamp: ts, cwd: CWD, gitBranch: BRANCH, + message: { + id, type: 'message', role: 'assistant', model: 'claude-sonnet-4-5', + content: tools.map((name, i) => ({ type: 'tool_use', id: `${id}_${i}`, name, input: {} })), + usage: { input_tokens: 100, output_tokens: 50 }, + }, + }) +} + +async function writeTranscript(): Promise { + const projDir = join(tmpDir, 'claude', 'projects', 'slice-proj') + await mkdir(projDir, { recursive: true }) + await writeFile(join(projDir, `${SESSION}.jsonl`), [ + // Before the range: the only turn carrying the branch (the cache elides an + // unchanged branch on later turns) and the only PR reference. + user('2026-07-19T09:00:00.000Z', `please finish ${PR}`), + assistant('2026-07-19T09:00:05.000Z', 'm1', ['Read']), + // Straddles the range start: the Edit is on the out-of-range call. + user('2026-07-19T23:50:00.000Z', 'keep going overnight'), + assistant('2026-07-19T23:50:10.000Z', 'm2', ['Edit']), + assistant('2026-07-20T00:10:00.000Z', 'm3', ['Read']), + // Fully inside the range. + user('2026-07-20T10:00:00.000Z', 'what changed?'), + assistant('2026-07-20T10:00:05.000Z', 'm4', ['Read']), + ].join('\n') + '\n', 'utf-8') +} + +function shape(turn: ClassifiedTurn): unknown { + return { + timestamp: turn.timestamp, + category: turn.category, + subCategory: turn.subCategory, + retries: turn.retries, + hasEdits: turn.hasEdits, + gitBranch: turn.gitBranch, + prRefs: turn.prRefs, + calls: turn.assistantCalls.map(c => c.timestamp), + } +} + +it('slices before classifying without changing carried branch, PR, or turn classification', async () => { + await loadPricing() + await writeTranscript() + + const sliced = await parseAllSessions(RANGE, 'claude') + // Reference: the old order — classify every turn from the full history, then + // apply the same range slice afterwards. + clearSessionCache() + const reference = filterProjectsByDateRange(await parseAllSessions(undefined, 'claude'), RANGE) + + const session = sliced[0]!.sessions[0]! + expect(session.turns.map(shape)).toEqual(reference[0]!.sessions[0]!.turns.map(shape)) + + // The branch anchor and the PR reference both live before the range. + expect(session.everHadBranch).toBe(true) + expect(session.turns.every(t => t.gitBranch === BRANCH)).toBe(true) + expect(session.prRefsAtRangeStart).toEqual([PR]) + + // The straddling turn kept only its in-range call, but was classified from + // the complete call list — the Edit it dropped still counts. + const straddled = session.turns[0]! + expect(straddled.assistantCalls.map(c => c.timestamp)).toEqual(['2026-07-20T00:10:00.000Z']) + expect(straddled.hasEdits).toBe(true) +})