From be1b212a3f6924372eea82b28b9daca75d0fa031 Mon Sep 17 00:00:00 2001 From: iamtoruk Date: Sun, 23 Aug 2026 04:49:58 -0700 Subject: [PATCH] feat: surface-wide progressive hydration via an explicit payload marker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TTY got the fast cold start in #1109; every JSON surface still waited for the full parse because a payload has no way to say "partial". Add one. - `hydration: { complete, indexedFiles, totalFiles }` on the menubar payload, add-only and emitted ONLY by the resident serve child (which polls, and therefore converges). Absence keeps meaning "complete", so every one-shot output is byte-identical and no script can be handed partial data. - serve: a cold child answers `status --format menubar-json` from the files the requested period can show (same mtime floor as the TUI) and schedules the unfloored fill behind it. Partial answers are never memoized; requests waiting behind the fill are heartbeated by id so the client's no-output watchdog stays armed. - `stale` keeps its own meaning: a first paint is fresh but partial, so it reports through `hydration` and never sets `stale`. - desktop app and web dash render an honest "indexing history · N/M files" notice while `complete` is false. --- app/renderer/App.tsx | 15 ++ app/renderer/lib/types.ts | 12 ++ dash/src/App.tsx | 16 +++ dash/src/lib/api.ts | 5 + src/menubar-json.ts | 23 +++ src/parser.ts | 29 ++++ src/serve.ts | 155 ++++++++++++++++++++- src/usage-aggregator.ts | 35 ++++- tests/serve-progressive-hydration.test.ts | 139 +++++++++++++++++++ tests/surface-hydration.test.ts | 162 ++++++++++++++++++++++ tests/usage-aggregator-freshness.test.ts | 6 + 11 files changed, 588 insertions(+), 9 deletions(-) create mode 100644 tests/serve-progressive-hydration.test.ts create mode 100644 tests/surface-hydration.test.ts diff --git a/app/renderer/App.tsx b/app/renderer/App.tsx index 8837ffc7..b7c5bc54 100644 --- a/app/renderer/App.tsx +++ b/app/renderer/App.tsx @@ -544,6 +544,7 @@ function AppMain() {
+ {page === 'usage' && } + {page === 'context' ? ( ) : showCombined ? ( diff --git a/dash/src/lib/api.ts b/dash/src/lib/api.ts index e7b5693d..f8da59e8 100644 --- a/dash/src/lib/api.ts +++ b/dash/src/lib/api.ts @@ -71,6 +71,10 @@ export type Current = { export type Payload = { generated: string + // Only a producer that its clients poll (the resident serve child) ever sends + // this, and only it may answer partially. `complete: false` means the totals + // cover the files indexed so far. Absence must be read as complete. + hydration?: { complete: boolean; indexedFiles: number; totalFiles: number } current: Current history: { daily: DailyEntry[]; timeline?: GranularHistory } } @@ -125,6 +129,7 @@ function normalizePayload(p?: Payload): Payload | undefined { } : undefined return { generated: p.generated, + ...(p.hydration ? { hydration: p.hydration } : {}), current: { label: c.label ?? '', cost: c.cost ?? 0, diff --git a/src/menubar-json.ts b/src/menubar-json.ts index 19d743fc..680dfabf 100644 --- a/src/menubar-json.ts +++ b/src/menubar-json.ts @@ -176,6 +176,16 @@ export type ClaudeConfigSelector = { options: ClaudeConfigOption[] } +/// How much of the corpus is behind the numbers in this payload (#1110). +/// `complete: false` means the totals cover only the files indexed so far and +/// a later poll will return more. The counts are progress indicators, not +/// inventory: they are only meaningful while `complete` is false. +export type HydrationState = { + complete: boolean + indexedFiles: number + totalFiles: number +} + export type MenubarPayload = { generated: string /// Optional. Present and `true` only when this payload was assembled from a @@ -184,6 +194,15 @@ export type MenubarPayload = { /// means "assume fresh," including for payloads from a CLI version that /// predates this field. stale?: boolean + /// Optional. Emitted ONLY by the resident `codeburn serve` child, the one + /// producer whose consumers poll and therefore converge. Every one-shot CLI + /// output omits it and is always a full parse, so absence must be read as + /// "complete" — including for payloads from a CLI that predates the field. + /// A consumer that renders totals MUST check this before presenting them as + /// final; it is the only in-band marker that separates a partial answer from + /// a converged one. Distinct from `stale`: a first paint is fresh but + /// partial, a stale payload is complete but old. + hydration?: HydrationState current: { label: string cost: number @@ -514,6 +533,7 @@ export function buildMenubarPayload( claudeConfigs?: ClaudeConfigSelector, granularHistory?: GranularHistory, stale?: boolean, + hydration?: HydrationState, ): MenubarPayload { const payload: MenubarPayload = { generated: new Date().toISOString(), @@ -567,5 +587,8 @@ export function buildMenubarPayload( if (stale) { payload.stale = true } + if (hydration) { + payload.hydration = hydration + } return payload } diff --git a/src/parser.ts b/src/parser.ts index 1ee1cb15..80842d72 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -4700,6 +4700,12 @@ export function isSessionHydrationComplete(): boolean { return sessionHydrationComplete } +// Why the most recent parse was incomplete, when it was: true only when the +// first-paint floor deferred files. Read by `sessionHydrationSnapshot` so the +// serve payload can label a converging first paint without also claiming the +// unrelated `stale` (read-only snapshot) condition. +let sessionFirstPaintDeferred = false + // Set by the read-only serving paths when the snapshot they served did NOT // match what is on disk: in read-only mode a changed file is served at its // stale fingerprint and a file with no cache entry is skipped entirely. A @@ -4799,6 +4805,28 @@ export function filesParsedFromSourceCount(): number { return filesParsedFromSource } +/** What the most recent parse left behind, for consumers that must present + * partiality honestly (#1110). `deferredForFirstPaint` is what separates a + * progressive cold start from the read-only stale case: both leave + * `complete` false, but only the latter is `stale` — a first paint is fresh + * data over a smaller file set, and it converges on its own. + * `indexedFiles` counts files parsed from source since this process started + * and `pendingFiles` the files the active first-paint scope deferred; both are + * progress numbers and only meaningful while `complete` is false. */ +export function sessionHydrationSnapshot(): { + complete: boolean + deferredForFirstPaint: boolean + indexedFiles: number + pendingFiles: number +} { + return { + complete: sessionHydrationComplete, + deferredForFirstPaint: sessionFirstPaintDeferred, + indexedFiles: filesParsedFromSource, + pendingFiles: firstPaintDeferredPaths?.size ?? 0, + } +} + /** Restrict every parse inside `fn` to files that can hold in-range data for a * view starting at `rangeStart`, and report how many files were deferred. * Cold-start first paint only: the caller MUST follow up with an unscoped @@ -5113,6 +5141,7 @@ async function runParseInner( // failure, reached the end of the scan without hydrating everything, and // the daily backfill must not finalize history off it. sessionHydrationComplete = (!readOnly || !readOnlyServedStale) && !deferredRetryableSource && !deferredForFirstPaint + sessionFirstPaintDeferred = deferredForFirstPaint // Merge across providers by normalised project path so the same repository // is not double-counted when it was worked on with more than one tool diff --git a/src/serve.ts b/src/serve.ts index 7b199570..cb8da96e 100644 --- a/src/serve.ts +++ b/src/serve.ts @@ -4,8 +4,10 @@ import { createHash } from 'crypto' import { createInterface } from 'readline' import type { Command } from 'commander' +import { getDateRange } from './cli-date.js' import { getConfigFilePath } from './config.js' import type { ParseReuseValidation } from './parser.js' +import { SERVE_HYDRATION_ENV } from './usage-aggregator.js' // --------------------------------------------------------------------------- // codeburn serve --stdio: a resident query server for the desktop app. @@ -41,6 +43,13 @@ const SERVE_MAX_RSS_BYTES = 3 * 1024 * 1024 * 1024 // prove the bound without a 45-second test. const DEFAULT_DRAIN_MS = 45_000 +// How long the background fill waits after a cold first paint before taking the +// queue. Long enough for the client's opening burst (it is serial, so its next +// request is already on the wire) to land in front of the fill, short enough +// that a client which then goes quiet still converges promptly. +// CODEBURN_SERVE_FILL_DELAY_MS overrides it for tests. +const DEFAULT_FILL_DELAY_MS = 3_000 + type OutputMemoEntry = { createdAt: number validatedFrom: number @@ -144,6 +153,60 @@ function allowed(args: string[]): boolean { return true } +/// Read a served option's value. `allowed()` has already proven the argv shape +/// (no positionals, every value-bearing option followed by a value that does +/// not start with '-'), so a token equal to the option name is always the +/// option and never someone's value. +function readServeOption(args: string[], name: string): string | undefined { + for (let i = 1; i < args.length; i++) { + const token = args[i]! + if (token === name) return args[i + 1] + if (token.startsWith(`${name}=`)) return token.slice(name.length + 1) + } + return undefined +} + +/// The range start a cold first paint of this request may be floored to, or +/// null when the request must not be answered partially (#1110). +/// +/// Only `status --format menubar-json` qualifies, because it is the ONLY served +/// output that carries the `hydration` marker. Every other served command +/// (`models`/`sessions`/`spend` --format json ...) is a one-shot shape whose +/// consumer has no in-band way to tell a partial answer from a final one, so it +/// keeps the full parse. Explicit `--day`/`--from`/`--to`/`--days` are excluded +/// for the same reason the TUI excludes them: the floor is derived from a named +/// period, and a custom range is a deliberate question that deserves a real +/// answer. +export function coldFirstPaintRangeStart( + args: string[], + rangeForPeriod: (period: string) => { range: { start: Date } }, +): Date | null { + if (args[0] !== 'status') return null + if (readServeOption(args, '--format') !== 'menubar-json') return null + for (const flag of ['--day', '--from', '--to', '--days']) { + if (readServeOption(args, flag) !== undefined) return null + } + // Mirrors the `status` --period default in main.ts. + const period = readServeOption(args, '--period') ?? 'today' + try { + return rangeForPeriod(period).range.start + } catch { + return null + } +} + +/// The request id of a protocol line, without committing to handling it. The +/// client's watchdog is a NO-OUTPUT timer, so a request waiting behind the +/// background fill has to be heartbeated by id before its turn comes. +function peekRequestId(line: string): string | number | null { + try { + const id = (JSON.parse(line) as { id?: unknown } | null)?.id + return typeof id === 'string' || typeof id === 'number' ? id : null + } catch { + return null + } +} + class ExitSignal extends Error { constructor(public readonly code: number) { super(`exit ${code}`) } } @@ -315,6 +378,12 @@ export async function runStdioServe(buildProgram: () => Command): Promise // re-running the discovery sweep per panel. Serve-only: one-shot CLI runs // never set this, so their results stay byte-exact. if (!process.env['CODEBURN_PARSE_BURST_MS']) process.env['CODEBURN_PARSE_BURST_MS'] = '10000' + // This process is the one producer allowed to answer partially, because its + // clients poll and therefore converge. The marker makes every menubar payload + // it emits carry `hydration` — including the warm ones, which report + // `complete: true` — so a consumer never has to infer completeness from a + // missing field within a single source. + process.env[SERVE_HYDRATION_ENV] = '1' // Event-driven reuse: while no watched session root has changed, a previous // parse stays valid past the burst window (capped in parser.ts, so a missed // filesystem event self-heals within minutes). This is what turns a warm @@ -367,10 +436,62 @@ export async function runStdioServe(buildProgram: () => Command): Promise // Strict serialization: each request chains on the previous one. let queue: Promise = Promise.resolve() + // Progressive cold start (#1110). Ids of requests received but not yet + // answered, so work that holds the queue can heartbeat them. + const awaiting = new Set() + const beat = (progress: string): void => { + for (const id of awaiting) write({ id, progress }) + } + // Cold detection runs at most once per paintable request until it answers + // "warm"; from then on this child is warm for good and every request takes + // the ordinary full path. + let firstPaintSettled = false + let fillTimer: ReturnType | undefined + + // The other half of the first paint: the same request, unfloored. It is an + // ordinary parse, so what it writes to the session and daily caches is + // exactly what a full cold parse would have written — the paint only + // sequenced those files behind the answer, it never dropped them. Killed + // mid-fill, nothing is stamped complete and the next launch re-enters cold. + const scheduleBackgroundFill = (args: string[]): void => { + if (fillTimer) return + const delayMs = Number(process.env['CODEBURN_SERVE_FILL_DELAY_MS']) || DEFAULT_FILL_DELAY_MS + fillTimer = setTimeout(() => { + queue = queue.then(async () => { + const { isColdCacheOnDisk } = await import('./session-cache.js') + // A request that landed in front of the fill may already have done the + // full parse (only the menubar payload is ever floored). + if (!await isColdCacheOnDisk()) return + const { startProgressKeepalive, stopProgressKeepalive } = await import('./parser.js') + startProgressKeepalive() + const startedAt = Date.now() + try { + const { output, code } = await runCaptured(buildProgram, args, beat) + const fingerprint = await getConfigFingerprint() + // Memoized so the poll that follows the fill answers instantly with + // the converged payload instead of re-deriving it. + if (code === 0 && fingerprint !== null) { + outputMemo.set(args.join('\u0000'), createOutputMemoEntry(startedAt, Date.now(), output, fingerprint)) + } + } catch { + // Best effort. A failed fill leaves the cache incomplete, which is + // exactly the state the next cold start knows how to resume from. + } finally { + stopProgressKeepalive() + } + }) + }, delayMs) + // The app owning this child is gone once stdin closes; an unstarted fill + // must not keep the process alive past that. + fillTimer.unref?.() + } + const rl = createInterface({ input: process.stdin, crlfDelay: Infinity }) rl.on('line', (line) => { const trimmed = line.trim() if (!trimmed) return + const waitingId = peekRequestId(trimmed) + if (waitingId !== null) awaiting.add(waitingId) queue = queue.then(async () => { let request: unknown try { @@ -408,22 +529,47 @@ export async function runStdioServe(buildProgram: () => Command): Promise write({ id: request.id, ok: true, output: memoHit.output }) return } + // Progressive cold start (#1110): on a cold cache the menubar payload is + // answered from the files the requested period can actually show, and the + // rest are indexed behind it. Same cold test as the TUI (session-cache + // envelope missing or complete !== true), same floor, and the answer says + // so in-band via `hydration.complete: false`. + const paintFrom = firstPaintSettled ? null : coldFirstPaintRangeStart(request.args, getDateRange) + let floor: Date | null = null + if (paintFrom) { + const { isColdCacheOnDisk } = await import('./session-cache.js') + floor = await isColdCacheOnDisk() ? paintFrom : null + firstPaintSettled = true + } // Heartbeat the WHOLE request, not just its parse: the aggregation and // payload serialization after a parse measured ~8s of further silence, and // it lands back-to-back with the parse's own quiet stretches. Wrapping // here (rather than at each command's exits) covers every served command // at one seam, and runCaptured routes the beats out as progress frames. - const { startProgressKeepalive, stopProgressKeepalive } = await import('./parser.js') + const { startProgressKeepalive, stopProgressKeepalive, withColdFirstPaintFloor } = await import('./parser.js') startProgressKeepalive() try { const parseStartedAt = Date.now() - const { output, code } = await runCaptured( + const run = () => runCaptured( buildProgram, request.args, progress => write({ id: request.id, progress }), ) + let deferredFiles = 0 + let result: Awaited> + if (floor) { + const painted = await withColdFirstPaintFloor(floor, run) + deferredFiles = painted.deferredFiles + result = painted.result + } else { + result = await run() + } + const { output, code } = result if (code === 0) { - if (configFingerprint !== null) { + // A partial answer is never memoized. The roots stay quiet while the + // fill converges, so a memo hit would pin the client to the first + // paint for the whole memo cap. + if (configFingerprint !== null && deferredFiles === 0) { outputMemo.set(memoKey, createOutputMemoEntry(parseStartedAt, Date.now(), output, configFingerprint)) } if (outputMemo.size > 32) { @@ -431,6 +577,7 @@ export async function runStdioServe(buildProgram: () => Command): Promise if (oldest) outputMemo.delete(oldest[0]) } write({ id: request.id, ok: true, output }) + if (deferredFiles > 0) scheduleBackgroundFill(request.args) } else write({ id: request.id, ok: false, error: `exit ${code}`, output }) } catch (err) { @@ -455,6 +602,8 @@ export async function runStdioServe(buildProgram: () => Command): Promise clearAntigravityCacheStates() if (typeof globalThis.gc === 'function') globalThis.gc() } + }).finally(() => { + if (waitingId !== null) awaiting.delete(waitingId) }) }) diff --git a/src/usage-aggregator.ts b/src/usage-aggregator.ts index 60a02429..5ed9976e 100644 --- a/src/usage-aggregator.ts +++ b/src/usage-aggregator.ts @@ -1,8 +1,8 @@ import { homedir } from 'node:os' import { CATEGORY_LABELS, type ProjectSummary, type TaskCategory, type DateRange } from './types.js' import { isBehavioralCall } from './behavioral-weight.js' -import { type PeriodData, type ProviderCost, type BreakdownArrays, type MenubarPayload, type ClaudeConfigSelector, buildMenubarPayload } from './menubar-json.js' -import { parseAllSessions, filterProjectsByName, filterProjectsByDays, filterProjectsByClaudeConfigSource, isSessionHydrationComplete } from './parser.js' +import { type PeriodData, type ProviderCost, type BreakdownArrays, type MenubarPayload, type ClaudeConfigSelector, type HydrationState, buildMenubarPayload } from './menubar-json.js' +import { parseAllSessions, filterProjectsByName, filterProjectsByDays, filterProjectsByClaudeConfigSource, isSessionHydrationComplete, sessionHydrationSnapshot } from './parser.js' import { findUnpricedModels, getFlatRateModelsConfigHash, getLocalModelSavingsConfigHash, getPriceOverridesConfigHash, getShortModelName, isExpectedFreeModel } from './models.js' import { getAllProviders, safeDiscoverSessions } from './providers/index.js' import { claude, getClaudeConfigDirs, getDesktopSessionsDirs } from './providers/claude.js' @@ -114,6 +114,23 @@ async function hydrateCache(): Promise { } } +/// The `hydration` block is emitted ONLY inside the resident serve child, which +/// sets this marker on itself at startup. That is the whole one-shot safety +/// rule in one place: a one-shot CLI process never sets it, so `--format +/// menubar-json` from a spawn (including the desktop app's spawn fallback, the +/// Swift menubar, and `codeburn web`) is byte-identical to before and can never +/// carry a partial-data label — it has no second poll to converge with. +export const SERVE_HYDRATION_ENV = 'CODEBURN_SERVE_HYDRATION' + +function hydrationStateFor(hydration: ReturnType | undefined): HydrationState | undefined { + if (process.env[SERVE_HYDRATION_ENV] !== '1' || !hydration) return undefined + return { + complete: hydration.complete, + indexedFiles: hydration.indexedFiles, + totalFiles: hydration.indexedFiles + hydration.pendingFiles, + } +} + export type PeriodInfo = { range: DateRange; label: string } export type AggregateOpts = { provider?: string @@ -573,14 +590,14 @@ export async function buildMenubarPayloadForRange(periodInfo: PeriodInfo, opts: // the ONLY safe read point for the module-level hydration global. Re-reading the // global later would race against this function's own history-block re-parse and // against concurrent requests (web-dashboard SWR, parallel MCP calls). - let hydrationComplete: boolean | undefined + let hydration: ReturnType | undefined let effectivelyScoped = false if (isClaudeConfigScoped) { // A config source scopes Claude usage only, so scan just Claude (main.ts // rejects a contradictory non-Claude --provider). This also avoids parsing // every other provider's corpus on each scoped refresh. const rawProjects = fp(await parseAllSessions(periodInfo.range, 'claude')) - hydrationComplete = isSessionHydrationComplete() + hydration = sessionHydrationSnapshot() const fullProjects = daysSelection ? filterProjectsByDays(rawProjects, daysSelection.days) : rawProjects claudeConfigs = await claudeConfigSelector(fullProjects, requestedClaudeConfigSourceId) const selectedSourceId = claudeConfigs?.selectedId ?? null @@ -607,7 +624,7 @@ export async function buildMenubarPayloadForRange(periodInfo: PeriodInfo, opts: exclude: opts.exclude, daysSelection, }) - hydrationComplete = isSessionHydrationComplete() + hydration = sessionHydrationSnapshot() currentData = durable.data scanProjects = durable.liveProjects scanRange = durable.scanRange @@ -951,5 +968,11 @@ export async function buildMenubarPayloadForRange(periodInfo: PeriodInfo, opts: const optimize = opts.optimize === false ? null : await scanAndDetect(scanProjects, scanRange, opts.provider) const granularRange = opts.daysSelection?.range ?? scanRange const granularHistory = opts.timeline === false ? undefined : buildGranularHistory(scanProjects, granularRange) - return buildMenubarPayload(currentData, providers, optimize, dailyHistory, retryTax, routingWaste, breakdowns, claudeConfigs, granularHistory, hydrationComplete === false ? true : undefined) + // `stale` keeps its original meaning: a read-only serve that could not see + // real files. A first paint is incomplete for a different reason (files + // deliberately sequenced behind it) and reports that through `hydration` + // instead, so the two are never conflated. + const partialFirstPaint = hydration?.deferredForFirstPaint === true + const stale = hydration?.complete === false && !partialFirstPaint ? true : undefined + return buildMenubarPayload(currentData, providers, optimize, dailyHistory, retryTax, routingWaste, breakdowns, claudeConfigs, granularHistory, stale, hydrationStateFor(hydration)) } diff --git a/tests/serve-progressive-hydration.test.ts b/tests/serve-progressive-hydration.test.ts new file mode 100644 index 00000000..ca4d4687 --- /dev/null +++ b/tests/serve-progressive-hydration.test.ts @@ -0,0 +1,139 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest' +import { spawn, type ChildProcess } from 'child_process' +import { mkdtemp, mkdir, writeFile, rm, utimes } from 'fs/promises' +import { join } from 'path' +import { tmpdir } from 'os' + +import { parseAllSessions, clearSessionCache } from '../src/parser.js' +import { clearLoadCacheMemo } from '../src/session-cache.js' +import { readCacheOnDisk } from './fixtures/session-cache-io.js' + +const DAY_MS = 24 * 60 * 60 * 1000 + +// End-to-end for the progressive cold start of the resident serve child +// (#1110): a cold cache is answered from the files the requested period can +// show, the answer says so in-band, and the background fill converges the +// on-disk cache to exactly what a full cold parse would have written. +describe('codeburn serve --stdio progressive cold start', () => { + let home: string + let child: ChildProcess + let buffer = '' + const waiters = new Map) => void>() + let readyResolve: () => void + const ready = new Promise(resolve => { readyResolve = resolve }) + + function request(id: number, args: string[]): Promise> { + return new Promise(resolve => { + waiters.set(id, resolve) + child.stdin!.write(JSON.stringify({ id, args }) + '\n') + }) + } + + async function session(name: string, ageDays: number): Promise { + const dir = join(home, '.claude', 'projects', 'proj') + await mkdir(dir, { recursive: true }) + const at = new Date(Date.now() - ageDays * DAY_MS) + const path = join(dir, `${name}.jsonl`) + await writeFile(path, JSON.stringify({ + type: 'assistant', + sessionId: name, + timestamp: at.toISOString(), + cwd: '/tmp/proj', + message: { + id: `msg-${name}`, type: 'message', role: 'assistant', model: 'claude-sonnet-4-5', + content: [], usage: { input_tokens: 100, output_tokens: 50 }, + }, + }) + '\n') + await utimes(path, at, at) + } + + const PAYLOAD_ARGS = ['status', '--format', 'menubar-json', '--period', 'week', '--no-timeline', '--no-optimize'] + + beforeAll(async () => { + home = await mkdtemp(join(tmpdir(), 'serve-progressive-')) + await session('recent', 1) + await session('old', 200) + + child = spawn(process.execPath, ['--import', 'tsx', join(__dirname, '..', 'src', 'cli.ts'), 'serve', '--stdio'], { + stdio: ['pipe', 'pipe', 'ignore'], + env: { + ...process.env, + HOME: home, + CLAUDE_CONFIG_DIR: join(home, '.claude'), + CODEBURN_CACHE_DIR: join(home, 'cache'), + CODEBURN_DESKTOP_SESSIONS_DIR: join(home, 'desktop-sessions'), + // The fill normally waits for the client's opening burst to land. + CODEBURN_SERVE_FILL_DELAY_MS: '200', + }, + }) + child.stdout!.setEncoding('utf8') + child.stdout!.on('data', (chunk: string) => { + buffer += chunk + let idx: number + while ((idx = buffer.indexOf('\n')) >= 0) { + const line = buffer.slice(0, idx).trim() + buffer = buffer.slice(idx + 1) + if (!line) continue + const msg = JSON.parse(line) as Record + if (msg['ready']) { readyResolve(); continue } + // Progress frames keep a waiting client's watchdog armed; they are not + // the answer. + if (typeof msg['progress'] === 'string') continue + const waiter = waiters.get(msg['id'] as number) + if (waiter) { waiters.delete(msg['id'] as number); waiter(msg) } + } + }) + await ready + }, 60_000) + + afterAll(async () => { + child?.kill('SIGKILL') + await rm(home, { recursive: true, force: true }) + }) + + it('answers cold with labelled partial data, then converges to complete', async () => { + const first = await request(1, PAYLOAD_ARGS) + expect(first['ok']).toBe(true) + const partial = JSON.parse(first['output'] as string) as { + hydration?: { complete: boolean; indexedFiles: number; totalFiles: number } + stale?: boolean + } + // Explicit partiality: a consumer can tell this apart from a final answer + // programmatically, and it is NOT the unrelated `stale` claim. + expect(partial.hydration?.complete).toBe(false) + expect(partial.hydration!.totalFiles).toBeGreaterThan(partial.hydration!.indexedFiles) + expect(partial.stale).toBeUndefined() + + // Poll the way every consumer does. The partial answer is never memoized, + // so a later poll re-derives (or picks up the fill's converged payload). + let converged: { hydration?: { complete: boolean; indexedFiles: number; totalFiles: number } } | null = null + for (let id = 2; id < 40; id++) { + const response = await request(id, PAYLOAD_ARGS) + const payload = JSON.parse(response['output'] as string) + if (payload.hydration?.complete === true) { converged = payload; break } + await new Promise(resolve => setTimeout(resolve, 250)) + } + expect(converged).not.toBeNull() + expect(converged!.hydration!.indexedFiles).toBe(converged!.hydration!.totalFiles) + }, 60_000) + + it('leaves the on-disk cache exactly where a full cold parse would', async () => { + process.env['CODEBURN_CACHE_DIR'] = join(home, 'cache') + clearSessionCache() + clearLoadCacheMemo() + const converged = await readCacheOnDisk() + expect(converged.complete).toBe(true) + + // Same corpus, one plain cold parse, in a cache dir of its own. + process.env['CLAUDE_CONFIG_DIR'] = join(home, '.claude') + process.env['CODEBURN_CACHE_DIR'] = join(home, 'cache-baseline') + process.env['CODEBURN_DESKTOP_SESSIONS_DIR'] = join(home, 'desktop-sessions') + clearSessionCache() + clearLoadCacheMemo() + await parseAllSessions() + const baseline = await readCacheOnDisk() + + expect(converged.providers['claude']?.files).toEqual(baseline.providers['claude']?.files) + expect(converged.complete).toBe(baseline.complete) + }, 60_000) +}) diff --git a/tests/surface-hydration.test.ts b/tests/surface-hydration.test.ts new file mode 100644 index 00000000..815a4afd --- /dev/null +++ b/tests/surface-hydration.test.ts @@ -0,0 +1,162 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { mkdtemp, mkdir, writeFile, rm, utimes } from 'fs/promises' +import { join } from 'path' +import { tmpdir } from 'os' + +import { parseAllSessions, clearSessionCache, withColdFirstPaintFloor } from '../src/parser.js' +import { clearLoadCacheMemo, isColdCacheOnDisk } from '../src/session-cache.js' +import { buildMenubarPayloadForRange, SERVE_HYDRATION_ENV } from '../src/usage-aggregator.js' +import { getDateRange } from '../src/cli-date.js' +import { coldFirstPaintRangeStart } from '../src/serve.js' + +const DAY_MS = 24 * 60 * 60 * 1000 + +let tmpDir: string + +beforeEach(async () => { + clearSessionCache() + clearLoadCacheMemo() + tmpDir = await mkdtemp(join(tmpdir(), 'surface-hydration-')) + process.env['CLAUDE_CONFIG_DIR'] = tmpDir + process.env['CODEBURN_CACHE_DIR'] = join(tmpDir, 'cache') + process.env['CODEBURN_DESKTOP_SESSIONS_DIR'] = join(tmpDir, 'desktop-sessions') + delete process.env[SERVE_HYDRATION_ENV] +}) + +afterEach(async () => { + clearSessionCache() + clearLoadCacheMemo() + delete process.env[SERVE_HYDRATION_ENV] + await rm(tmpDir, { recursive: true, force: true }) +}) + +async function writeSession(name: string, ageDays: number): Promise { + const dir = join(tmpDir, 'projects', 'proj') + await mkdir(dir, { recursive: true }) + const at = new Date(Date.now() - ageDays * DAY_MS) + const path = join(dir, `${name}.jsonl`) + await writeFile(path, JSON.stringify({ + type: 'assistant', + sessionId: name, + timestamp: at.toISOString(), + cwd: '/tmp/proj', + message: { + id: `msg-${name}`, type: 'message', role: 'assistant', model: 'claude-sonnet-4-5', + content: [], usage: { input_tokens: 100, output_tokens: 50 }, + }, + }) + '\n') + await utimes(path, at, at) +} + +const weekPayload = () => buildMenubarPayloadForRange(getDateRange('week'), { optimize: false, timeline: false }) + +describe('serve first-paint gate', () => { + const start = (args: string[]) => coldFirstPaintRangeStart(args, getDateRange) + + it('accepts only the menubar payload, the one served output that carries the marker', () => { + expect(start(['status', '--format', 'menubar-json'])).toBeInstanceOf(Date) + expect(start(['status', '--format=menubar-json', '--no-timeline'])).toBeInstanceOf(Date) + // Every other served command is a one-shot shape: its consumer has no + // in-band way to tell a partial answer from a final one. + expect(start(['status', '--format', 'json'])).toBeNull() + expect(start(['status'])).toBeNull() + expect(start(['models', '--format', 'json'])).toBeNull() + expect(start(['sessions', '--format', 'json'])).toBeNull() + }) + + it('floors to the requested period and defaults to the CLI default', () => { + const week = start(['status', '--format', 'menubar-json', '--period', 'week']) + expect(week?.getTime()).toBe(getDateRange('week').range.start.getTime()) + expect(start(['status', '--format', 'menubar-json'])?.getTime()) + .toBe(getDateRange('today').range.start.getTime()) + expect(start(['status', '--format', 'menubar-json', '--period', 'nonsense'])).toBeNull() + }) + + it('declines an explicit range, which is a question that deserves a real answer', () => { + expect(start(['status', '--format', 'menubar-json', '--day', '2026-01-01'])).toBeNull() + expect(start(['status', '--format', 'menubar-json', '--from', '2026-01-01'])).toBeNull() + expect(start(['status', '--format', 'menubar-json', '--to', '2026-01-01'])).toBeNull() + expect(start(['status', '--format', 'menubar-json', '--days', '2026-01-01,2026-01-02'])).toBeNull() + }) +}) + +describe('hydration in the menubar payload', () => { + it('labels a floored first paint partial, and never as stale', async () => { + await writeSession('recent', 1) + await writeSession('old', 200) + process.env[SERVE_HYDRATION_ENV] = '1' + + const { result: payload, deferredFiles } = await withColdFirstPaintFloor( + getDateRange('week').range.start, + weekPayload, + ) + expect(deferredFiles).toBe(1) + expect(payload.hydration).toEqual({ complete: false, indexedFiles: expect.any(Number), totalFiles: expect.any(Number) }) + expect(payload.hydration!.totalFiles).toBeGreaterThan(payload.hydration!.indexedFiles) + // `stale` is a different claim (a read-only snapshot that could not see + // real files) and must not ride along with a converging first paint. + expect(payload.stale).toBeUndefined() + }) + + it('reports complete once the fill has run', async () => { + await writeSession('recent', 1) + await writeSession('old', 200) + process.env[SERVE_HYDRATION_ENV] = '1' + + await withColdFirstPaintFloor(getDateRange('week').range.start, weekPayload) + clearSessionCache() + const payload = await weekPayload() + + expect(payload.hydration?.complete).toBe(true) + expect(payload.stale).toBeUndefined() + expect(await isColdCacheOnDisk()).toBe(false) + }) + + it('omits the field entirely outside the resident serve process', async () => { + await writeSession('recent', 1) + await writeSession('old', 200) + + // The one-shot shape: a full parse, and no field for a script to mistake + // for a completeness claim either way. + const oneShot = await weekPayload() + expect(oneShot.hydration).toBeUndefined() + expect(await isColdCacheOnDisk()).toBe(false) + + // Even under a floor, an unmarked process emits no hydration block — the + // floor is only ever set by the serve child, which does set the marker. + clearSessionCache() + clearLoadCacheMemo() + process.env['CODEBURN_CACHE_DIR'] = join(tmpDir, 'cache-2') + const floored = await withColdFirstPaintFloor(getDateRange('week').range.start, weekPayload) + expect(floored.deferredFiles).toBe(1) + expect(floored.result.hydration).toBeUndefined() + }) + + it('reports complete for a floored run that had nothing to defer', async () => { + await writeSession('recent', 1) + process.env[SERVE_HYDRATION_ENV] = '1' + + const { result: payload, deferredFiles } = await withColdFirstPaintFloor( + getDateRange('week').range.start, + weekPayload, + ) + expect(deferredFiles).toBe(0) + expect(payload.hydration?.complete).toBe(true) + }) + + it('keeps the spawn-fallback path on a full parse', async () => { + await writeSession('recent', 1) + await writeSession('old', 200) + process.env[SERVE_HYDRATION_ENV] = '1' + + // The spawn fallback runs the same command with no floor: it is a one-shot + // that cannot converge, so it must see every file. + const payload = await weekPayload() + expect(payload.hydration?.complete).toBe(true) + // Nothing pending means indexed === total, whatever the process-lifetime + // counter happens to stand at. + expect(payload.hydration?.indexedFiles).toBe(payload.hydration?.totalFiles) + const sessions = (await parseAllSessions()).flatMap(p => p.sessions).map(s => s.sessionId).sort() + expect(sessions).toEqual(['old', 'recent']) + }) +}) diff --git a/tests/usage-aggregator-freshness.test.ts b/tests/usage-aggregator-freshness.test.ts index c09b4f54..cfa8ba2a 100644 --- a/tests/usage-aggregator-freshness.test.ts +++ b/tests/usage-aggregator-freshness.test.ts @@ -100,6 +100,12 @@ vi.mock('../src/parser.js', async (importOriginal) => { return fixtureProjects() }), isSessionHydrationComplete: vi.fn(() => parseCalls === 1), + sessionHydrationSnapshot: vi.fn(() => ({ + complete: parseCalls === 1, + deferredForFirstPaint: false, + indexedFiles: parseCalls, + pendingFiles: 0, + })), } })