diff --git a/src/dashboard.tsx b/src/dashboard.tsx index d8c94a38..7396a2c1 100644 --- a/src/dashboard.tsx +++ b/src/dashboard.tsx @@ -6,7 +6,8 @@ import { render, Box, Text, measureElement, useInput, useApp, useWindowSize, typ import { CATEGORY_LABELS, type DateRange, type ProjectSummary, type TaskCategory } from './types.js' import { formatCost, formatTokens, markEstimated, carriedCostNote } from './format.js' import { aggregateModelEfficiency } from './model-efficiency.js' -import { parseAllSessions, filterProjectsByDateRange, filterProjectsByName, setInteractiveScanUI, withSinglePassParse } from './parser.js' +import { parseAllSessions, filterProjectsByDateRange, filterProjectsByName, setInteractiveScanUI, withSinglePassParse, withColdFirstPaintFloor, filesParsedFromSourceCount } from './parser.js' +import { isColdCacheOnDisk } from './session-cache.js' import { findUnpricedModels, isExpectedFreeModel, loadPricing } from './models.js' import { aggregateModelTotals } from './model-breakdown.js' import { buildDurablePeriod } from './usage-aggregator.js' @@ -32,6 +33,7 @@ export type DailyActivityRow = { } export const DAILY_ACTIVITY_PAGE_SIZE = 10 +export const INDEX_PROGRESS_TICK_MS = 1000 export const INTERACTIVE_RENDER_OPTIONS = { alternateScreen: true } as const export const RESIZE_DEBOUNCE_MS = 150 @@ -1465,7 +1467,7 @@ function ScrollableViewport({ children, width, lineScroll = true }: { children: ) } -export function InteractiveDashboard({ initialProjects, initialDailyHistoryProjects, initialPeriod, initialProvider, initialPlanUsages, initialDurable, refreshSeconds, projectFilter, excludeFilter, customRange, customRangeLabel, initialDay, windowColumns }: { +export function InteractiveDashboard({ initialProjects, initialDailyHistoryProjects, initialPeriod, initialProvider, initialPlanUsages, initialDurable, refreshSeconds, projectFilter, excludeFilter, customRange, customRangeLabel, initialDay, windowColumns, initialIndexPendingFiles }: { initialProjects: ProjectSummary[] initialDailyHistoryProjects?: ProjectSummary[] initialPeriod: Period @@ -1479,6 +1481,9 @@ export function InteractiveDashboard({ initialProjects, initialDailyHistoryProje customRangeLabel?: string initialDay?: string windowColumns: number + /// Files the cold first paint deferred (#1107). Non-zero means the numbers on + /// screen cover only what is indexed so far, and a background fill is owed. + initialIndexPendingFiles?: number }) { const { exit } = useApp() const [period, setPeriod] = useState(initialPeriod) @@ -1503,6 +1508,9 @@ export function InteractiveDashboard({ initialProjects, initialDailyHistoryProje // Which coaching note the footer shows; advanced on a slow interval so the // whole set rotates through without demanding attention. const [noteTick, setNoteTick] = useState(0) + const indexPendingFiles = initialIndexPendingFiles ?? 0 + const [indexing, setIndexing] = useState(indexPendingFiles > 0) + const [indexedFiles, setIndexedFiles] = useState(0) const isDayMode = dayDate != null const isCustomRange = customRange != null && !isDayMode const scrollableDailyHistory = !isCustomRange && !isDayMode @@ -1672,6 +1680,23 @@ export function InteractiveDashboard({ initialProjects, initialDailyHistoryProje return () => clearInterval(id) }, [refreshSeconds, period, activeProvider, dayDate, reloadData, view]) + // Background fill for the progressive cold start. The first paint deliberately + // skipped every file too old to show in the selected period; this is the pass + // that indexes them, and it is an ordinary unscoped reload, so what it writes + // to the caches — and what it puts back on screen — is exactly what a full + // cold parse would have produced. Mount-only: one fill per launch. + useEffect(() => { + if (indexPendingFiles === 0) return + const parsedBefore = filesParsedFromSourceCount() + const id = setInterval(() => setIndexedFiles(filesParsedFromSourceCount() - parsedBefore), INDEX_PROGRESS_TICK_MS) + void reloadData(initialPeriod, initialProvider, initialDay ?? null, true).finally(() => { + clearInterval(id) + setIndexing(false) + }) + return () => clearInterval(id) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + const switchPeriod = useCallback((np: Period) => { if (np === period && !dayDate) return // Clear projects + flip loading synchronously so the dashboard never @@ -1798,6 +1823,7 @@ export function InteractiveDashboard({ initialProjects, initialDailyHistoryProje {!isCustomRange && !isDayMode && } {isDayMode && } {isCustomRange && } + {indexing && } {view === 'compare' ? @@ -1817,6 +1843,7 @@ export function InteractiveDashboard({ initialProjects, initialDailyHistoryProje {!isCustomRange && !isDayMode && } {isDayMode && } {isCustomRange && } + {indexing && } {view === 'compare' ? setView('dashboard')} /> : view === 'optimize' && optimizeResult @@ -1842,6 +1869,20 @@ export function InteractiveDashboard({ initialProjects, initialDailyHistoryProje ) } +/// Honest partial state while the background fill runs: the panels below show +/// only what is indexed so far, and every period the fill has not reached yet +/// (month, 6 months, lifetime) reads short until it lands. +function IndexingBanner({ width, done, total }: { width: number; done: number; total: number }) { + return ( + + + indexing + history · {Math.min(done, total)}/{total} files · totals below cover what is indexed so far + + + ) +} + function DayBanner({ label, width }: { label: string; width: number }) { return ( @@ -1918,13 +1959,27 @@ export async function renderDashboard(period: Period = 'week', provider: string const isTTY = Boolean(process.stdin.isTTY && process.stdout.isTTY) const scrollableDailyHistory = isTTY && dayRange == null && customRange == null const range = getDashboardScanRange(period, customRange, initialDay ?? null, scrollableDailyHistory) - const { scannedProjects, filteredProjects, planUsages, initialDurable } = - await assembleDashboardData(period, provider, projectFilter, excludeFilter, customRange, initialDay ?? null, scrollableDailyHistory) + // Progressive cold start (#1107): the first paint of the standard dated view + // only needs the files that can hold data the selected period displays, so on + // a cold cache it parses those first and hands the rest to a background fill + // once the dashboard is on screen. Interactive only — every one-shot output + // (json/csv/markdown, report, sessions, the app and menubar payloads) keeps + // the full parse and can never return a partial total. + const progressive = isTTY && dayRange == null && customRange == null && await isColdCacheOnDisk() + const assemble = () => + assembleDashboardData(period, provider, projectFilter, excludeFilter, customRange, initialDay ?? null, scrollableDailyHistory) + const paint = progressive + ? await withColdFirstPaintFloor(getPeriodRange(period).start, assemble) + : { result: await assemble(), deferredFiles: 0 } + if (process.env['CODEBURN_VERBOSE'] === '1') { + process.stderr.write(`codeburn: progressive cold start ${progressive ? 'on' : 'off'}, ${paint.deferredFiles} files deferred to the background fill\n`) + } + const { scannedProjects, filteredProjects, planUsages, initialDurable } = paint.result const label = initialDay ? formatDayRangeLabel(initialDay) : customRangeLabel patchStdoutForWindows() if (isTTY) { const app = renderDebouncedInteractive(process.stdout, ({ columns }) => ( - + )) try { await app.waitUntilExit() diff --git a/src/parser.ts b/src/parser.ts index a35af849..1ee1cb15 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -2000,6 +2000,7 @@ async function scanProjectDirs( if (readOnly && action.action !== 'unchanged') readOnlyServedStale = true unchangedFiles.push({ filePath, dirName, source, cached: section.files[filePath]! }) } else if (!readOnly) { + if (deferToBackgroundFill(filePath, fp, cached)) continue if (action.action === 'appended') { changedFiles.push({ filePath, @@ -2097,6 +2098,7 @@ async function scanProjectDirs( try { for (const { filePath, info, append } of changedFiles) { + filesParsedFromSource++ // Marked here, not after the re-parse: an unreadable file `continue`s out // below, and the deletion would otherwise live only in memory. delete section.files[filePath] @@ -3290,6 +3292,7 @@ export async function parseProviderSources( if (readOnly && action.action !== 'unchanged') readOnlyServedStale = true unchangedSources.push({ source, cached }) } else if (!readOnly) { + if (deferToBackgroundFill(source.path, fp, cached)) continue changedSources.push({ source, fp }) } else { // Read-only with no cache entry at all — see scanProjectDirs. @@ -3387,6 +3390,7 @@ export async function parseProviderSources( if (dateRange) { if (fp.mtimeMs < dateRange.start.getTime()) continue } + filesParsedFromSource++ // Clear stale entry before parse — but only once per path so that // multiple sources mapping to the same file path can merge their turns. @@ -4207,7 +4211,12 @@ function cacheKey(dateRange: DateRange | undefined, providerFilter: string | und // Flat-rate marks do not change parse-time cost (still $0 without a LiteLLM // row); findUnpricedModels / coverage apply them at render time, so they // stay out of this serve-memo key on purpose. - return `${s}:${providerFilter ?? 'all'}:${claudeRoots}:${getProxyPathsConfigHash()}:${getModelAliasesConfigHash()}:${getPriceOverridesConfigHash()}:${getLocalModelSavingsConfigHash()}` + // A first-paint parse sees a deliberately smaller file set, so its result may + // not be served to (or burst-reused by) an unfloored request — including the + // background fill that follows it moments later. Absent outside the scope, so + // every non-cold-start key is byte-identical to what it was. + const floor = firstPaintFloorMs === null ? '' : `:paint${firstPaintFloorMs}` + return `${s}:${providerFilter ?? 'all'}:${claudeRoots}:${getProxyPathsConfigHash()}:${getModelAliasesConfigHash()}:${getPriceOverridesConfigHash()}:${getLocalModelSavingsConfigHash()}${floor}` } export function clearSessionCache(): void { @@ -4761,6 +4770,75 @@ function singlePassParse(dateRange: DateRange | undefined, providerFilter: strin return parsed.then(projects => filterProjectsByDateRange(projects, dateRange)) } +// Progressive cold start (#1107). A session log is append-only, so its last +// event timestamp is <= its mtime: a file whose mtime predates the start of the +// range being displayed provably holds nothing that range can show. On a COLD +// start that is what makes a fast first paint honest — the deferred files are +// not dropped, only sequenced behind the paint, and the background fill parses +// them into the same per-file cache a full cold parse would have written. +// +// The margin is pure paranoia about mtimes that lie: a restored backup, a +// machine whose clock jumped, an rsync that preserved a wrong stamp. It only +// widens the set that gets parsed BEFORE the paint, so it can never lose data. +export const FIRST_PAINT_MTIME_MARGIN_MS = 48 * 60 * 60 * 1000 + +let firstPaintFloorMs: number | null = null +// Files this scope deferred, as a SET of paths: one first paint runs several +// parses (the scan, the plan window, the durable backfill) and each defers the +// same old files, so a running count would report a multiple of the real work. +let firstPaintDeferredPaths: Set | null = null +// Same count, but reset per runParse: a run that deferred NOTHING did exactly +// what an unfloored run would have done, so it is allowed to mark the cache +// complete and report full hydration. +let firstPaintDeferredThisRun = 0 + +/** Files parsed from source (not served from cache) since the process started. + * The background-fill indicator reads it to show live N/M progress. */ +let filesParsedFromSource = 0 +export function filesParsedFromSourceCount(): number { + return filesParsedFromSource +} + +/** Restrict every parse inside `fn` to files that can hold in-range data for a + * view starting at `rangeStart`, and report how many files were deferred. + * Cold-start first paint only: the caller MUST follow up with an unscoped + * parse (the background fill) before the run can be treated as hydrated. */ +export async function withColdFirstPaintFloor( + rangeStart: Date, + fn: () => Promise, +): Promise<{ result: T; deferredFiles: number }> { + const outer = firstPaintFloorMs + const outerPaths = firstPaintDeferredPaths + firstPaintFloorMs = rangeStart.getTime() - FIRST_PAINT_MTIME_MARGIN_MS + firstPaintDeferredPaths = new Set() + try { + const result = await fn() + return { result, deferredFiles: firstPaintDeferredPaths.size } + } finally { + firstPaintFloorMs = outer + firstPaintDeferredPaths = outerPaths + } +} + +/** True when this file's whole-file parse can be deferred to the background + * fill. A file with a cache entry is never deferred: it has something to serve + * and re-reading it is incremental, so deferring would only make the served + * snapshot staler for no saving. */ +export function shouldDeferToBackgroundFill( + fp: { mtimeMs: number }, + cached: unknown, + floorMs: number | null, +): boolean { + return floorMs !== null && cached === undefined && fp.mtimeMs < floorMs +} + +function deferToBackgroundFill(path: string, fp: { mtimeMs: number }, cached: unknown): boolean { + if (!shouldDeferToBackgroundFill(fp, cached, firstPaintFloorMs)) return false + firstPaintDeferredPaths?.add(path) + firstPaintDeferredThisRun++ + return true +} + export function parseAllSessions(dateRange?: DateRange, providerFilter?: string): Promise { const scoped = singlePassParse(dateRange, providerFilter) if (scoped) return scoped @@ -4901,6 +4979,7 @@ async function runParseInner( const { isCold = false, readOnly = false, refreshLock } = options readOnlyServedStale = false deferredRetryableSource = false + firstPaintDeferredThisRun = 0 const seenMsgIds = new Set() const seenKeys = new Set() const allSources = await discoverAllSessions(providerFilter) @@ -5012,9 +5091,15 @@ async function runParseInner( // on every launch, and the completeness marker the daily backfill + splash rely // on is durable. A run killed before here never reaches this, so its throttled // partial saves keep `complete: false` and the next launch resumes cold. + // A first-paint run that deferred files never saw the whole corpus, so it may + // not stamp the cache complete — the next launch (or this run's own + // background fill) has to come back cold and finish the job. A floored run + // that deferred NOTHING parsed exactly what an unfloored run would have, so + // it keeps the normal stamp. + const deferredForFirstPaint = firstPaintDeferredThisRun > 0 const wasComplete = isCacheComplete(diskCache) - if (!readOnly && !wasComplete) diskCache.complete = true - if (!readOnly && (isCacheDirty(diskCache) || !wasComplete)) { + if (!readOnly && !wasComplete && !deferredForFirstPaint) diskCache.complete = true + if (!readOnly && (isCacheDirty(diskCache) || (!wasComplete && !deferredForFirstPaint))) { try { const published = await saveCache(diskCache, refreshLock?.verifyStillOwner) if (!published) throw new RefreshFenceLostError() @@ -5027,7 +5112,7 @@ async function runParseInner( // files, or a write run that deferred a changed source on a retryable // failure, reached the end of the scan without hydrating everything, and // the daily backfill must not finalize history off it. - sessionHydrationComplete = (!readOnly || !readOnlyServedStale) && !deferredRetryableSource + sessionHydrationComplete = (!readOnly || !readOnlyServedStale) && !deferredRetryableSource && !deferredForFirstPaint // Merge across providers by normalised project path so the same repository // is not double-counted when it was worked on with more than one tool diff --git a/src/session-cache.ts b/src/session-cache.ts index 736a6f52..30047f2e 100644 --- a/src/session-cache.ts +++ b/src/session-cache.ts @@ -528,6 +528,17 @@ export function isCacheComplete(cache: SessionCache): boolean { return cache.complete === true } +/** Pre-parse probe of the same question `isCacheComplete` answers after a load: + * is the next parse going to be a cold hydration? Reads only the (tiny) + * envelope, so a caller can branch on coldness before paying for the shards. + * A cache still in a legacy layout has no envelope and reads as cold — the + * adoption in `loadCache` may still make it warm, which costs the caller + * nothing: a warm cache has an entry for every discovered file, so a + * cold-start optimisation keyed on missing entries simply finds no work. */ +export async function isColdCacheOnDisk(): Promise { + return (await readEnvelope(sessionCacheDir()))?.complete !== true +} + function isNum(v: unknown): v is number { return typeof v === 'number' && Number.isFinite(v) } diff --git a/tests/progressive-cold-start.test.ts b/tests/progressive-cold-start.test.ts new file mode 100644 index 00000000..03c28c33 --- /dev/null +++ b/tests/progressive-cold-start.test.ts @@ -0,0 +1,175 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { mkdtemp, mkdir, writeFile, rm, utimes } from 'fs/promises' +import { join } from 'path' +import { tmpdir } from 'os' + +import { + parseAllSessions, + clearSessionCache, + withColdFirstPaintFloor, + shouldDeferToBackgroundFill, + filesParsedFromSourceCount, + isSessionHydrationComplete, + FIRST_PAINT_MTIME_MARGIN_MS, +} from '../src/parser.js' +import { clearLoadCacheMemo, isColdCacheOnDisk } from '../src/session-cache.js' +import { readCacheOnDisk } from './fixtures/session-cache-io.js' + +const DAY_MS = 24 * 60 * 60 * 1000 + +let tmpDir: string + +beforeEach(async () => { + clearSessionCache() + clearLoadCacheMemo() + tmpDir = await mkdtemp(join(tmpdir(), 'progressive-cold-')) + process.env['CLAUDE_CONFIG_DIR'] = tmpDir + process.env['CODEBURN_CACHE_DIR'] = join(tmpDir, 'cache') + process.env['CODEBURN_DESKTOP_SESSIONS_DIR'] = join(tmpDir, 'desktop-sessions') +}) + +afterEach(async () => { + clearSessionCache() + clearLoadCacheMemo() + await rm(tmpDir, { recursive: true, force: true }) +}) + +/** One Claude session file whose only turn is `ageDays` old, with an mtime to + * match — the shape the mtime argument is about. */ +async function writeSession(name: string, ageDays: number): Promise { + const dir = join(tmpDir, 'projects', 'proj') + await mkdir(dir, { recursive: true }) + const at = new Date(Date.now() - ageDays * DAY_MS) + const path = join(dir, `${name}.jsonl`) + await writeFile(path, JSON.stringify({ + type: 'assistant', + sessionId: name, + timestamp: at.toISOString(), + cwd: '/tmp/proj', + message: { + id: `msg-${name}`, type: 'message', role: 'assistant', model: 'claude-sonnet-4-5', + content: [], usage: { input_tokens: 100, output_tokens: 50 }, + }, + }) + '\n') + await utimes(path, at, at) + return path +} + +function lastWeek(): Date { + return new Date(Date.now() - 7 * DAY_MS) +} + +async function cachedClaudePaths(): Promise { + const raw = await readCacheOnDisk() + return Object.keys(raw.providers['claude']?.files ?? {}).sort() +} + +describe('first-paint deferral predicate', () => { + const floor = 1_000_000 + + it('defers only files strictly below the floor', () => { + expect(shouldDeferToBackgroundFill({ mtimeMs: floor - 1 }, undefined, floor)).toBe(true) + expect(shouldDeferToBackgroundFill({ mtimeMs: floor }, undefined, floor)).toBe(false) + expect(shouldDeferToBackgroundFill({ mtimeMs: floor + 1 }, undefined, floor)).toBe(false) + }) + + it('keeps a file the clock-skew margin rescues', () => { + // The floor the dashboard passes is rangeStart - MARGIN, so a file stamped + // up to MARGIN before the range start is still parsed for the first paint. + const rangeStart = 10 * DAY_MS + const skewed = rangeStart - FIRST_PAINT_MTIME_MARGIN_MS + 1 + expect(shouldDeferToBackgroundFill({ mtimeMs: skewed }, undefined, rangeStart - FIRST_PAINT_MTIME_MARGIN_MS)).toBe(false) + expect(shouldDeferToBackgroundFill({ mtimeMs: skewed - 2 }, undefined, rangeStart - FIRST_PAINT_MTIME_MARGIN_MS)).toBe(true) + }) + + it('never defers a file that has a cache entry, or any file outside the scope', () => { + const cached = { fingerprint: { dev: 0, ino: 0, mtimeMs: 0, sizeBytes: 0 }, mcpInventory: [], turns: [] } + expect(shouldDeferToBackgroundFill({ mtimeMs: floor - 1 }, cached, floor)).toBe(false) + // floor === null is every warm run and every one-shot command. + expect(shouldDeferToBackgroundFill({ mtimeMs: floor - 1 }, undefined, null)).toBe(false) + }) + + it('never defers a network source', () => { + // Network providers have no on-disk file; they enter the parse with a + // synthetic fingerprint stamped `Date.now()` (and skip this check entirely), + // so no floor a dated view can produce ever holds them back. + expect(shouldDeferToBackgroundFill({ mtimeMs: Date.now() }, undefined, lastWeek().getTime())).toBe(false) + }) +}) + +describe('progressive cold start', () => { + it('paints from the recent files and leaves the rest to the fill', async () => { + await writeSession('recent', 1) + await writeSession('old', 200) + + const { result, deferredFiles } = await withColdFirstPaintFloor(lastWeek(), () => parseAllSessions()) + expect(deferredFiles).toBe(1) + expect(result.flatMap(p => p.sessions).map(s => s.sessionId)).toEqual(['recent']) + // A pass that deferred files is a partial hydration: it must not stamp the + // session cache complete (the next launch has to come back cold) and must + // not let the daily backfill finalize history off it. + expect(isSessionHydrationComplete()).toBe(false) + expect(await isColdCacheOnDisk()).toBe(true) + // Durable partial progress: the file it DID parse is already cached. + expect(await cachedClaudePaths()).toEqual([join(tmpDir, 'projects', 'proj', 'recent.jsonl')]) + }) + + it('converges on the full-cold-parse state, without re-parsing what pass 1 did', async () => { + await writeSession('recent', 1) + await writeSession('old', 200) + + await withColdFirstPaintFloor(lastWeek(), () => parseAllSessions()) + clearSessionCache() + + // The background fill is an ordinary unscoped parse in the same process. + const parsedBefore = filesParsedFromSourceCount() + const filled = await parseAllSessions() + // Exactly one file parsed: the deferred one. The pass-1 file is served from + // its cache entry, so it is neither re-read nor counted twice. + expect(filesParsedFromSourceCount() - parsedBefore).toBe(1) + expect(filled.flatMap(p => p.sessions).map(s => s.sessionId).sort()).toEqual(['old', 'recent']) + expect(isSessionHydrationComplete()).toBe(true) + expect(await isColdCacheOnDisk()).toBe(false) + + const progressive = await readCacheOnDisk() + + // Same corpus, a plain full cold parse, in a cache dir of its own. + process.env['CODEBURN_CACHE_DIR'] = join(tmpDir, 'cache-baseline') + clearSessionCache() + clearLoadCacheMemo() + await parseAllSessions() + const baseline = await readCacheOnDisk() + + expect(progressive.providers['claude']?.files).toEqual(baseline.providers['claude']?.files) + expect(progressive.complete).toBe(baseline.complete) + }) + + it('resumes after a fill that never ran', async () => { + await writeSession('recent', 1) + await writeSession('old', 200) + + // Pass 1 only, then the process dies: what survives on disk is the recent + // file's entry and an INCOMPLETE marker. + await withColdFirstPaintFloor(lastWeek(), () => parseAllSessions()) + expect(await isColdCacheOnDisk()).toBe(true) + + // Next launch. The deferred file has no cache entry, so it is still + // discovered as changed and parsed; nothing is stranded behind a "seen" mark. + clearSessionCache() + clearLoadCacheMemo() + const resumed = await parseAllSessions() + expect(resumed.flatMap(p => p.sessions).map(s => s.sessionId).sort()).toEqual(['old', 'recent']) + expect(await isColdCacheOnDisk()).toBe(false) + }) + + it('is a no-op when every file is recent enough to paint', async () => { + await writeSession('recent', 1) + + const { deferredFiles } = await withColdFirstPaintFloor(lastWeek(), () => parseAllSessions()) + // Nothing deferred means this pass saw the whole corpus, so it keeps the + // ordinary completeness stamp instead of owing a fill. + expect(deferredFiles).toBe(0) + expect(isSessionHydrationComplete()).toBe(true) + expect(await isColdCacheOnDisk()).toBe(false) + }) +})