mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-08-27 17:32:46 +00:00
Merge pull request #1109 from getagentseal/perf/1107-progressive-cold
perf: progressive cold start — paint the dated TUI first, index history in the background
This commit is contained in:
commit
796a504ece
4 changed files with 335 additions and 9 deletions
|
|
@ -6,7 +6,8 @@ import { render, Box, Text, measureElement, useInput, useApp, useWindowSize, typ
|
|||
import { CATEGORY_LABELS, type DateRange, type ProjectSummary, type TaskCategory } from './types.js'
|
||||
import { formatCost, formatTokens, markEstimated, carriedCostNote } from './format.js'
|
||||
import { aggregateModelEfficiency } from './model-efficiency.js'
|
||||
import { parseAllSessions, filterProjectsByDateRange, filterProjectsByName, setInteractiveScanUI, withSinglePassParse } from './parser.js'
|
||||
import { parseAllSessions, filterProjectsByDateRange, filterProjectsByName, setInteractiveScanUI, withSinglePassParse, withColdFirstPaintFloor, filesParsedFromSourceCount } from './parser.js'
|
||||
import { isColdCacheOnDisk } from './session-cache.js'
|
||||
import { findUnpricedModels, isExpectedFreeModel, loadPricing } from './models.js'
|
||||
import { aggregateModelTotals } from './model-breakdown.js'
|
||||
import { buildDurablePeriod } from './usage-aggregator.js'
|
||||
|
|
@ -32,6 +33,7 @@ export type DailyActivityRow = {
|
|||
}
|
||||
|
||||
export const DAILY_ACTIVITY_PAGE_SIZE = 10
|
||||
export const INDEX_PROGRESS_TICK_MS = 1000
|
||||
export const INTERACTIVE_RENDER_OPTIONS = { alternateScreen: true } as const
|
||||
export const RESIZE_DEBOUNCE_MS = 150
|
||||
|
||||
|
|
@ -1465,7 +1467,7 @@ function ScrollableViewport({ children, width, lineScroll = true }: { children:
|
|||
)
|
||||
}
|
||||
|
||||
export function InteractiveDashboard({ initialProjects, initialDailyHistoryProjects, initialPeriod, initialProvider, initialPlanUsages, initialDurable, refreshSeconds, projectFilter, excludeFilter, customRange, customRangeLabel, initialDay, windowColumns }: {
|
||||
export function InteractiveDashboard({ initialProjects, initialDailyHistoryProjects, initialPeriod, initialProvider, initialPlanUsages, initialDurable, refreshSeconds, projectFilter, excludeFilter, customRange, customRangeLabel, initialDay, windowColumns, initialIndexPendingFiles }: {
|
||||
initialProjects: ProjectSummary[]
|
||||
initialDailyHistoryProjects?: ProjectSummary[]
|
||||
initialPeriod: Period
|
||||
|
|
@ -1479,6 +1481,9 @@ export function InteractiveDashboard({ initialProjects, initialDailyHistoryProje
|
|||
customRangeLabel?: string
|
||||
initialDay?: string
|
||||
windowColumns: number
|
||||
/// Files the cold first paint deferred (#1107). Non-zero means the numbers on
|
||||
/// screen cover only what is indexed so far, and a background fill is owed.
|
||||
initialIndexPendingFiles?: number
|
||||
}) {
|
||||
const { exit } = useApp()
|
||||
const [period, setPeriod] = useState<Period>(initialPeriod)
|
||||
|
|
@ -1503,6 +1508,9 @@ export function InteractiveDashboard({ initialProjects, initialDailyHistoryProje
|
|||
// Which coaching note the footer shows; advanced on a slow interval so the
|
||||
// whole set rotates through without demanding attention.
|
||||
const [noteTick, setNoteTick] = useState(0)
|
||||
const indexPendingFiles = initialIndexPendingFiles ?? 0
|
||||
const [indexing, setIndexing] = useState(indexPendingFiles > 0)
|
||||
const [indexedFiles, setIndexedFiles] = useState(0)
|
||||
const isDayMode = dayDate != null
|
||||
const isCustomRange = customRange != null && !isDayMode
|
||||
const scrollableDailyHistory = !isCustomRange && !isDayMode
|
||||
|
|
@ -1672,6 +1680,23 @@ export function InteractiveDashboard({ initialProjects, initialDailyHistoryProje
|
|||
return () => clearInterval(id)
|
||||
}, [refreshSeconds, period, activeProvider, dayDate, reloadData, view])
|
||||
|
||||
// Background fill for the progressive cold start. The first paint deliberately
|
||||
// skipped every file too old to show in the selected period; this is the pass
|
||||
// that indexes them, and it is an ordinary unscoped reload, so what it writes
|
||||
// to the caches — and what it puts back on screen — is exactly what a full
|
||||
// cold parse would have produced. Mount-only: one fill per launch.
|
||||
useEffect(() => {
|
||||
if (indexPendingFiles === 0) return
|
||||
const parsedBefore = filesParsedFromSourceCount()
|
||||
const id = setInterval(() => setIndexedFiles(filesParsedFromSourceCount() - parsedBefore), INDEX_PROGRESS_TICK_MS)
|
||||
void reloadData(initialPeriod, initialProvider, initialDay ?? null, true).finally(() => {
|
||||
clearInterval(id)
|
||||
setIndexing(false)
|
||||
})
|
||||
return () => clearInterval(id)
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [])
|
||||
|
||||
const switchPeriod = useCallback((np: Period) => {
|
||||
if (np === period && !dayDate) return
|
||||
// Clear projects + flip loading synchronously so the dashboard never
|
||||
|
|
@ -1798,6 +1823,7 @@ export function InteractiveDashboard({ initialProjects, initialDailyHistoryProje
|
|||
{!isCustomRange && !isDayMode && <PeriodTabs active={period} providerName={activeProvider} showProvider={view !== 'compare' && multipleProviders} />}
|
||||
{isDayMode && <DayBanner label={headerLabel} width={dashWidth} />}
|
||||
{isCustomRange && <CustomRangeBanner label={headerLabel} width={dashWidth} />}
|
||||
{indexing && <IndexingBanner width={dashWidth} done={indexedFiles} total={indexPendingFiles} />}
|
||||
{view === 'compare'
|
||||
? <Box flexDirection="column" paddingX={2} paddingY={1}>
|
||||
<Box flexDirection="column" borderStyle="round" borderColor={ORANGE} paddingX={1}>
|
||||
|
|
@ -1817,6 +1843,7 @@ export function InteractiveDashboard({ initialProjects, initialDailyHistoryProje
|
|||
{!isCustomRange && !isDayMode && <PeriodTabs active={period} providerName={activeProvider} showProvider={multipleProviders && view !== 'compare'} />}
|
||||
{isDayMode && <DayBanner label={headerLabel} width={dashWidth} />}
|
||||
{isCustomRange && <CustomRangeBanner label={headerLabel} width={dashWidth} />}
|
||||
{indexing && <IndexingBanner width={dashWidth} done={indexedFiles} total={indexPendingFiles} />}
|
||||
{view === 'compare'
|
||||
? <CompareView projects={projects} onBack={() => setView('dashboard')} />
|
||||
: view === 'optimize' && optimizeResult
|
||||
|
|
@ -1842,6 +1869,20 @@ export function InteractiveDashboard({ initialProjects, initialDailyHistoryProje
|
|||
)
|
||||
}
|
||||
|
||||
/// Honest partial state while the background fill runs: the panels below show
|
||||
/// only what is indexed so far, and every period the fill has not reached yet
|
||||
/// (month, 6 months, lifetime) reads short until it lands.
|
||||
function IndexingBanner({ width, done, total }: { width: number; done: number; total: number }) {
|
||||
return (
|
||||
<Box width={width} paddingX={1}>
|
||||
<Text wrap="truncate-end">
|
||||
<Text color={ORANGE} bold>indexing </Text>
|
||||
<Text dimColor>history · {Math.min(done, total)}/{total} files · totals below cover what is indexed so far</Text>
|
||||
</Text>
|
||||
</Box>
|
||||
)
|
||||
}
|
||||
|
||||
function DayBanner({ label, width }: { label: string; width: number }) {
|
||||
return (
|
||||
<Box width={width} paddingX={1} marginBottom={1}>
|
||||
|
|
@ -1918,13 +1959,27 @@ export async function renderDashboard(period: Period = 'week', provider: string
|
|||
const isTTY = Boolean(process.stdin.isTTY && process.stdout.isTTY)
|
||||
const scrollableDailyHistory = isTTY && dayRange == null && customRange == null
|
||||
const range = getDashboardScanRange(period, customRange, initialDay ?? null, scrollableDailyHistory)
|
||||
const { scannedProjects, filteredProjects, planUsages, initialDurable } =
|
||||
await assembleDashboardData(period, provider, projectFilter, excludeFilter, customRange, initialDay ?? null, scrollableDailyHistory)
|
||||
// Progressive cold start (#1107): the first paint of the standard dated view
|
||||
// only needs the files that can hold data the selected period displays, so on
|
||||
// a cold cache it parses those first and hands the rest to a background fill
|
||||
// once the dashboard is on screen. Interactive only — every one-shot output
|
||||
// (json/csv/markdown, report, sessions, the app and menubar payloads) keeps
|
||||
// the full parse and can never return a partial total.
|
||||
const progressive = isTTY && dayRange == null && customRange == null && await isColdCacheOnDisk()
|
||||
const assemble = () =>
|
||||
assembleDashboardData(period, provider, projectFilter, excludeFilter, customRange, initialDay ?? null, scrollableDailyHistory)
|
||||
const paint = progressive
|
||||
? await withColdFirstPaintFloor(getPeriodRange(period).start, assemble)
|
||||
: { result: await assemble(), deferredFiles: 0 }
|
||||
if (process.env['CODEBURN_VERBOSE'] === '1') {
|
||||
process.stderr.write(`codeburn: progressive cold start ${progressive ? 'on' : 'off'}, ${paint.deferredFiles} files deferred to the background fill\n`)
|
||||
}
|
||||
const { scannedProjects, filteredProjects, planUsages, initialDurable } = paint.result
|
||||
const label = initialDay ? formatDayRangeLabel(initialDay) : customRangeLabel
|
||||
patchStdoutForWindows()
|
||||
if (isTTY) {
|
||||
const app = renderDebouncedInteractive(process.stdout, ({ columns }) => (
|
||||
<InteractiveDashboard initialProjects={filteredProjects} initialDailyHistoryProjects={scrollableDailyHistory ? scannedProjects : undefined} initialPeriod={period} initialProvider={provider} initialPlanUsages={planUsages} initialDurable={initialDurable} refreshSeconds={refreshSeconds} projectFilter={projectFilter} excludeFilter={excludeFilter} customRange={customRange} customRangeLabel={customRangeLabel} initialDay={initialDay} windowColumns={columns} />
|
||||
<InteractiveDashboard initialProjects={filteredProjects} initialDailyHistoryProjects={scrollableDailyHistory ? scannedProjects : undefined} initialPeriod={period} initialProvider={provider} initialPlanUsages={planUsages} initialDurable={initialDurable} refreshSeconds={refreshSeconds} projectFilter={projectFilter} excludeFilter={excludeFilter} customRange={customRange} customRangeLabel={customRangeLabel} initialDay={initialDay} windowColumns={columns} initialIndexPendingFiles={paint.deferredFiles} />
|
||||
))
|
||||
try {
|
||||
await app.waitUntilExit()
|
||||
|
|
|
|||
|
|
@ -2000,6 +2000,7 @@ async function scanProjectDirs(
|
|||
if (readOnly && action.action !== 'unchanged') readOnlyServedStale = true
|
||||
unchangedFiles.push({ filePath, dirName, source, cached: section.files[filePath]! })
|
||||
} else if (!readOnly) {
|
||||
if (deferToBackgroundFill(filePath, fp, cached)) continue
|
||||
if (action.action === 'appended') {
|
||||
changedFiles.push({
|
||||
filePath,
|
||||
|
|
@ -2097,6 +2098,7 @@ async function scanProjectDirs(
|
|||
|
||||
try {
|
||||
for (const { filePath, info, append } of changedFiles) {
|
||||
filesParsedFromSource++
|
||||
// Marked here, not after the re-parse: an unreadable file `continue`s out
|
||||
// below, and the deletion would otherwise live only in memory.
|
||||
delete section.files[filePath]
|
||||
|
|
@ -3290,6 +3292,7 @@ export async function parseProviderSources(
|
|||
if (readOnly && action.action !== 'unchanged') readOnlyServedStale = true
|
||||
unchangedSources.push({ source, cached })
|
||||
} else if (!readOnly) {
|
||||
if (deferToBackgroundFill(source.path, fp, cached)) continue
|
||||
changedSources.push({ source, fp })
|
||||
} else {
|
||||
// Read-only with no cache entry at all — see scanProjectDirs.
|
||||
|
|
@ -3387,6 +3390,7 @@ export async function parseProviderSources(
|
|||
if (dateRange) {
|
||||
if (fp.mtimeMs < dateRange.start.getTime()) continue
|
||||
}
|
||||
filesParsedFromSource++
|
||||
|
||||
// Clear stale entry before parse — but only once per path so that
|
||||
// multiple sources mapping to the same file path can merge their turns.
|
||||
|
|
@ -4207,7 +4211,12 @@ function cacheKey(dateRange: DateRange | undefined, providerFilter: string | und
|
|||
// Flat-rate marks do not change parse-time cost (still $0 without a LiteLLM
|
||||
// row); findUnpricedModels / coverage apply them at render time, so they
|
||||
// stay out of this serve-memo key on purpose.
|
||||
return `${s}:${providerFilter ?? 'all'}:${claudeRoots}:${getProxyPathsConfigHash()}:${getModelAliasesConfigHash()}:${getPriceOverridesConfigHash()}:${getLocalModelSavingsConfigHash()}`
|
||||
// A first-paint parse sees a deliberately smaller file set, so its result may
|
||||
// not be served to (or burst-reused by) an unfloored request — including the
|
||||
// background fill that follows it moments later. Absent outside the scope, so
|
||||
// every non-cold-start key is byte-identical to what it was.
|
||||
const floor = firstPaintFloorMs === null ? '' : `:paint${firstPaintFloorMs}`
|
||||
return `${s}:${providerFilter ?? 'all'}:${claudeRoots}:${getProxyPathsConfigHash()}:${getModelAliasesConfigHash()}:${getPriceOverridesConfigHash()}:${getLocalModelSavingsConfigHash()}${floor}`
|
||||
}
|
||||
|
||||
export function clearSessionCache(): void {
|
||||
|
|
@ -4761,6 +4770,75 @@ function singlePassParse(dateRange: DateRange | undefined, providerFilter: strin
|
|||
return parsed.then(projects => filterProjectsByDateRange(projects, dateRange))
|
||||
}
|
||||
|
||||
// Progressive cold start (#1107). A session log is append-only, so its last
|
||||
// event timestamp is <= its mtime: a file whose mtime predates the start of the
|
||||
// range being displayed provably holds nothing that range can show. On a COLD
|
||||
// start that is what makes a fast first paint honest — the deferred files are
|
||||
// not dropped, only sequenced behind the paint, and the background fill parses
|
||||
// them into the same per-file cache a full cold parse would have written.
|
||||
//
|
||||
// The margin is pure paranoia about mtimes that lie: a restored backup, a
|
||||
// machine whose clock jumped, an rsync that preserved a wrong stamp. It only
|
||||
// widens the set that gets parsed BEFORE the paint, so it can never lose data.
|
||||
export const FIRST_PAINT_MTIME_MARGIN_MS = 48 * 60 * 60 * 1000
|
||||
|
||||
let firstPaintFloorMs: number | null = null
|
||||
// Files this scope deferred, as a SET of paths: one first paint runs several
|
||||
// parses (the scan, the plan window, the durable backfill) and each defers the
|
||||
// same old files, so a running count would report a multiple of the real work.
|
||||
let firstPaintDeferredPaths: Set<string> | null = null
|
||||
// Same count, but reset per runParse: a run that deferred NOTHING did exactly
|
||||
// what an unfloored run would have done, so it is allowed to mark the cache
|
||||
// complete and report full hydration.
|
||||
let firstPaintDeferredThisRun = 0
|
||||
|
||||
/** Files parsed from source (not served from cache) since the process started.
|
||||
* The background-fill indicator reads it to show live N/M progress. */
|
||||
let filesParsedFromSource = 0
|
||||
export function filesParsedFromSourceCount(): number {
|
||||
return filesParsedFromSource
|
||||
}
|
||||
|
||||
/** Restrict every parse inside `fn` to files that can hold in-range data for a
|
||||
* view starting at `rangeStart`, and report how many files were deferred.
|
||||
* Cold-start first paint only: the caller MUST follow up with an unscoped
|
||||
* parse (the background fill) before the run can be treated as hydrated. */
|
||||
export async function withColdFirstPaintFloor<T>(
|
||||
rangeStart: Date,
|
||||
fn: () => Promise<T>,
|
||||
): Promise<{ result: T; deferredFiles: number }> {
|
||||
const outer = firstPaintFloorMs
|
||||
const outerPaths = firstPaintDeferredPaths
|
||||
firstPaintFloorMs = rangeStart.getTime() - FIRST_PAINT_MTIME_MARGIN_MS
|
||||
firstPaintDeferredPaths = new Set()
|
||||
try {
|
||||
const result = await fn()
|
||||
return { result, deferredFiles: firstPaintDeferredPaths.size }
|
||||
} finally {
|
||||
firstPaintFloorMs = outer
|
||||
firstPaintDeferredPaths = outerPaths
|
||||
}
|
||||
}
|
||||
|
||||
/** True when this file's whole-file parse can be deferred to the background
|
||||
* fill. A file with a cache entry is never deferred: it has something to serve
|
||||
* and re-reading it is incremental, so deferring would only make the served
|
||||
* snapshot staler for no saving. */
|
||||
export function shouldDeferToBackgroundFill(
|
||||
fp: { mtimeMs: number },
|
||||
cached: unknown,
|
||||
floorMs: number | null,
|
||||
): boolean {
|
||||
return floorMs !== null && cached === undefined && fp.mtimeMs < floorMs
|
||||
}
|
||||
|
||||
function deferToBackgroundFill(path: string, fp: { mtimeMs: number }, cached: unknown): boolean {
|
||||
if (!shouldDeferToBackgroundFill(fp, cached, firstPaintFloorMs)) return false
|
||||
firstPaintDeferredPaths?.add(path)
|
||||
firstPaintDeferredThisRun++
|
||||
return true
|
||||
}
|
||||
|
||||
export function parseAllSessions(dateRange?: DateRange, providerFilter?: string): Promise<ProjectSummary[]> {
|
||||
const scoped = singlePassParse(dateRange, providerFilter)
|
||||
if (scoped) return scoped
|
||||
|
|
@ -4901,6 +4979,7 @@ async function runParseInner(
|
|||
const { isCold = false, readOnly = false, refreshLock } = options
|
||||
readOnlyServedStale = false
|
||||
deferredRetryableSource = false
|
||||
firstPaintDeferredThisRun = 0
|
||||
const seenMsgIds = new Set<string>()
|
||||
const seenKeys = new Set<string>()
|
||||
const allSources = await discoverAllSessions(providerFilter)
|
||||
|
|
@ -5012,9 +5091,15 @@ async function runParseInner(
|
|||
// on every launch, and the completeness marker the daily backfill + splash rely
|
||||
// on is durable. A run killed before here never reaches this, so its throttled
|
||||
// partial saves keep `complete: false` and the next launch resumes cold.
|
||||
// A first-paint run that deferred files never saw the whole corpus, so it may
|
||||
// not stamp the cache complete — the next launch (or this run's own
|
||||
// background fill) has to come back cold and finish the job. A floored run
|
||||
// that deferred NOTHING parsed exactly what an unfloored run would have, so
|
||||
// it keeps the normal stamp.
|
||||
const deferredForFirstPaint = firstPaintDeferredThisRun > 0
|
||||
const wasComplete = isCacheComplete(diskCache)
|
||||
if (!readOnly && !wasComplete) diskCache.complete = true
|
||||
if (!readOnly && (isCacheDirty(diskCache) || !wasComplete)) {
|
||||
if (!readOnly && !wasComplete && !deferredForFirstPaint) diskCache.complete = true
|
||||
if (!readOnly && (isCacheDirty(diskCache) || (!wasComplete && !deferredForFirstPaint))) {
|
||||
try {
|
||||
const published = await saveCache(diskCache, refreshLock?.verifyStillOwner)
|
||||
if (!published) throw new RefreshFenceLostError()
|
||||
|
|
@ -5027,7 +5112,7 @@ async function runParseInner(
|
|||
// files, or a write run that deferred a changed source on a retryable
|
||||
// failure, reached the end of the scan without hydrating everything, and
|
||||
// the daily backfill must not finalize history off it.
|
||||
sessionHydrationComplete = (!readOnly || !readOnlyServedStale) && !deferredRetryableSource
|
||||
sessionHydrationComplete = (!readOnly || !readOnlyServedStale) && !deferredRetryableSource && !deferredForFirstPaint
|
||||
|
||||
// Merge across providers by normalised project path so the same repository
|
||||
// is not double-counted when it was worked on with more than one tool
|
||||
|
|
|
|||
|
|
@ -528,6 +528,17 @@ export function isCacheComplete(cache: SessionCache): boolean {
|
|||
return cache.complete === true
|
||||
}
|
||||
|
||||
/** Pre-parse probe of the same question `isCacheComplete` answers after a load:
|
||||
* is the next parse going to be a cold hydration? Reads only the (tiny)
|
||||
* envelope, so a caller can branch on coldness before paying for the shards.
|
||||
* A cache still in a legacy layout has no envelope and reads as cold — the
|
||||
* adoption in `loadCache` may still make it warm, which costs the caller
|
||||
* nothing: a warm cache has an entry for every discovered file, so a
|
||||
* cold-start optimisation keyed on missing entries simply finds no work. */
|
||||
export async function isColdCacheOnDisk(): Promise<boolean> {
|
||||
return (await readEnvelope(sessionCacheDir()))?.complete !== true
|
||||
}
|
||||
|
||||
function isNum(v: unknown): v is number {
|
||||
return typeof v === 'number' && Number.isFinite(v)
|
||||
}
|
||||
|
|
|
|||
175
tests/progressive-cold-start.test.ts
Normal file
175
tests/progressive-cold-start.test.ts
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { mkdtemp, mkdir, writeFile, rm, utimes } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
import { tmpdir } from 'os'
|
||||
|
||||
import {
|
||||
parseAllSessions,
|
||||
clearSessionCache,
|
||||
withColdFirstPaintFloor,
|
||||
shouldDeferToBackgroundFill,
|
||||
filesParsedFromSourceCount,
|
||||
isSessionHydrationComplete,
|
||||
FIRST_PAINT_MTIME_MARGIN_MS,
|
||||
} from '../src/parser.js'
|
||||
import { clearLoadCacheMemo, isColdCacheOnDisk } from '../src/session-cache.js'
|
||||
import { readCacheOnDisk } from './fixtures/session-cache-io.js'
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000
|
||||
|
||||
let tmpDir: string
|
||||
|
||||
beforeEach(async () => {
|
||||
clearSessionCache()
|
||||
clearLoadCacheMemo()
|
||||
tmpDir = await mkdtemp(join(tmpdir(), 'progressive-cold-'))
|
||||
process.env['CLAUDE_CONFIG_DIR'] = tmpDir
|
||||
process.env['CODEBURN_CACHE_DIR'] = join(tmpDir, 'cache')
|
||||
process.env['CODEBURN_DESKTOP_SESSIONS_DIR'] = join(tmpDir, 'desktop-sessions')
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
clearSessionCache()
|
||||
clearLoadCacheMemo()
|
||||
await rm(tmpDir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
/** One Claude session file whose only turn is `ageDays` old, with an mtime to
|
||||
* match — the shape the mtime argument is about. */
|
||||
async function writeSession(name: string, ageDays: number): Promise<string> {
|
||||
const dir = join(tmpDir, 'projects', 'proj')
|
||||
await mkdir(dir, { recursive: true })
|
||||
const at = new Date(Date.now() - ageDays * DAY_MS)
|
||||
const path = join(dir, `${name}.jsonl`)
|
||||
await writeFile(path, JSON.stringify({
|
||||
type: 'assistant',
|
||||
sessionId: name,
|
||||
timestamp: at.toISOString(),
|
||||
cwd: '/tmp/proj',
|
||||
message: {
|
||||
id: `msg-${name}`, type: 'message', role: 'assistant', model: 'claude-sonnet-4-5',
|
||||
content: [], usage: { input_tokens: 100, output_tokens: 50 },
|
||||
},
|
||||
}) + '\n')
|
||||
await utimes(path, at, at)
|
||||
return path
|
||||
}
|
||||
|
||||
function lastWeek(): Date {
|
||||
return new Date(Date.now() - 7 * DAY_MS)
|
||||
}
|
||||
|
||||
async function cachedClaudePaths(): Promise<string[]> {
|
||||
const raw = await readCacheOnDisk()
|
||||
return Object.keys(raw.providers['claude']?.files ?? {}).sort()
|
||||
}
|
||||
|
||||
describe('first-paint deferral predicate', () => {
|
||||
const floor = 1_000_000
|
||||
|
||||
it('defers only files strictly below the floor', () => {
|
||||
expect(shouldDeferToBackgroundFill({ mtimeMs: floor - 1 }, undefined, floor)).toBe(true)
|
||||
expect(shouldDeferToBackgroundFill({ mtimeMs: floor }, undefined, floor)).toBe(false)
|
||||
expect(shouldDeferToBackgroundFill({ mtimeMs: floor + 1 }, undefined, floor)).toBe(false)
|
||||
})
|
||||
|
||||
it('keeps a file the clock-skew margin rescues', () => {
|
||||
// The floor the dashboard passes is rangeStart - MARGIN, so a file stamped
|
||||
// up to MARGIN before the range start is still parsed for the first paint.
|
||||
const rangeStart = 10 * DAY_MS
|
||||
const skewed = rangeStart - FIRST_PAINT_MTIME_MARGIN_MS + 1
|
||||
expect(shouldDeferToBackgroundFill({ mtimeMs: skewed }, undefined, rangeStart - FIRST_PAINT_MTIME_MARGIN_MS)).toBe(false)
|
||||
expect(shouldDeferToBackgroundFill({ mtimeMs: skewed - 2 }, undefined, rangeStart - FIRST_PAINT_MTIME_MARGIN_MS)).toBe(true)
|
||||
})
|
||||
|
||||
it('never defers a file that has a cache entry, or any file outside the scope', () => {
|
||||
const cached = { fingerprint: { dev: 0, ino: 0, mtimeMs: 0, sizeBytes: 0 }, mcpInventory: [], turns: [] }
|
||||
expect(shouldDeferToBackgroundFill({ mtimeMs: floor - 1 }, cached, floor)).toBe(false)
|
||||
// floor === null is every warm run and every one-shot command.
|
||||
expect(shouldDeferToBackgroundFill({ mtimeMs: floor - 1 }, undefined, null)).toBe(false)
|
||||
})
|
||||
|
||||
it('never defers a network source', () => {
|
||||
// Network providers have no on-disk file; they enter the parse with a
|
||||
// synthetic fingerprint stamped `Date.now()` (and skip this check entirely),
|
||||
// so no floor a dated view can produce ever holds them back.
|
||||
expect(shouldDeferToBackgroundFill({ mtimeMs: Date.now() }, undefined, lastWeek().getTime())).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('progressive cold start', () => {
|
||||
it('paints from the recent files and leaves the rest to the fill', async () => {
|
||||
await writeSession('recent', 1)
|
||||
await writeSession('old', 200)
|
||||
|
||||
const { result, deferredFiles } = await withColdFirstPaintFloor(lastWeek(), () => parseAllSessions())
|
||||
expect(deferredFiles).toBe(1)
|
||||
expect(result.flatMap(p => p.sessions).map(s => s.sessionId)).toEqual(['recent'])
|
||||
// A pass that deferred files is a partial hydration: it must not stamp the
|
||||
// session cache complete (the next launch has to come back cold) and must
|
||||
// not let the daily backfill finalize history off it.
|
||||
expect(isSessionHydrationComplete()).toBe(false)
|
||||
expect(await isColdCacheOnDisk()).toBe(true)
|
||||
// Durable partial progress: the file it DID parse is already cached.
|
||||
expect(await cachedClaudePaths()).toEqual([join(tmpDir, 'projects', 'proj', 'recent.jsonl')])
|
||||
})
|
||||
|
||||
it('converges on the full-cold-parse state, without re-parsing what pass 1 did', async () => {
|
||||
await writeSession('recent', 1)
|
||||
await writeSession('old', 200)
|
||||
|
||||
await withColdFirstPaintFloor(lastWeek(), () => parseAllSessions())
|
||||
clearSessionCache()
|
||||
|
||||
// The background fill is an ordinary unscoped parse in the same process.
|
||||
const parsedBefore = filesParsedFromSourceCount()
|
||||
const filled = await parseAllSessions()
|
||||
// Exactly one file parsed: the deferred one. The pass-1 file is served from
|
||||
// its cache entry, so it is neither re-read nor counted twice.
|
||||
expect(filesParsedFromSourceCount() - parsedBefore).toBe(1)
|
||||
expect(filled.flatMap(p => p.sessions).map(s => s.sessionId).sort()).toEqual(['old', 'recent'])
|
||||
expect(isSessionHydrationComplete()).toBe(true)
|
||||
expect(await isColdCacheOnDisk()).toBe(false)
|
||||
|
||||
const progressive = await readCacheOnDisk()
|
||||
|
||||
// Same corpus, a plain full cold parse, in a cache dir of its own.
|
||||
process.env['CODEBURN_CACHE_DIR'] = join(tmpDir, 'cache-baseline')
|
||||
clearSessionCache()
|
||||
clearLoadCacheMemo()
|
||||
await parseAllSessions()
|
||||
const baseline = await readCacheOnDisk()
|
||||
|
||||
expect(progressive.providers['claude']?.files).toEqual(baseline.providers['claude']?.files)
|
||||
expect(progressive.complete).toBe(baseline.complete)
|
||||
})
|
||||
|
||||
it('resumes after a fill that never ran', async () => {
|
||||
await writeSession('recent', 1)
|
||||
await writeSession('old', 200)
|
||||
|
||||
// Pass 1 only, then the process dies: what survives on disk is the recent
|
||||
// file's entry and an INCOMPLETE marker.
|
||||
await withColdFirstPaintFloor(lastWeek(), () => parseAllSessions())
|
||||
expect(await isColdCacheOnDisk()).toBe(true)
|
||||
|
||||
// Next launch. The deferred file has no cache entry, so it is still
|
||||
// discovered as changed and parsed; nothing is stranded behind a "seen" mark.
|
||||
clearSessionCache()
|
||||
clearLoadCacheMemo()
|
||||
const resumed = await parseAllSessions()
|
||||
expect(resumed.flatMap(p => p.sessions).map(s => s.sessionId).sort()).toEqual(['old', 'recent'])
|
||||
expect(await isColdCacheOnDisk()).toBe(false)
|
||||
})
|
||||
|
||||
it('is a no-op when every file is recent enough to paint', async () => {
|
||||
await writeSession('recent', 1)
|
||||
|
||||
const { deferredFiles } = await withColdFirstPaintFloor(lastWeek(), () => parseAllSessions())
|
||||
// Nothing deferred means this pass saw the whole corpus, so it keeps the
|
||||
// ordinary completeness stamp instead of owing a fill.
|
||||
expect(deferredFiles).toBe(0)
|
||||
expect(isSessionHydrationComplete()).toBe(true)
|
||||
expect(await isColdCacheOnDisk()).toBe(false)
|
||||
})
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue