feat(cli): scan progress protocol, incremental cache saves, --no-timeline

- CODEBURN_PROGRESS=1 emits newline-JSON progress on stderr (provider
  list, per-provider start/done, per-file ticks) for GUI first-run
  loaders; plain CLI output untouched
- parseAllSessions saves the session cache atomically every 5s during
  hydration and after each provider group, so an interrupted cold scan
  leaves a warm partial cache instead of nothing
- status --no-timeline skips the granular timeline computation
  (~300-450ms per call) for clients that never render it; default
  unchanged for the menubar
This commit is contained in:
iamtoruk 2026-07-16 10:51:18 -07:00
parent 3163ccb5c4
commit ded4e4e844
5 changed files with 179 additions and 10 deletions

View file

@ -776,6 +776,7 @@ program
.option('--to <date>', 'End date (YYYY-MM-DD) for custom range')
.option('--days <dates>', 'Comma-separated dates (YYYY-MM-DD) for multi-day selection')
.option('--no-optimize', 'Skip optimize findings (menubar-json only, faster)')
.option('--no-timeline', 'Skip the granular timeline (menubar-json only, faster)')
.addOption(new Option('--claude-config-source <id>').hideHelp())
.action(async (opts) => {
assertFormat(opts.format, ['terminal', 'menubar-json', 'json'], 'status')
@ -826,6 +827,7 @@ program
exclude: opts.exclude,
daysSelection,
optimize: opts.optimize !== false,
timeline: opts.timeline !== false,
claudeConfigSourceId: opts.claudeConfigSource,
})
if (opts.scope === 'combined') {

View file

@ -1657,6 +1657,10 @@ async function scanProjectDirs(
seenMsgIds: Set<string>,
diskCache: SessionCache,
dateRange?: DateRange,
// Cold-run robustness: called after every parsed Claude file so a throttled
// caller (parseAllSessions) can persist partial progress. A run killed
// mid-scan then resumes from a warm cache instead of re-parsing from zero.
onFileParsed?: () => Promise<void>,
): Promise<ProjectSummary[]> {
const section = getOrCreateProviderSection(diskCache, 'claude')
const allDiscoveredFiles = new Set<string>()
@ -1696,7 +1700,9 @@ async function scanProjectDirs(
}
const parseProgress = createScanProgress('parsing changed claude sessions', changedFiles.length)
const progressTotal = changedFiles.length
let filesDone = 0
emitScanProgress({ kind: 'tick', provider: 'claude', done: 0, total: progressTotal })
for (const { filePath, info } of changedFiles) {
delete section.files[filePath]
@ -1730,6 +1736,12 @@ async function scanProjectDirs(
}
filesDone++
await parseProgress.tick(filesDone)
// Machine-readable tick for the app splash (throttled to ~every 50 files so
// a large cold run doesn't flood stderr), plus a partial-progress save.
if (filesDone % 50 === 0 || filesDone === progressTotal) {
emitScanProgress({ kind: 'tick', provider: 'claude', done: filesDone, total: progressTotal })
}
if (onFileParsed) await onFileParsed()
}
parseProgress.finish()
@ -2152,6 +2164,28 @@ export function setInteractiveScanUI(active = true): void {
interactiveScanUI = active
}
// Machine-readable scan progress for the desktop app's first-run splash. Plain
// CLI/terminal usage is untouched: emission is gated on CODEBURN_PROGRESS=1,
// which only the app's cold-start warmup spawn sets. Each event is one
// newline-delimited JSON object behind a sentinel prefix so the reader can pick
// it out of stderr that may also carry provider warnings. This is orthogonal to
// createScanProgress's `\r` TTY line (that one never fires under a piped spawn).
export const PROGRESS_LINE_PREFIX = 'CODEBURN_PROGRESS '
export type ScanProgressEvent =
| { kind: 'providers'; providers: string[] }
| { kind: 'provider'; provider: string; state: 'start' | 'done'; files?: number }
| { kind: 'tick'; provider: string; done: number; total: number }
export function emitScanProgress(event: ScanProgressEvent): void {
if (process.env['CODEBURN_PROGRESS'] !== '1') return
try { process.stderr.write(`${PROGRESS_LINE_PREFIX}${JSON.stringify(event)}\n`) } catch { /* stderr closed */ }
}
// Minimum spacing between partial-progress saves during a cold parse. Low enough
// that an interrupted long run loses little work, high enough that repeated
// full-cache writes never dominate a fast warm run.
const PROGRESS_SAVE_THROTTLE_MS = 5000
export function createScanProgress(label: string, total: number) {
const show = !interactiveScanUI && total > 20 && process.stderr.isTTY === true
let lastWrite = 0
@ -2611,15 +2645,6 @@ export async function parseAllSessions(dateRange?: DateRange, providerFilter?: s
const claudeSources = allSources.filter(s => s.provider === 'claude')
const nonClaudeSources = allSources.filter(s => s.provider !== 'claude')
const claudeDirs = claudeSources.map(s => ({
path: s.path,
name: s.project,
source: s.sourceId && s.sourceLabel && s.sourcePath && s.sourceKind
? { id: s.sourceId, label: s.sourceLabel, path: s.sourcePath, kind: s.sourceKind }
: undefined,
}))
const claudeProjects = await scanProjectDirs(claudeDirs, seenMsgIds, diskCache, dateRange)
const providerGroups = new Map<string, SessionSource[]>()
for (const source of nonClaudeSources) {
const existing = providerGroups.get(source.provider) ?? []
@ -2627,10 +2652,41 @@ export async function parseAllSessions(dateRange?: DateRange, providerFilter?: s
providerGroups.set(source.provider, existing)
}
// Cold-run robustness: persist partial progress during a long parse (throttled)
// so a run interrupted before the single end-of-parse save still leaves a warm
// cache behind. saveCache is atomic (temp + rename) and clears `_dirty`, so this
// never races the final save below.
let lastSaveAt = Date.now()
const saveProgress = async (): Promise<void> => {
if (!(diskCache as { _dirty?: boolean })._dirty) return
if (Date.now() - lastSaveAt < PROGRESS_SAVE_THROTTLE_MS) return
lastSaveAt = Date.now()
try { await saveCache(diskCache) } catch { /* best-effort partial save */ }
}
emitScanProgress({ kind: 'providers', providers: [
...(claudeSources.length > 0 ? ['claude'] : []),
...providerGroups.keys(),
] })
const claudeDirs = claudeSources.map(s => ({
path: s.path,
name: s.project,
source: s.sourceId && s.sourceLabel && s.sourcePath && s.sourceKind
? { id: s.sourceId, label: s.sourceLabel, path: s.sourcePath, kind: s.sourceKind }
: undefined,
}))
if (claudeSources.length > 0) emitScanProgress({ kind: 'provider', provider: 'claude', state: 'start' })
const claudeProjects = await scanProjectDirs(claudeDirs, seenMsgIds, diskCache, dateRange, saveProgress)
if (claudeSources.length > 0) emitScanProgress({ kind: 'provider', provider: 'claude', state: 'done', files: claudeSources.length })
const otherProjects: ProjectSummary[] = []
for (const [providerName, sources] of providerGroups) {
emitScanProgress({ kind: 'provider', provider: providerName, state: 'start' })
const projects = await parseProviderSources(providerName, sources, seenKeys, diskCache, dateRange)
emitScanProgress({ kind: 'provider', provider: providerName, state: 'done', files: sources.length })
otherProjects.push(...projects)
await saveProgress()
}
// Durable providers with cached data but NO discovered sources (all files pruned

View file

@ -96,6 +96,10 @@ export type AggregateOpts = {
daysSelection?: { range: DateRange; label: string; days: Set<string> } | null
optimize?: boolean
claudeConfigSourceId?: string | null
/// Build the granular per-bucket timeline (`history.timeline`). Defaults to
/// true. The desktop app never renders it, so it passes `--no-timeline` to
/// skip the buildGranularHistory pass on every menubar poll.
timeline?: boolean
}
type ConfigOption = { id: string; label: string; path: string }
@ -525,6 +529,6 @@ export async function buildMenubarPayloadForRange(periodInfo: PeriodInfo, opts:
const optimize = opts.optimize === false ? null : await scanAndDetect(scanProjects, scanRange)
const granularRange = opts.daysSelection?.range ?? scanRange
const granularHistory = buildGranularHistory(scanProjects, granularRange)
const granularHistory = opts.timeline === false ? undefined : buildGranularHistory(scanProjects, granularRange)
return buildMenubarPayload(currentData, providers, optimize, dailyHistory, retryTax, routingWaste, breakdowns, claudeConfigs, granularHistory)
}

View file

@ -0,0 +1,81 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
import { mkdtemp, mkdir, writeFile, readFile, rm } from 'fs/promises'
import { join } from 'path'
import { tmpdir } from 'os'
import { parseAllSessions, clearSessionCache } from '../src/parser.js'
import { CACHE_VERSION } from '../src/session-cache.js'
let tmpDir: string
let cacheDir: string
beforeEach(async () => {
clearSessionCache()
tmpDir = await mkdtemp(join(tmpdir(), 'coldstart-'))
cacheDir = join(tmpDir, 'cache')
process.env['CLAUDE_CONFIG_DIR'] = tmpDir
process.env['CODEBURN_CACHE_DIR'] = cacheDir
process.env['CODEBURN_DESKTOP_SESSIONS_DIR'] = join(tmpDir, 'desktop-sessions')
})
afterEach(async () => {
clearSessionCache()
delete process.env['CODEBURN_PROGRESS']
await rm(tmpDir, { recursive: true, force: true })
})
async function writeSession(): Promise<void> {
const dir = join(tmpDir, 'projects', 'proj')
await mkdir(dir, { recursive: true })
await writeFile(join(dir, 'sess.jsonl'), JSON.stringify({
type: 'assistant',
sessionId: 'sess',
timestamp: '2026-05-15T10:00:00Z',
cwd: '/tmp/proj',
message: {
id: 'msg-1', type: 'message', role: 'assistant', model: 'claude-sonnet-4-5',
content: [], usage: { input_tokens: 100, output_tokens: 50 },
},
}) + '\n')
}
describe('cold-start cache persistence', () => {
// The desktop cold-start bug is "the cache never persists". This pins the
// invariant the fix depends on: a run that reaches the end of parseAllSessions
// writes the current-version session cache to disk, so the next launch is warm.
it('a completed parseAllSessions persists the current-version cache to disk', async () => {
await writeSession()
const projects = await parseAllSessions()
expect(projects.length).toBeGreaterThan(0)
const raw = JSON.parse(await readFile(join(cacheDir, 'session-cache.json'), 'utf-8'))
expect(raw.version).toBe(CACHE_VERSION)
const claudeFiles = Object.keys(raw.providers?.claude?.files ?? {})
expect(claudeFiles.length).toBeGreaterThan(0)
// The persisted entry carries the parsed turn, so a warm reload serves it
// without re-reading the source file.
expect(raw.providers.claude.files[claudeFiles[0]!].turns.length).toBeGreaterThan(0)
})
it('streams per-provider scan progress only when CODEBURN_PROGRESS=1', async () => {
await writeSession()
const lines: string[] = []
const spy = vi.spyOn(process.stderr, 'write').mockImplementation(((chunk: unknown) => {
lines.push(String(chunk)); return true
}) as typeof process.stderr.write)
try {
// Off by default: plain CLI/terminal output is untouched.
await parseAllSessions()
expect(lines.filter(l => l.startsWith('CODEBURN_PROGRESS ')).length).toBe(0)
clearSessionCache()
process.env['CODEBURN_PROGRESS'] = '1'
await parseAllSessions()
const progress = lines.filter(l => l.startsWith('CODEBURN_PROGRESS '))
expect(progress.some(l => l.includes('"providers"'))).toBe(true)
expect(progress.some(l => l.includes('"provider":"claude"') && l.includes('"start"'))).toBe(true)
} finally {
spy.mockRestore()
}
})
})

View file

@ -108,6 +108,32 @@ describe('codeburn status --format menubar-json', () => {
}
})
it('omits history.timeline with --no-timeline, includes it by default', async () => {
const home = await mkdtemp(join(tmpdir(), 'codeburn-menubar-tl-'))
try {
const projectDir = join(home, '.claude', 'projects', 'myapp')
await mkdir(projectDir, { recursive: true })
const base = new Date(Date.now() - 3600_000)
const ts = (offset: number) => new Date(base.getTime() + offset).toISOString().replace(/\.\d+Z$/, 'Z')
await writeFile(
join(projectDir, 'session.jsonl'),
[userLine('s1', ts(0)), assistantLine('s1', ts(60_000), 'msg-1')].join('\n'),
)
const withTimeline = runCli(['status', '--format', 'menubar-json', '--period', 'today', '--no-optimize'], home)
expect(withTimeline.status, `stderr: ${withTimeline.stderr}`).toBe(0)
const withHistory = (JSON.parse(withTimeline.stdout) as { history: Record<string, unknown> }).history
expect(withHistory).toHaveProperty('timeline')
const noTimeline = runCli(['status', '--format', 'menubar-json', '--period', 'today', '--no-optimize', '--no-timeline'], home)
expect(noTimeline.status, `stderr: ${noTimeline.stderr}`).toBe(0)
const noHistory = (JSON.parse(noTimeline.stdout) as { history: Record<string, unknown> }).history
expect(noHistory).not.toHaveProperty('timeline')
} finally {
await rm(home, { recursive: true, force: true })
}
})
it('filters menubar payloads to a selected review day with --day', async () => {
const home = await mkdtemp(join(tmpdir(), 'codeburn-menubar-day-'))