mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-07-28 18:35:29 +00:00
feat(cli): scan progress protocol, incremental cache saves, --no-timeline
- CODEBURN_PROGRESS=1 emits newline-JSON progress on stderr (provider list, per-provider start/done, per-file ticks) for GUI first-run loaders; plain CLI output untouched - parseAllSessions saves the session cache atomically every 5s during hydration and after each provider group, so an interrupted cold scan leaves a warm partial cache instead of nothing - status --no-timeline skips the granular timeline computation (~300-450ms per call) for clients that never render it; default unchanged for the menubar
This commit is contained in:
parent
3163ccb5c4
commit
ded4e4e844
5 changed files with 179 additions and 10 deletions
|
|
@ -776,6 +776,7 @@ program
|
|||
.option('--to <date>', 'End date (YYYY-MM-DD) for custom range')
|
||||
.option('--days <dates>', 'Comma-separated dates (YYYY-MM-DD) for multi-day selection')
|
||||
.option('--no-optimize', 'Skip optimize findings (menubar-json only, faster)')
|
||||
.option('--no-timeline', 'Skip the granular timeline (menubar-json only, faster)')
|
||||
.addOption(new Option('--claude-config-source <id>').hideHelp())
|
||||
.action(async (opts) => {
|
||||
assertFormat(opts.format, ['terminal', 'menubar-json', 'json'], 'status')
|
||||
|
|
@ -826,6 +827,7 @@ program
|
|||
exclude: opts.exclude,
|
||||
daysSelection,
|
||||
optimize: opts.optimize !== false,
|
||||
timeline: opts.timeline !== false,
|
||||
claudeConfigSourceId: opts.claudeConfigSource,
|
||||
})
|
||||
if (opts.scope === 'combined') {
|
||||
|
|
|
|||
|
|
@ -1657,6 +1657,10 @@ async function scanProjectDirs(
|
|||
seenMsgIds: Set<string>,
|
||||
diskCache: SessionCache,
|
||||
dateRange?: DateRange,
|
||||
// Cold-run robustness: called after every parsed Claude file so a throttled
|
||||
// caller (parseAllSessions) can persist partial progress. A run killed
|
||||
// mid-scan then resumes from a warm cache instead of re-parsing from zero.
|
||||
onFileParsed?: () => Promise<void>,
|
||||
): Promise<ProjectSummary[]> {
|
||||
const section = getOrCreateProviderSection(diskCache, 'claude')
|
||||
const allDiscoveredFiles = new Set<string>()
|
||||
|
|
@ -1696,7 +1700,9 @@ async function scanProjectDirs(
|
|||
}
|
||||
|
||||
const parseProgress = createScanProgress('parsing changed claude sessions', changedFiles.length)
|
||||
const progressTotal = changedFiles.length
|
||||
let filesDone = 0
|
||||
emitScanProgress({ kind: 'tick', provider: 'claude', done: 0, total: progressTotal })
|
||||
for (const { filePath, info } of changedFiles) {
|
||||
delete section.files[filePath]
|
||||
|
||||
|
|
@ -1730,6 +1736,12 @@ async function scanProjectDirs(
|
|||
}
|
||||
filesDone++
|
||||
await parseProgress.tick(filesDone)
|
||||
// Machine-readable tick for the app splash (throttled to ~every 50 files so
|
||||
// a large cold run doesn't flood stderr), plus a partial-progress save.
|
||||
if (filesDone % 50 === 0 || filesDone === progressTotal) {
|
||||
emitScanProgress({ kind: 'tick', provider: 'claude', done: filesDone, total: progressTotal })
|
||||
}
|
||||
if (onFileParsed) await onFileParsed()
|
||||
}
|
||||
parseProgress.finish()
|
||||
|
||||
|
|
@ -2152,6 +2164,28 @@ export function setInteractiveScanUI(active = true): void {
|
|||
interactiveScanUI = active
|
||||
}
|
||||
|
||||
// Machine-readable scan progress for the desktop app's first-run splash. Plain
|
||||
// CLI/terminal usage is untouched: emission is gated on CODEBURN_PROGRESS=1,
|
||||
// which only the app's cold-start warmup spawn sets. Each event is one
|
||||
// newline-delimited JSON object behind a sentinel prefix so the reader can pick
|
||||
// it out of stderr that may also carry provider warnings. This is orthogonal to
|
||||
// createScanProgress's `\r` TTY line (that one never fires under a piped spawn).
|
||||
export const PROGRESS_LINE_PREFIX = 'CODEBURN_PROGRESS '
|
||||
export type ScanProgressEvent =
|
||||
| { kind: 'providers'; providers: string[] }
|
||||
| { kind: 'provider'; provider: string; state: 'start' | 'done'; files?: number }
|
||||
| { kind: 'tick'; provider: string; done: number; total: number }
|
||||
|
||||
export function emitScanProgress(event: ScanProgressEvent): void {
|
||||
if (process.env['CODEBURN_PROGRESS'] !== '1') return
|
||||
try { process.stderr.write(`${PROGRESS_LINE_PREFIX}${JSON.stringify(event)}\n`) } catch { /* stderr closed */ }
|
||||
}
|
||||
|
||||
// Minimum spacing between partial-progress saves during a cold parse. Low enough
|
||||
// that an interrupted long run loses little work, high enough that repeated
|
||||
// full-cache writes never dominate a fast warm run.
|
||||
const PROGRESS_SAVE_THROTTLE_MS = 5000
|
||||
|
||||
export function createScanProgress(label: string, total: number) {
|
||||
const show = !interactiveScanUI && total > 20 && process.stderr.isTTY === true
|
||||
let lastWrite = 0
|
||||
|
|
@ -2611,15 +2645,6 @@ export async function parseAllSessions(dateRange?: DateRange, providerFilter?: s
|
|||
const claudeSources = allSources.filter(s => s.provider === 'claude')
|
||||
const nonClaudeSources = allSources.filter(s => s.provider !== 'claude')
|
||||
|
||||
const claudeDirs = claudeSources.map(s => ({
|
||||
path: s.path,
|
||||
name: s.project,
|
||||
source: s.sourceId && s.sourceLabel && s.sourcePath && s.sourceKind
|
||||
? { id: s.sourceId, label: s.sourceLabel, path: s.sourcePath, kind: s.sourceKind }
|
||||
: undefined,
|
||||
}))
|
||||
const claudeProjects = await scanProjectDirs(claudeDirs, seenMsgIds, diskCache, dateRange)
|
||||
|
||||
const providerGroups = new Map<string, SessionSource[]>()
|
||||
for (const source of nonClaudeSources) {
|
||||
const existing = providerGroups.get(source.provider) ?? []
|
||||
|
|
@ -2627,10 +2652,41 @@ export async function parseAllSessions(dateRange?: DateRange, providerFilter?: s
|
|||
providerGroups.set(source.provider, existing)
|
||||
}
|
||||
|
||||
// Cold-run robustness: persist partial progress during a long parse (throttled)
|
||||
// so a run interrupted before the single end-of-parse save still leaves a warm
|
||||
// cache behind. saveCache is atomic (temp + rename) and clears `_dirty`, so this
|
||||
// never races the final save below.
|
||||
let lastSaveAt = Date.now()
|
||||
const saveProgress = async (): Promise<void> => {
|
||||
if (!(diskCache as { _dirty?: boolean })._dirty) return
|
||||
if (Date.now() - lastSaveAt < PROGRESS_SAVE_THROTTLE_MS) return
|
||||
lastSaveAt = Date.now()
|
||||
try { await saveCache(diskCache) } catch { /* best-effort partial save */ }
|
||||
}
|
||||
|
||||
emitScanProgress({ kind: 'providers', providers: [
|
||||
...(claudeSources.length > 0 ? ['claude'] : []),
|
||||
...providerGroups.keys(),
|
||||
] })
|
||||
|
||||
const claudeDirs = claudeSources.map(s => ({
|
||||
path: s.path,
|
||||
name: s.project,
|
||||
source: s.sourceId && s.sourceLabel && s.sourcePath && s.sourceKind
|
||||
? { id: s.sourceId, label: s.sourceLabel, path: s.sourcePath, kind: s.sourceKind }
|
||||
: undefined,
|
||||
}))
|
||||
if (claudeSources.length > 0) emitScanProgress({ kind: 'provider', provider: 'claude', state: 'start' })
|
||||
const claudeProjects = await scanProjectDirs(claudeDirs, seenMsgIds, diskCache, dateRange, saveProgress)
|
||||
if (claudeSources.length > 0) emitScanProgress({ kind: 'provider', provider: 'claude', state: 'done', files: claudeSources.length })
|
||||
|
||||
const otherProjects: ProjectSummary[] = []
|
||||
for (const [providerName, sources] of providerGroups) {
|
||||
emitScanProgress({ kind: 'provider', provider: providerName, state: 'start' })
|
||||
const projects = await parseProviderSources(providerName, sources, seenKeys, diskCache, dateRange)
|
||||
emitScanProgress({ kind: 'provider', provider: providerName, state: 'done', files: sources.length })
|
||||
otherProjects.push(...projects)
|
||||
await saveProgress()
|
||||
}
|
||||
|
||||
// Durable providers with cached data but NO discovered sources (all files pruned
|
||||
|
|
|
|||
|
|
@ -96,6 +96,10 @@ export type AggregateOpts = {
|
|||
daysSelection?: { range: DateRange; label: string; days: Set<string> } | null
|
||||
optimize?: boolean
|
||||
claudeConfigSourceId?: string | null
|
||||
/// Build the granular per-bucket timeline (`history.timeline`). Defaults to
|
||||
/// true. The desktop app never renders it, so it passes `--no-timeline` to
|
||||
/// skip the buildGranularHistory pass on every menubar poll.
|
||||
timeline?: boolean
|
||||
}
|
||||
|
||||
type ConfigOption = { id: string; label: string; path: string }
|
||||
|
|
@ -525,6 +529,6 @@ export async function buildMenubarPayloadForRange(periodInfo: PeriodInfo, opts:
|
|||
|
||||
const optimize = opts.optimize === false ? null : await scanAndDetect(scanProjects, scanRange)
|
||||
const granularRange = opts.daysSelection?.range ?? scanRange
|
||||
const granularHistory = buildGranularHistory(scanProjects, granularRange)
|
||||
const granularHistory = opts.timeline === false ? undefined : buildGranularHistory(scanProjects, granularRange)
|
||||
return buildMenubarPayload(currentData, providers, optimize, dailyHistory, retryTax, routingWaste, breakdowns, claudeConfigs, granularHistory)
|
||||
}
|
||||
|
|
|
|||
81
tests/cache-persistence-coldstart.test.ts
Normal file
81
tests/cache-persistence-coldstart.test.ts
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { mkdtemp, mkdir, writeFile, readFile, rm } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
import { tmpdir } from 'os'
|
||||
|
||||
import { parseAllSessions, clearSessionCache } from '../src/parser.js'
|
||||
import { CACHE_VERSION } from '../src/session-cache.js'
|
||||
|
||||
let tmpDir: string
|
||||
let cacheDir: string
|
||||
|
||||
beforeEach(async () => {
|
||||
clearSessionCache()
|
||||
tmpDir = await mkdtemp(join(tmpdir(), 'coldstart-'))
|
||||
cacheDir = join(tmpDir, 'cache')
|
||||
process.env['CLAUDE_CONFIG_DIR'] = tmpDir
|
||||
process.env['CODEBURN_CACHE_DIR'] = cacheDir
|
||||
process.env['CODEBURN_DESKTOP_SESSIONS_DIR'] = join(tmpDir, 'desktop-sessions')
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
clearSessionCache()
|
||||
delete process.env['CODEBURN_PROGRESS']
|
||||
await rm(tmpDir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
async function writeSession(): Promise<void> {
|
||||
const dir = join(tmpDir, 'projects', 'proj')
|
||||
await mkdir(dir, { recursive: true })
|
||||
await writeFile(join(dir, 'sess.jsonl'), JSON.stringify({
|
||||
type: 'assistant',
|
||||
sessionId: 'sess',
|
||||
timestamp: '2026-05-15T10:00:00Z',
|
||||
cwd: '/tmp/proj',
|
||||
message: {
|
||||
id: 'msg-1', type: 'message', role: 'assistant', model: 'claude-sonnet-4-5',
|
||||
content: [], usage: { input_tokens: 100, output_tokens: 50 },
|
||||
},
|
||||
}) + '\n')
|
||||
}
|
||||
|
||||
describe('cold-start cache persistence', () => {
|
||||
// The desktop cold-start bug is "the cache never persists". This pins the
|
||||
// invariant the fix depends on: a run that reaches the end of parseAllSessions
|
||||
// writes the current-version session cache to disk, so the next launch is warm.
|
||||
it('a completed parseAllSessions persists the current-version cache to disk', async () => {
|
||||
await writeSession()
|
||||
const projects = await parseAllSessions()
|
||||
expect(projects.length).toBeGreaterThan(0)
|
||||
|
||||
const raw = JSON.parse(await readFile(join(cacheDir, 'session-cache.json'), 'utf-8'))
|
||||
expect(raw.version).toBe(CACHE_VERSION)
|
||||
const claudeFiles = Object.keys(raw.providers?.claude?.files ?? {})
|
||||
expect(claudeFiles.length).toBeGreaterThan(0)
|
||||
// The persisted entry carries the parsed turn, so a warm reload serves it
|
||||
// without re-reading the source file.
|
||||
expect(raw.providers.claude.files[claudeFiles[0]!].turns.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('streams per-provider scan progress only when CODEBURN_PROGRESS=1', async () => {
|
||||
await writeSession()
|
||||
const lines: string[] = []
|
||||
const spy = vi.spyOn(process.stderr, 'write').mockImplementation(((chunk: unknown) => {
|
||||
lines.push(String(chunk)); return true
|
||||
}) as typeof process.stderr.write)
|
||||
try {
|
||||
// Off by default: plain CLI/terminal output is untouched.
|
||||
await parseAllSessions()
|
||||
expect(lines.filter(l => l.startsWith('CODEBURN_PROGRESS ')).length).toBe(0)
|
||||
|
||||
clearSessionCache()
|
||||
process.env['CODEBURN_PROGRESS'] = '1'
|
||||
await parseAllSessions()
|
||||
const progress = lines.filter(l => l.startsWith('CODEBURN_PROGRESS '))
|
||||
expect(progress.some(l => l.includes('"providers"'))).toBe(true)
|
||||
expect(progress.some(l => l.includes('"provider":"claude"') && l.includes('"start"'))).toBe(true)
|
||||
} finally {
|
||||
spy.mockRestore()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
|
@ -108,6 +108,32 @@ describe('codeburn status --format menubar-json', () => {
|
|||
}
|
||||
})
|
||||
|
||||
it('omits history.timeline with --no-timeline, includes it by default', async () => {
|
||||
const home = await mkdtemp(join(tmpdir(), 'codeburn-menubar-tl-'))
|
||||
try {
|
||||
const projectDir = join(home, '.claude', 'projects', 'myapp')
|
||||
await mkdir(projectDir, { recursive: true })
|
||||
const base = new Date(Date.now() - 3600_000)
|
||||
const ts = (offset: number) => new Date(base.getTime() + offset).toISOString().replace(/\.\d+Z$/, 'Z')
|
||||
await writeFile(
|
||||
join(projectDir, 'session.jsonl'),
|
||||
[userLine('s1', ts(0)), assistantLine('s1', ts(60_000), 'msg-1')].join('\n'),
|
||||
)
|
||||
|
||||
const withTimeline = runCli(['status', '--format', 'menubar-json', '--period', 'today', '--no-optimize'], home)
|
||||
expect(withTimeline.status, `stderr: ${withTimeline.stderr}`).toBe(0)
|
||||
const withHistory = (JSON.parse(withTimeline.stdout) as { history: Record<string, unknown> }).history
|
||||
expect(withHistory).toHaveProperty('timeline')
|
||||
|
||||
const noTimeline = runCli(['status', '--format', 'menubar-json', '--period', 'today', '--no-optimize', '--no-timeline'], home)
|
||||
expect(noTimeline.status, `stderr: ${noTimeline.stderr}`).toBe(0)
|
||||
const noHistory = (JSON.parse(noTimeline.stdout) as { history: Record<string, unknown> }).history
|
||||
expect(noHistory).not.toHaveProperty('timeline')
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('filters menubar payloads to a selected review day with --day', async () => {
|
||||
const home = await mkdtemp(join(tmpdir(), 'codeburn-menubar-day-'))
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue