mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-08-29 10:22:42 +00:00
feat: surface-wide progressive hydration via an explicit payload marker
The TTY got the fast cold start in #1109; every JSON surface still waited for the full parse because a payload has no way to say "partial". Add one. - `hydration: { complete, indexedFiles, totalFiles }` on the menubar payload, add-only and emitted ONLY by the resident serve child (which polls, and therefore converges). Absence keeps meaning "complete", so every one-shot output is byte-identical and no script can be handed partial data. - serve: a cold child answers `status --format menubar-json` from the files the requested period can show (same mtime floor as the TUI) and schedules the unfloored fill behind it. Partial answers are never memoized; requests waiting behind the fill are heartbeated by id so the client's no-output watchdog stays armed. - `stale` keeps its own meaning: a first paint is fresh but partial, so it reports through `hydration` and never sets `stale`. - desktop app and web dash render an honest "indexing history · N/M files" notice while `complete` is false.
This commit is contained in:
parent
c9dab7deb6
commit
be1b212a3f
11 changed files with 588 additions and 9 deletions
|
|
@ -544,6 +544,7 @@ function AppMain() {
|
|||
<div className="ct">
|
||||
<div className={overview.switching ? 'switch-line on' : 'switch-line'} aria-hidden="true" />
|
||||
<UpdateBanner />
|
||||
<IndexingBanner payload={overview.data ?? null} />
|
||||
<DailyBudgetBanner payload={overview.data ?? null} provider={provider} />
|
||||
<ErrorBoundary key={section}>
|
||||
{section === 'plans' ? (
|
||||
|
|
@ -625,6 +626,20 @@ function SectionPlaceholder({ title }: { title: string }) {
|
|||
)
|
||||
}
|
||||
|
||||
/** Honest partiality (#1110): on a cold cache the resident serve child answers
|
||||
* from the files the selected period can show and indexes the rest behind it.
|
||||
* Wording mirrors the TUI banner. Absent `hydration` means a full parse (a
|
||||
* one-shot spawn, or a CLI predating the field), so nothing is shown. */
|
||||
function IndexingBanner({ payload }: { payload: MenubarPayload | null }) {
|
||||
const hydration = payload?.hydration
|
||||
if (!hydration || hydration.complete) return null
|
||||
return (
|
||||
<div role="status" className="stale-banner">
|
||||
Indexing history · {Math.min(hydration.indexedFiles, hydration.totalFiles)}/{hydration.totalFiles} files · totals below cover what is indexed so far
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/** App-wide daily-budget alert: reads today's usage from the overview payload and
|
||||
* warns at >=80% / alerts at >=100% of the configured cap. Dismissible per day. */
|
||||
function DailyBudgetBanner({ payload, provider }: { payload: MenubarPayload | null; provider: string }) {
|
||||
|
|
|
|||
|
|
@ -127,11 +127,23 @@ export type ClaudeConfigSelector = {
|
|||
options: ClaudeConfigOption[]
|
||||
}
|
||||
|
||||
export type HydrationState = {
|
||||
complete: boolean
|
||||
indexedFiles: number
|
||||
totalFiles: number
|
||||
}
|
||||
|
||||
export type MenubarPayload = {
|
||||
generated: string
|
||||
// Optional: older CLIs omit it. Present and true only on a stale read-only
|
||||
// serve; absent otherwise. Absence must always be read as "assume fresh."
|
||||
stale?: boolean
|
||||
// Optional: only the resident serve child emits it, and only it may answer
|
||||
// partially (it polls, so it converges). `complete: false` means the totals
|
||||
// cover the files indexed so far and a later poll returns more. Absence — an
|
||||
// older CLI, or any one-shot spawn including the spawn fallback — must be
|
||||
// read as complete.
|
||||
hydration?: HydrationState
|
||||
current: {
|
||||
label: string
|
||||
cost: number
|
||||
|
|
|
|||
|
|
@ -75,6 +75,20 @@ function Stat({ label: lbl, value }: { label: string; value: string }) {
|
|||
|
||||
// One device's full dashboard. Remote devices arrive sanitized, so their
|
||||
// project and session detail is intentionally absent.
|
||||
/** Honest partiality: a payload from a producer that answers a cold start from
|
||||
* the files the period needs and indexes the rest behind it says so with
|
||||
* `hydration.complete: false`. Absent (every one-shot CLI output, and any
|
||||
* older CLI) means a full parse, so nothing is shown. */
|
||||
function IndexingNotice({ payload }: { payload?: Payload }) {
|
||||
const hydration = payload?.hydration
|
||||
if (!hydration || hydration.complete) return null
|
||||
return (
|
||||
<div role="status" className="mb-3 border-l-2 border-primary px-2.5 py-1 text-[12px] text-muted-foreground">
|
||||
Indexing history · {Math.min(hydration.indexedFiles, hydration.totalFiles)}/{hydration.totalFiles} files · totals below cover what is indexed so far
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function DeviceView({ payload, isRemote, unit }: { payload?: Payload; isRemote: boolean; unit: Unit }) {
|
||||
const c = payload?.current
|
||||
// Cache cards read the period-scoped `current` totals, matching Cost/Calls/
|
||||
|
|
@ -786,6 +800,8 @@ export function App() {
|
|||
<span className="text-xs text-tertiary-foreground">{page === 'usage' ? label : ''}</span>
|
||||
</div>
|
||||
|
||||
{page === 'usage' && <IndexingNotice payload={primary?.payload} />}
|
||||
|
||||
{page === 'context' ? (
|
||||
<ContextExplorer />
|
||||
) : showCombined ? (
|
||||
|
|
|
|||
|
|
@ -71,6 +71,10 @@ export type Current = {
|
|||
|
||||
export type Payload = {
|
||||
generated: string
|
||||
// Only a producer that its clients poll (the resident serve child) ever sends
|
||||
// this, and only it may answer partially. `complete: false` means the totals
|
||||
// cover the files indexed so far. Absence must be read as complete.
|
||||
hydration?: { complete: boolean; indexedFiles: number; totalFiles: number }
|
||||
current: Current
|
||||
history: { daily: DailyEntry[]; timeline?: GranularHistory }
|
||||
}
|
||||
|
|
@ -125,6 +129,7 @@ function normalizePayload(p?: Payload): Payload | undefined {
|
|||
} : undefined
|
||||
return {
|
||||
generated: p.generated,
|
||||
...(p.hydration ? { hydration: p.hydration } : {}),
|
||||
current: {
|
||||
label: c.label ?? '',
|
||||
cost: c.cost ?? 0,
|
||||
|
|
|
|||
|
|
@ -176,6 +176,16 @@ export type ClaudeConfigSelector = {
|
|||
options: ClaudeConfigOption[]
|
||||
}
|
||||
|
||||
/// How much of the corpus is behind the numbers in this payload (#1110).
|
||||
/// `complete: false` means the totals cover only the files indexed so far and
|
||||
/// a later poll will return more. The counts are progress indicators, not
|
||||
/// inventory: they are only meaningful while `complete` is false.
|
||||
export type HydrationState = {
|
||||
complete: boolean
|
||||
indexedFiles: number
|
||||
totalFiles: number
|
||||
}
|
||||
|
||||
export type MenubarPayload = {
|
||||
generated: string
|
||||
/// Optional. Present and `true` only when this payload was assembled from a
|
||||
|
|
@ -184,6 +194,15 @@ export type MenubarPayload = {
|
|||
/// means "assume fresh," including for payloads from a CLI version that
|
||||
/// predates this field.
|
||||
stale?: boolean
|
||||
/// Optional. Emitted ONLY by the resident `codeburn serve` child, the one
|
||||
/// producer whose consumers poll and therefore converge. Every one-shot CLI
|
||||
/// output omits it and is always a full parse, so absence must be read as
|
||||
/// "complete" — including for payloads from a CLI that predates the field.
|
||||
/// A consumer that renders totals MUST check this before presenting them as
|
||||
/// final; it is the only in-band marker that separates a partial answer from
|
||||
/// a converged one. Distinct from `stale`: a first paint is fresh but
|
||||
/// partial, a stale payload is complete but old.
|
||||
hydration?: HydrationState
|
||||
current: {
|
||||
label: string
|
||||
cost: number
|
||||
|
|
@ -514,6 +533,7 @@ export function buildMenubarPayload(
|
|||
claudeConfigs?: ClaudeConfigSelector,
|
||||
granularHistory?: GranularHistory,
|
||||
stale?: boolean,
|
||||
hydration?: HydrationState,
|
||||
): MenubarPayload {
|
||||
const payload: MenubarPayload = {
|
||||
generated: new Date().toISOString(),
|
||||
|
|
@ -567,5 +587,8 @@ export function buildMenubarPayload(
|
|||
if (stale) {
|
||||
payload.stale = true
|
||||
}
|
||||
if (hydration) {
|
||||
payload.hydration = hydration
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4700,6 +4700,12 @@ export function isSessionHydrationComplete(): boolean {
|
|||
return sessionHydrationComplete
|
||||
}
|
||||
|
||||
// Why the most recent parse was incomplete, when it was: true only when the
|
||||
// first-paint floor deferred files. Read by `sessionHydrationSnapshot` so the
|
||||
// serve payload can label a converging first paint without also claiming the
|
||||
// unrelated `stale` (read-only snapshot) condition.
|
||||
let sessionFirstPaintDeferred = false
|
||||
|
||||
// Set by the read-only serving paths when the snapshot they served did NOT
|
||||
// match what is on disk: in read-only mode a changed file is served at its
|
||||
// stale fingerprint and a file with no cache entry is skipped entirely. A
|
||||
|
|
@ -4799,6 +4805,28 @@ export function filesParsedFromSourceCount(): number {
|
|||
return filesParsedFromSource
|
||||
}
|
||||
|
||||
/** What the most recent parse left behind, for consumers that must present
|
||||
* partiality honestly (#1110). `deferredForFirstPaint` is what separates a
|
||||
* progressive cold start from the read-only stale case: both leave
|
||||
* `complete` false, but only the latter is `stale` — a first paint is fresh
|
||||
* data over a smaller file set, and it converges on its own.
|
||||
* `indexedFiles` counts files parsed from source since this process started
|
||||
* and `pendingFiles` the files the active first-paint scope deferred; both are
|
||||
* progress numbers and only meaningful while `complete` is false. */
|
||||
export function sessionHydrationSnapshot(): {
|
||||
complete: boolean
|
||||
deferredForFirstPaint: boolean
|
||||
indexedFiles: number
|
||||
pendingFiles: number
|
||||
} {
|
||||
return {
|
||||
complete: sessionHydrationComplete,
|
||||
deferredForFirstPaint: sessionFirstPaintDeferred,
|
||||
indexedFiles: filesParsedFromSource,
|
||||
pendingFiles: firstPaintDeferredPaths?.size ?? 0,
|
||||
}
|
||||
}
|
||||
|
||||
/** Restrict every parse inside `fn` to files that can hold in-range data for a
|
||||
* view starting at `rangeStart`, and report how many files were deferred.
|
||||
* Cold-start first paint only: the caller MUST follow up with an unscoped
|
||||
|
|
@ -5113,6 +5141,7 @@ async function runParseInner(
|
|||
// failure, reached the end of the scan without hydrating everything, and
|
||||
// the daily backfill must not finalize history off it.
|
||||
sessionHydrationComplete = (!readOnly || !readOnlyServedStale) && !deferredRetryableSource && !deferredForFirstPaint
|
||||
sessionFirstPaintDeferred = deferredForFirstPaint
|
||||
|
||||
// Merge across providers by normalised project path so the same repository
|
||||
// is not double-counted when it was worked on with more than one tool
|
||||
|
|
|
|||
155
src/serve.ts
155
src/serve.ts
|
|
@ -4,8 +4,10 @@ import { createHash } from 'crypto'
|
|||
import { createInterface } from 'readline'
|
||||
|
||||
import type { Command } from 'commander'
|
||||
import { getDateRange } from './cli-date.js'
|
||||
import { getConfigFilePath } from './config.js'
|
||||
import type { ParseReuseValidation } from './parser.js'
|
||||
import { SERVE_HYDRATION_ENV } from './usage-aggregator.js'
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// codeburn serve --stdio: a resident query server for the desktop app.
|
||||
|
|
@ -41,6 +43,13 @@ const SERVE_MAX_RSS_BYTES = 3 * 1024 * 1024 * 1024
|
|||
// prove the bound without a 45-second test.
|
||||
const DEFAULT_DRAIN_MS = 45_000
|
||||
|
||||
// How long the background fill waits after a cold first paint before taking the
|
||||
// queue. Long enough for the client's opening burst (it is serial, so its next
|
||||
// request is already on the wire) to land in front of the fill, short enough
|
||||
// that a client which then goes quiet still converges promptly.
|
||||
// CODEBURN_SERVE_FILL_DELAY_MS overrides it for tests.
|
||||
const DEFAULT_FILL_DELAY_MS = 3_000
|
||||
|
||||
type OutputMemoEntry = {
|
||||
createdAt: number
|
||||
validatedFrom: number
|
||||
|
|
@ -144,6 +153,60 @@ function allowed(args: string[]): boolean {
|
|||
return true
|
||||
}
|
||||
|
||||
/// Read a served option's value. `allowed()` has already proven the argv shape
|
||||
/// (no positionals, every value-bearing option followed by a value that does
|
||||
/// not start with '-'), so a token equal to the option name is always the
|
||||
/// option and never someone's value.
|
||||
function readServeOption(args: string[], name: string): string | undefined {
|
||||
for (let i = 1; i < args.length; i++) {
|
||||
const token = args[i]!
|
||||
if (token === name) return args[i + 1]
|
||||
if (token.startsWith(`${name}=`)) return token.slice(name.length + 1)
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
/// The range start a cold first paint of this request may be floored to, or
|
||||
/// null when the request must not be answered partially (#1110).
|
||||
///
|
||||
/// Only `status --format menubar-json` qualifies, because it is the ONLY served
|
||||
/// output that carries the `hydration` marker. Every other served command
|
||||
/// (`models`/`sessions`/`spend` --format json ...) is a one-shot shape whose
|
||||
/// consumer has no in-band way to tell a partial answer from a final one, so it
|
||||
/// keeps the full parse. Explicit `--day`/`--from`/`--to`/`--days` are excluded
|
||||
/// for the same reason the TUI excludes them: the floor is derived from a named
|
||||
/// period, and a custom range is a deliberate question that deserves a real
|
||||
/// answer.
|
||||
export function coldFirstPaintRangeStart(
|
||||
args: string[],
|
||||
rangeForPeriod: (period: string) => { range: { start: Date } },
|
||||
): Date | null {
|
||||
if (args[0] !== 'status') return null
|
||||
if (readServeOption(args, '--format') !== 'menubar-json') return null
|
||||
for (const flag of ['--day', '--from', '--to', '--days']) {
|
||||
if (readServeOption(args, flag) !== undefined) return null
|
||||
}
|
||||
// Mirrors the `status` --period default in main.ts.
|
||||
const period = readServeOption(args, '--period') ?? 'today'
|
||||
try {
|
||||
return rangeForPeriod(period).range.start
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/// The request id of a protocol line, without committing to handling it. The
|
||||
/// client's watchdog is a NO-OUTPUT timer, so a request waiting behind the
|
||||
/// background fill has to be heartbeated by id before its turn comes.
|
||||
function peekRequestId(line: string): string | number | null {
|
||||
try {
|
||||
const id = (JSON.parse(line) as { id?: unknown } | null)?.id
|
||||
return typeof id === 'string' || typeof id === 'number' ? id : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
class ExitSignal extends Error {
|
||||
constructor(public readonly code: number) { super(`exit ${code}`) }
|
||||
}
|
||||
|
|
@ -315,6 +378,12 @@ export async function runStdioServe(buildProgram: () => Command): Promise<void>
|
|||
// re-running the discovery sweep per panel. Serve-only: one-shot CLI runs
|
||||
// never set this, so their results stay byte-exact.
|
||||
if (!process.env['CODEBURN_PARSE_BURST_MS']) process.env['CODEBURN_PARSE_BURST_MS'] = '10000'
|
||||
// This process is the one producer allowed to answer partially, because its
|
||||
// clients poll and therefore converge. The marker makes every menubar payload
|
||||
// it emits carry `hydration` — including the warm ones, which report
|
||||
// `complete: true` — so a consumer never has to infer completeness from a
|
||||
// missing field within a single source.
|
||||
process.env[SERVE_HYDRATION_ENV] = '1'
|
||||
// Event-driven reuse: while no watched session root has changed, a previous
|
||||
// parse stays valid past the burst window (capped in parser.ts, so a missed
|
||||
// filesystem event self-heals within minutes). This is what turns a warm
|
||||
|
|
@ -367,10 +436,62 @@ export async function runStdioServe(buildProgram: () => Command): Promise<void>
|
|||
// Strict serialization: each request chains on the previous one.
|
||||
let queue: Promise<void> = Promise.resolve()
|
||||
|
||||
// Progressive cold start (#1110). Ids of requests received but not yet
|
||||
// answered, so work that holds the queue can heartbeat them.
|
||||
const awaiting = new Set<string | number>()
|
||||
const beat = (progress: string): void => {
|
||||
for (const id of awaiting) write({ id, progress })
|
||||
}
|
||||
// Cold detection runs at most once per paintable request until it answers
|
||||
// "warm"; from then on this child is warm for good and every request takes
|
||||
// the ordinary full path.
|
||||
let firstPaintSettled = false
|
||||
let fillTimer: ReturnType<typeof setTimeout> | undefined
|
||||
|
||||
// The other half of the first paint: the same request, unfloored. It is an
|
||||
// ordinary parse, so what it writes to the session and daily caches is
|
||||
// exactly what a full cold parse would have written — the paint only
|
||||
// sequenced those files behind the answer, it never dropped them. Killed
|
||||
// mid-fill, nothing is stamped complete and the next launch re-enters cold.
|
||||
const scheduleBackgroundFill = (args: string[]): void => {
|
||||
if (fillTimer) return
|
||||
const delayMs = Number(process.env['CODEBURN_SERVE_FILL_DELAY_MS']) || DEFAULT_FILL_DELAY_MS
|
||||
fillTimer = setTimeout(() => {
|
||||
queue = queue.then(async () => {
|
||||
const { isColdCacheOnDisk } = await import('./session-cache.js')
|
||||
// A request that landed in front of the fill may already have done the
|
||||
// full parse (only the menubar payload is ever floored).
|
||||
if (!await isColdCacheOnDisk()) return
|
||||
const { startProgressKeepalive, stopProgressKeepalive } = await import('./parser.js')
|
||||
startProgressKeepalive()
|
||||
const startedAt = Date.now()
|
||||
try {
|
||||
const { output, code } = await runCaptured(buildProgram, args, beat)
|
||||
const fingerprint = await getConfigFingerprint()
|
||||
// Memoized so the poll that follows the fill answers instantly with
|
||||
// the converged payload instead of re-deriving it.
|
||||
if (code === 0 && fingerprint !== null) {
|
||||
outputMemo.set(args.join('\u0000'), createOutputMemoEntry(startedAt, Date.now(), output, fingerprint))
|
||||
}
|
||||
} catch {
|
||||
// Best effort. A failed fill leaves the cache incomplete, which is
|
||||
// exactly the state the next cold start knows how to resume from.
|
||||
} finally {
|
||||
stopProgressKeepalive()
|
||||
}
|
||||
})
|
||||
}, delayMs)
|
||||
// The app owning this child is gone once stdin closes; an unstarted fill
|
||||
// must not keep the process alive past that.
|
||||
fillTimer.unref?.()
|
||||
}
|
||||
|
||||
const rl = createInterface({ input: process.stdin, crlfDelay: Infinity })
|
||||
rl.on('line', (line) => {
|
||||
const trimmed = line.trim()
|
||||
if (!trimmed) return
|
||||
const waitingId = peekRequestId(trimmed)
|
||||
if (waitingId !== null) awaiting.add(waitingId)
|
||||
queue = queue.then(async () => {
|
||||
let request: unknown
|
||||
try {
|
||||
|
|
@ -408,22 +529,47 @@ export async function runStdioServe(buildProgram: () => Command): Promise<void>
|
|||
write({ id: request.id, ok: true, output: memoHit.output })
|
||||
return
|
||||
}
|
||||
// Progressive cold start (#1110): on a cold cache the menubar payload is
|
||||
// answered from the files the requested period can actually show, and the
|
||||
// rest are indexed behind it. Same cold test as the TUI (session-cache
|
||||
// envelope missing or complete !== true), same floor, and the answer says
|
||||
// so in-band via `hydration.complete: false`.
|
||||
const paintFrom = firstPaintSettled ? null : coldFirstPaintRangeStart(request.args, getDateRange)
|
||||
let floor: Date | null = null
|
||||
if (paintFrom) {
|
||||
const { isColdCacheOnDisk } = await import('./session-cache.js')
|
||||
floor = await isColdCacheOnDisk() ? paintFrom : null
|
||||
firstPaintSettled = true
|
||||
}
|
||||
// Heartbeat the WHOLE request, not just its parse: the aggregation and
|
||||
// payload serialization after a parse measured ~8s of further silence, and
|
||||
// it lands back-to-back with the parse's own quiet stretches. Wrapping
|
||||
// here (rather than at each command's exits) covers every served command
|
||||
// at one seam, and runCaptured routes the beats out as progress frames.
|
||||
const { startProgressKeepalive, stopProgressKeepalive } = await import('./parser.js')
|
||||
const { startProgressKeepalive, stopProgressKeepalive, withColdFirstPaintFloor } = await import('./parser.js')
|
||||
startProgressKeepalive()
|
||||
try {
|
||||
const parseStartedAt = Date.now()
|
||||
const { output, code } = await runCaptured(
|
||||
const run = () => runCaptured(
|
||||
buildProgram,
|
||||
request.args,
|
||||
progress => write({ id: request.id, progress }),
|
||||
)
|
||||
let deferredFiles = 0
|
||||
let result: Awaited<ReturnType<typeof run>>
|
||||
if (floor) {
|
||||
const painted = await withColdFirstPaintFloor(floor, run)
|
||||
deferredFiles = painted.deferredFiles
|
||||
result = painted.result
|
||||
} else {
|
||||
result = await run()
|
||||
}
|
||||
const { output, code } = result
|
||||
if (code === 0) {
|
||||
if (configFingerprint !== null) {
|
||||
// A partial answer is never memoized. The roots stay quiet while the
|
||||
// fill converges, so a memo hit would pin the client to the first
|
||||
// paint for the whole memo cap.
|
||||
if (configFingerprint !== null && deferredFiles === 0) {
|
||||
outputMemo.set(memoKey, createOutputMemoEntry(parseStartedAt, Date.now(), output, configFingerprint))
|
||||
}
|
||||
if (outputMemo.size > 32) {
|
||||
|
|
@ -431,6 +577,7 @@ export async function runStdioServe(buildProgram: () => Command): Promise<void>
|
|||
if (oldest) outputMemo.delete(oldest[0])
|
||||
}
|
||||
write({ id: request.id, ok: true, output })
|
||||
if (deferredFiles > 0) scheduleBackgroundFill(request.args)
|
||||
}
|
||||
else write({ id: request.id, ok: false, error: `exit ${code}`, output })
|
||||
} catch (err) {
|
||||
|
|
@ -455,6 +602,8 @@ export async function runStdioServe(buildProgram: () => Command): Promise<void>
|
|||
clearAntigravityCacheStates()
|
||||
if (typeof globalThis.gc === 'function') globalThis.gc()
|
||||
}
|
||||
}).finally(() => {
|
||||
if (waitingId !== null) awaiting.delete(waitingId)
|
||||
})
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import { homedir } from 'node:os'
|
||||
import { CATEGORY_LABELS, type ProjectSummary, type TaskCategory, type DateRange } from './types.js'
|
||||
import { isBehavioralCall } from './behavioral-weight.js'
|
||||
import { type PeriodData, type ProviderCost, type BreakdownArrays, type MenubarPayload, type ClaudeConfigSelector, buildMenubarPayload } from './menubar-json.js'
|
||||
import { parseAllSessions, filterProjectsByName, filterProjectsByDays, filterProjectsByClaudeConfigSource, isSessionHydrationComplete } from './parser.js'
|
||||
import { type PeriodData, type ProviderCost, type BreakdownArrays, type MenubarPayload, type ClaudeConfigSelector, type HydrationState, buildMenubarPayload } from './menubar-json.js'
|
||||
import { parseAllSessions, filterProjectsByName, filterProjectsByDays, filterProjectsByClaudeConfigSource, isSessionHydrationComplete, sessionHydrationSnapshot } from './parser.js'
|
||||
import { findUnpricedModels, getFlatRateModelsConfigHash, getLocalModelSavingsConfigHash, getPriceOverridesConfigHash, getShortModelName, isExpectedFreeModel } from './models.js'
|
||||
import { getAllProviders, safeDiscoverSessions } from './providers/index.js'
|
||||
import { claude, getClaudeConfigDirs, getDesktopSessionsDirs } from './providers/claude.js'
|
||||
|
|
@ -114,6 +114,23 @@ async function hydrateCache(): Promise<DailyCache> {
|
|||
}
|
||||
}
|
||||
|
||||
/// The `hydration` block is emitted ONLY inside the resident serve child, which
|
||||
/// sets this marker on itself at startup. That is the whole one-shot safety
|
||||
/// rule in one place: a one-shot CLI process never sets it, so `--format
|
||||
/// menubar-json` from a spawn (including the desktop app's spawn fallback, the
|
||||
/// Swift menubar, and `codeburn web`) is byte-identical to before and can never
|
||||
/// carry a partial-data label — it has no second poll to converge with.
|
||||
export const SERVE_HYDRATION_ENV = 'CODEBURN_SERVE_HYDRATION'
|
||||
|
||||
function hydrationStateFor(hydration: ReturnType<typeof sessionHydrationSnapshot> | undefined): HydrationState | undefined {
|
||||
if (process.env[SERVE_HYDRATION_ENV] !== '1' || !hydration) return undefined
|
||||
return {
|
||||
complete: hydration.complete,
|
||||
indexedFiles: hydration.indexedFiles,
|
||||
totalFiles: hydration.indexedFiles + hydration.pendingFiles,
|
||||
}
|
||||
}
|
||||
|
||||
export type PeriodInfo = { range: DateRange; label: string }
|
||||
export type AggregateOpts = {
|
||||
provider?: string
|
||||
|
|
@ -573,14 +590,14 @@ export async function buildMenubarPayloadForRange(periodInfo: PeriodInfo, opts:
|
|||
// the ONLY safe read point for the module-level hydration global. Re-reading the
|
||||
// global later would race against this function's own history-block re-parse and
|
||||
// against concurrent requests (web-dashboard SWR, parallel MCP calls).
|
||||
let hydrationComplete: boolean | undefined
|
||||
let hydration: ReturnType<typeof sessionHydrationSnapshot> | undefined
|
||||
let effectivelyScoped = false
|
||||
if (isClaudeConfigScoped) {
|
||||
// A config source scopes Claude usage only, so scan just Claude (main.ts
|
||||
// rejects a contradictory non-Claude --provider). This also avoids parsing
|
||||
// every other provider's corpus on each scoped refresh.
|
||||
const rawProjects = fp(await parseAllSessions(periodInfo.range, 'claude'))
|
||||
hydrationComplete = isSessionHydrationComplete()
|
||||
hydration = sessionHydrationSnapshot()
|
||||
const fullProjects = daysSelection ? filterProjectsByDays(rawProjects, daysSelection.days) : rawProjects
|
||||
claudeConfigs = await claudeConfigSelector(fullProjects, requestedClaudeConfigSourceId)
|
||||
const selectedSourceId = claudeConfigs?.selectedId ?? null
|
||||
|
|
@ -607,7 +624,7 @@ export async function buildMenubarPayloadForRange(periodInfo: PeriodInfo, opts:
|
|||
exclude: opts.exclude,
|
||||
daysSelection,
|
||||
})
|
||||
hydrationComplete = isSessionHydrationComplete()
|
||||
hydration = sessionHydrationSnapshot()
|
||||
currentData = durable.data
|
||||
scanProjects = durable.liveProjects
|
||||
scanRange = durable.scanRange
|
||||
|
|
@ -951,5 +968,11 @@ export async function buildMenubarPayloadForRange(periodInfo: PeriodInfo, opts:
|
|||
const optimize = opts.optimize === false ? null : await scanAndDetect(scanProjects, scanRange, opts.provider)
|
||||
const granularRange = opts.daysSelection?.range ?? scanRange
|
||||
const granularHistory = opts.timeline === false ? undefined : buildGranularHistory(scanProjects, granularRange)
|
||||
return buildMenubarPayload(currentData, providers, optimize, dailyHistory, retryTax, routingWaste, breakdowns, claudeConfigs, granularHistory, hydrationComplete === false ? true : undefined)
|
||||
// `stale` keeps its original meaning: a read-only serve that could not see
|
||||
// real files. A first paint is incomplete for a different reason (files
|
||||
// deliberately sequenced behind it) and reports that through `hydration`
|
||||
// instead, so the two are never conflated.
|
||||
const partialFirstPaint = hydration?.deferredForFirstPaint === true
|
||||
const stale = hydration?.complete === false && !partialFirstPaint ? true : undefined
|
||||
return buildMenubarPayload(currentData, providers, optimize, dailyHistory, retryTax, routingWaste, breakdowns, claudeConfigs, granularHistory, stale, hydrationStateFor(hydration))
|
||||
}
|
||||
|
|
|
|||
139
tests/serve-progressive-hydration.test.ts
Normal file
139
tests/serve-progressive-hydration.test.ts
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
|
||||
import { spawn, type ChildProcess } from 'child_process'
|
||||
import { mkdtemp, mkdir, writeFile, rm, utimes } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
import { tmpdir } from 'os'
|
||||
|
||||
import { parseAllSessions, clearSessionCache } from '../src/parser.js'
|
||||
import { clearLoadCacheMemo } from '../src/session-cache.js'
|
||||
import { readCacheOnDisk } from './fixtures/session-cache-io.js'
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000
|
||||
|
||||
// End-to-end for the progressive cold start of the resident serve child
|
||||
// (#1110): a cold cache is answered from the files the requested period can
|
||||
// show, the answer says so in-band, and the background fill converges the
|
||||
// on-disk cache to exactly what a full cold parse would have written.
|
||||
describe('codeburn serve --stdio progressive cold start', () => {
|
||||
let home: string
|
||||
let child: ChildProcess
|
||||
let buffer = ''
|
||||
const waiters = new Map<number, (msg: Record<string, unknown>) => void>()
|
||||
let readyResolve: () => void
|
||||
const ready = new Promise<void>(resolve => { readyResolve = resolve })
|
||||
|
||||
function request(id: number, args: string[]): Promise<Record<string, unknown>> {
|
||||
return new Promise(resolve => {
|
||||
waiters.set(id, resolve)
|
||||
child.stdin!.write(JSON.stringify({ id, args }) + '\n')
|
||||
})
|
||||
}
|
||||
|
||||
async function session(name: string, ageDays: number): Promise<void> {
|
||||
const dir = join(home, '.claude', 'projects', 'proj')
|
||||
await mkdir(dir, { recursive: true })
|
||||
const at = new Date(Date.now() - ageDays * DAY_MS)
|
||||
const path = join(dir, `${name}.jsonl`)
|
||||
await writeFile(path, JSON.stringify({
|
||||
type: 'assistant',
|
||||
sessionId: name,
|
||||
timestamp: at.toISOString(),
|
||||
cwd: '/tmp/proj',
|
||||
message: {
|
||||
id: `msg-${name}`, type: 'message', role: 'assistant', model: 'claude-sonnet-4-5',
|
||||
content: [], usage: { input_tokens: 100, output_tokens: 50 },
|
||||
},
|
||||
}) + '\n')
|
||||
await utimes(path, at, at)
|
||||
}
|
||||
|
||||
const PAYLOAD_ARGS = ['status', '--format', 'menubar-json', '--period', 'week', '--no-timeline', '--no-optimize']
|
||||
|
||||
beforeAll(async () => {
|
||||
home = await mkdtemp(join(tmpdir(), 'serve-progressive-'))
|
||||
await session('recent', 1)
|
||||
await session('old', 200)
|
||||
|
||||
child = spawn(process.execPath, ['--import', 'tsx', join(__dirname, '..', 'src', 'cli.ts'), 'serve', '--stdio'], {
|
||||
stdio: ['pipe', 'pipe', 'ignore'],
|
||||
env: {
|
||||
...process.env,
|
||||
HOME: home,
|
||||
CLAUDE_CONFIG_DIR: join(home, '.claude'),
|
||||
CODEBURN_CACHE_DIR: join(home, 'cache'),
|
||||
CODEBURN_DESKTOP_SESSIONS_DIR: join(home, 'desktop-sessions'),
|
||||
// The fill normally waits for the client's opening burst to land.
|
||||
CODEBURN_SERVE_FILL_DELAY_MS: '200',
|
||||
},
|
||||
})
|
||||
child.stdout!.setEncoding('utf8')
|
||||
child.stdout!.on('data', (chunk: string) => {
|
||||
buffer += chunk
|
||||
let idx: number
|
||||
while ((idx = buffer.indexOf('\n')) >= 0) {
|
||||
const line = buffer.slice(0, idx).trim()
|
||||
buffer = buffer.slice(idx + 1)
|
||||
if (!line) continue
|
||||
const msg = JSON.parse(line) as Record<string, unknown>
|
||||
if (msg['ready']) { readyResolve(); continue }
|
||||
// Progress frames keep a waiting client's watchdog armed; they are not
|
||||
// the answer.
|
||||
if (typeof msg['progress'] === 'string') continue
|
||||
const waiter = waiters.get(msg['id'] as number)
|
||||
if (waiter) { waiters.delete(msg['id'] as number); waiter(msg) }
|
||||
}
|
||||
})
|
||||
await ready
|
||||
}, 60_000)
|
||||
|
||||
afterAll(async () => {
|
||||
child?.kill('SIGKILL')
|
||||
await rm(home, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('answers cold with labelled partial data, then converges to complete', async () => {
|
||||
const first = await request(1, PAYLOAD_ARGS)
|
||||
expect(first['ok']).toBe(true)
|
||||
const partial = JSON.parse(first['output'] as string) as {
|
||||
hydration?: { complete: boolean; indexedFiles: number; totalFiles: number }
|
||||
stale?: boolean
|
||||
}
|
||||
// Explicit partiality: a consumer can tell this apart from a final answer
|
||||
// programmatically, and it is NOT the unrelated `stale` claim.
|
||||
expect(partial.hydration?.complete).toBe(false)
|
||||
expect(partial.hydration!.totalFiles).toBeGreaterThan(partial.hydration!.indexedFiles)
|
||||
expect(partial.stale).toBeUndefined()
|
||||
|
||||
// Poll the way every consumer does. The partial answer is never memoized,
|
||||
// so a later poll re-derives (or picks up the fill's converged payload).
|
||||
let converged: { hydration?: { complete: boolean; indexedFiles: number; totalFiles: number } } | null = null
|
||||
for (let id = 2; id < 40; id++) {
|
||||
const response = await request(id, PAYLOAD_ARGS)
|
||||
const payload = JSON.parse(response['output'] as string)
|
||||
if (payload.hydration?.complete === true) { converged = payload; break }
|
||||
await new Promise(resolve => setTimeout(resolve, 250))
|
||||
}
|
||||
expect(converged).not.toBeNull()
|
||||
expect(converged!.hydration!.indexedFiles).toBe(converged!.hydration!.totalFiles)
|
||||
}, 60_000)
|
||||
|
||||
it('leaves the on-disk cache exactly where a full cold parse would', async () => {
|
||||
process.env['CODEBURN_CACHE_DIR'] = join(home, 'cache')
|
||||
clearSessionCache()
|
||||
clearLoadCacheMemo()
|
||||
const converged = await readCacheOnDisk()
|
||||
expect(converged.complete).toBe(true)
|
||||
|
||||
// Same corpus, one plain cold parse, in a cache dir of its own.
|
||||
process.env['CLAUDE_CONFIG_DIR'] = join(home, '.claude')
|
||||
process.env['CODEBURN_CACHE_DIR'] = join(home, 'cache-baseline')
|
||||
process.env['CODEBURN_DESKTOP_SESSIONS_DIR'] = join(home, 'desktop-sessions')
|
||||
clearSessionCache()
|
||||
clearLoadCacheMemo()
|
||||
await parseAllSessions()
|
||||
const baseline = await readCacheOnDisk()
|
||||
|
||||
expect(converged.providers['claude']?.files).toEqual(baseline.providers['claude']?.files)
|
||||
expect(converged.complete).toBe(baseline.complete)
|
||||
}, 60_000)
|
||||
})
|
||||
162
tests/surface-hydration.test.ts
Normal file
162
tests/surface-hydration.test.ts
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
||||
import { mkdtemp, mkdir, writeFile, rm, utimes } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
import { tmpdir } from 'os'
|
||||
|
||||
import { parseAllSessions, clearSessionCache, withColdFirstPaintFloor } from '../src/parser.js'
|
||||
import { clearLoadCacheMemo, isColdCacheOnDisk } from '../src/session-cache.js'
|
||||
import { buildMenubarPayloadForRange, SERVE_HYDRATION_ENV } from '../src/usage-aggregator.js'
|
||||
import { getDateRange } from '../src/cli-date.js'
|
||||
import { coldFirstPaintRangeStart } from '../src/serve.js'
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000
|
||||
|
||||
let tmpDir: string
|
||||
|
||||
beforeEach(async () => {
|
||||
clearSessionCache()
|
||||
clearLoadCacheMemo()
|
||||
tmpDir = await mkdtemp(join(tmpdir(), 'surface-hydration-'))
|
||||
process.env['CLAUDE_CONFIG_DIR'] = tmpDir
|
||||
process.env['CODEBURN_CACHE_DIR'] = join(tmpDir, 'cache')
|
||||
process.env['CODEBURN_DESKTOP_SESSIONS_DIR'] = join(tmpDir, 'desktop-sessions')
|
||||
delete process.env[SERVE_HYDRATION_ENV]
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
clearSessionCache()
|
||||
clearLoadCacheMemo()
|
||||
delete process.env[SERVE_HYDRATION_ENV]
|
||||
await rm(tmpDir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
async function writeSession(name: string, ageDays: number): Promise<void> {
|
||||
const dir = join(tmpDir, 'projects', 'proj')
|
||||
await mkdir(dir, { recursive: true })
|
||||
const at = new Date(Date.now() - ageDays * DAY_MS)
|
||||
const path = join(dir, `${name}.jsonl`)
|
||||
await writeFile(path, JSON.stringify({
|
||||
type: 'assistant',
|
||||
sessionId: name,
|
||||
timestamp: at.toISOString(),
|
||||
cwd: '/tmp/proj',
|
||||
message: {
|
||||
id: `msg-${name}`, type: 'message', role: 'assistant', model: 'claude-sonnet-4-5',
|
||||
content: [], usage: { input_tokens: 100, output_tokens: 50 },
|
||||
},
|
||||
}) + '\n')
|
||||
await utimes(path, at, at)
|
||||
}
|
||||
|
||||
const weekPayload = () => buildMenubarPayloadForRange(getDateRange('week'), { optimize: false, timeline: false })
|
||||
|
||||
describe('serve first-paint gate', () => {
|
||||
const start = (args: string[]) => coldFirstPaintRangeStart(args, getDateRange)
|
||||
|
||||
it('accepts only the menubar payload, the one served output that carries the marker', () => {
|
||||
expect(start(['status', '--format', 'menubar-json'])).toBeInstanceOf(Date)
|
||||
expect(start(['status', '--format=menubar-json', '--no-timeline'])).toBeInstanceOf(Date)
|
||||
// Every other served command is a one-shot shape: its consumer has no
|
||||
// in-band way to tell a partial answer from a final one.
|
||||
expect(start(['status', '--format', 'json'])).toBeNull()
|
||||
expect(start(['status'])).toBeNull()
|
||||
expect(start(['models', '--format', 'json'])).toBeNull()
|
||||
expect(start(['sessions', '--format', 'json'])).toBeNull()
|
||||
})
|
||||
|
||||
it('floors to the requested period and defaults to the CLI default', () => {
|
||||
const week = start(['status', '--format', 'menubar-json', '--period', 'week'])
|
||||
expect(week?.getTime()).toBe(getDateRange('week').range.start.getTime())
|
||||
expect(start(['status', '--format', 'menubar-json'])?.getTime())
|
||||
.toBe(getDateRange('today').range.start.getTime())
|
||||
expect(start(['status', '--format', 'menubar-json', '--period', 'nonsense'])).toBeNull()
|
||||
})
|
||||
|
||||
it('declines an explicit range, which is a question that deserves a real answer', () => {
|
||||
expect(start(['status', '--format', 'menubar-json', '--day', '2026-01-01'])).toBeNull()
|
||||
expect(start(['status', '--format', 'menubar-json', '--from', '2026-01-01'])).toBeNull()
|
||||
expect(start(['status', '--format', 'menubar-json', '--to', '2026-01-01'])).toBeNull()
|
||||
expect(start(['status', '--format', 'menubar-json', '--days', '2026-01-01,2026-01-02'])).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('hydration in the menubar payload', () => {
|
||||
it('labels a floored first paint partial, and never as stale', async () => {
|
||||
await writeSession('recent', 1)
|
||||
await writeSession('old', 200)
|
||||
process.env[SERVE_HYDRATION_ENV] = '1'
|
||||
|
||||
const { result: payload, deferredFiles } = await withColdFirstPaintFloor(
|
||||
getDateRange('week').range.start,
|
||||
weekPayload,
|
||||
)
|
||||
expect(deferredFiles).toBe(1)
|
||||
expect(payload.hydration).toEqual({ complete: false, indexedFiles: expect.any(Number), totalFiles: expect.any(Number) })
|
||||
expect(payload.hydration!.totalFiles).toBeGreaterThan(payload.hydration!.indexedFiles)
|
||||
// `stale` is a different claim (a read-only snapshot that could not see
|
||||
// real files) and must not ride along with a converging first paint.
|
||||
expect(payload.stale).toBeUndefined()
|
||||
})
|
||||
|
||||
it('reports complete once the fill has run', async () => {
|
||||
await writeSession('recent', 1)
|
||||
await writeSession('old', 200)
|
||||
process.env[SERVE_HYDRATION_ENV] = '1'
|
||||
|
||||
await withColdFirstPaintFloor(getDateRange('week').range.start, weekPayload)
|
||||
clearSessionCache()
|
||||
const payload = await weekPayload()
|
||||
|
||||
expect(payload.hydration?.complete).toBe(true)
|
||||
expect(payload.stale).toBeUndefined()
|
||||
expect(await isColdCacheOnDisk()).toBe(false)
|
||||
})
|
||||
|
||||
it('omits the field entirely outside the resident serve process', async () => {
|
||||
await writeSession('recent', 1)
|
||||
await writeSession('old', 200)
|
||||
|
||||
// The one-shot shape: a full parse, and no field for a script to mistake
|
||||
// for a completeness claim either way.
|
||||
const oneShot = await weekPayload()
|
||||
expect(oneShot.hydration).toBeUndefined()
|
||||
expect(await isColdCacheOnDisk()).toBe(false)
|
||||
|
||||
// Even under a floor, an unmarked process emits no hydration block — the
|
||||
// floor is only ever set by the serve child, which does set the marker.
|
||||
clearSessionCache()
|
||||
clearLoadCacheMemo()
|
||||
process.env['CODEBURN_CACHE_DIR'] = join(tmpDir, 'cache-2')
|
||||
const floored = await withColdFirstPaintFloor(getDateRange('week').range.start, weekPayload)
|
||||
expect(floored.deferredFiles).toBe(1)
|
||||
expect(floored.result.hydration).toBeUndefined()
|
||||
})
|
||||
|
||||
it('reports complete for a floored run that had nothing to defer', async () => {
|
||||
await writeSession('recent', 1)
|
||||
process.env[SERVE_HYDRATION_ENV] = '1'
|
||||
|
||||
const { result: payload, deferredFiles } = await withColdFirstPaintFloor(
|
||||
getDateRange('week').range.start,
|
||||
weekPayload,
|
||||
)
|
||||
expect(deferredFiles).toBe(0)
|
||||
expect(payload.hydration?.complete).toBe(true)
|
||||
})
|
||||
|
||||
it('keeps the spawn-fallback path on a full parse', async () => {
|
||||
await writeSession('recent', 1)
|
||||
await writeSession('old', 200)
|
||||
process.env[SERVE_HYDRATION_ENV] = '1'
|
||||
|
||||
// The spawn fallback runs the same command with no floor: it is a one-shot
|
||||
// that cannot converge, so it must see every file.
|
||||
const payload = await weekPayload()
|
||||
expect(payload.hydration?.complete).toBe(true)
|
||||
// Nothing pending means indexed === total, whatever the process-lifetime
|
||||
// counter happens to stand at.
|
||||
expect(payload.hydration?.indexedFiles).toBe(payload.hydration?.totalFiles)
|
||||
const sessions = (await parseAllSessions()).flatMap(p => p.sessions).map(s => s.sessionId).sort()
|
||||
expect(sessions).toEqual(['old', 'recent'])
|
||||
})
|
||||
})
|
||||
|
|
@ -100,6 +100,12 @@ vi.mock('../src/parser.js', async (importOriginal) => {
|
|||
return fixtureProjects()
|
||||
}),
|
||||
isSessionHydrationComplete: vi.fn(() => parseCalls === 1),
|
||||
sessionHydrationSnapshot: vi.fn(() => ({
|
||||
complete: parseCalls === 1,
|
||||
deferredForFirstPaint: false,
|
||||
indexedFiles: parseCalls,
|
||||
pendingFiles: 0,
|
||||
})),
|
||||
}
|
||||
})
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue