diff --git a/.github/workflows/upgrade-path.yml b/.github/workflows/upgrade-path.yml new file mode 100644 index 00000000..4791d6a8 --- /dev/null +++ b/.github/workflows/upgrade-path.yml @@ -0,0 +1,53 @@ +name: Upgrade path + +# Every existing user upgrading from the last published CLI (0.9.20) crosses the +# session-cache v7 -> v9 re-layout and the daily-cache v17 -> v19 re-derivation on +# their first run. Unit tests cover the migration in isolation on one platform; this +# job proves it against a cache that the REAL 0.9.20 binary wrote, on all three +# platforms, at both the package floor and the newest 22.x. +on: + pull_request: + paths: + - 'src/**' + - 'scripts/upgrade-path/**' + - '.github/workflows/upgrade-path.yml' + workflow_dispatch: + +jobs: + upgrade-path: + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + # Package floor, and the newest 22.x. The floor matters here: node:zlib + # gained zstd in 22.15, so dsh degrades below it (the corpus writes the + # uncompressed dsh variant so both legs still count the same numbers). + node-version: [22.13.0, 22] + + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: npm + - run: npm ci + # dist/cli.js + dist/parse-worker.js. The dash bundle is not exercised by + # any command this job runs, so the full `npm run build` is not paid for. + - run: npm run build:cli + + - name: Upgrade path from codeburn@0.9.20 + run: npm run verify:upgrade + env: + # Under runner.temp so the artifact step below can find it. The space + # is deliberate: a real Windows HOME almost always has one. + UPGRADE_PATH_WORK: ${{ runner.temp }}/codeburn upgrade path + + - name: Upload payloads on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: upgrade-path-${{ matrix.os }}-node${{ matrix.node-version }} + path: ${{ runner.temp }}/codeburn upgrade path/payloads + if-no-files-found: ignore diff --git a/package.json b/package.json index c1288196..0e12120b 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "test": "vitest run tests --exclude \"tests/cache-refresh-lock*\"", "test:locks": "vitest run tests/cache-refresh-lock.test.ts tests/cache-refresh-lock-corrupt-body.test.ts tests/cache-refresh-lock-process.test.ts --poolOptions.forks.singleFork=true", "test:watch": "vitest tests --exclude \"tests/cache-refresh-lock*\"", + "verify:upgrade": "node scripts/upgrade-path/run.mjs", "prepublishOnly": "npm run build" }, "keywords": [ diff --git a/scripts/upgrade-path/compare.mjs b/scripts/upgrade-path/compare.mjs new file mode 100644 index 00000000..0d0ef49a --- /dev/null +++ b/scripts/upgrade-path/compare.mjs @@ -0,0 +1,130 @@ +// Per-provider payload parity between the published CLI and this build. +// +// node scripts/upgrade-path/compare.mjs +// +// Each dir holds the payloads run.mjs captured: `export.json` (per-call records, +// the token/call source) and `menubar.json` (the unrounded per-provider cost). +// Prints one row per provider and exits non-zero on a diff that is not expected. +// +// Expectations, and why: +// claude, codex, gemini, kiro, cursor parse identically either side of the +// upgrade. Calls and every token field must match EXACTLY; cost is allowed +// COST_TOLERANCE of drift because the two binaries carry different bundled +// LiteLLM price snapshots and only agree when the shared pricing cache in +// CODEBURN_CACHE_DIR is warm (which it is, unless the runner is offline). +// grok changed by design in #1015: usage now comes from the CLI's own +// turn_completed records instead of a context-curve estimate. The change +// is REPORTED, never asserted — not even directionally. On real corpora +// the changelog documents totals rising, but that is a property of real +// Grok sessions, and the direction here would only reflect how the +// generator happened to size its synthetic context curve against its +// synthetic usage records. Both sides must still count the same SESSIONS, +// which is the part the corpus can honestly establish. +// dsh did not exist in the published CLI. Reported; required to be absent +// in the baseline and present after the upgrade. +const EXACT = ['claude', 'codex', 'gemini', 'kiro', 'cursor'] +const CHANGED_BY_DESIGN = ['grok'] +const NEW_IN_THIS_RELEASE = ['dsh'] + +const COST_TOLERANCE = 0.005 // 0.5% relative + +import { readFileSync } from 'node:fs' +import { join } from 'node:path' + +const [baseDir, upDir] = process.argv.slice(2) +if (!baseDir || !upDir) { + console.error('usage: compare.mjs ') + process.exit(2) +} + +const TOKEN_FIELDS = ['inputTokens', 'outputTokens', 'reasoningTokens', 'cacheWriteTokens', 'cacheReadTokens'] + +function load(dir) { + const exported = JSON.parse(readFileSync(join(dir, 'export.json'), 'utf8')) + const menubar = JSON.parse(readFileSync(join(dir, 'menubar.json'), 'utf8')) + const byProvider = {} + for (const r of exported.records ?? []) { + const p = r.provider || 'unknown' + const acc = (byProvider[p] ??= { calls: 0, cost: 0, ...Object.fromEntries(TOKEN_FIELDS.map(f => [f, 0])) }) + acc.calls++ + acc.cost += r.cost ?? 0 + for (const f of TOKEN_FIELDS) acc[f] += r[f] ?? 0 + } + // Prefer the unrounded cost, keyed by the provider's internal id. + // `providerDetails` is the only place that pairing exists — the sibling + // `providers` map is keyed by lowercased display name. The per-record sum + // above stands in when a binary predates providerDetails; it is rounded per + // record, so it is the coarser of the two. + for (const d of menubar.current?.providerDetails ?? []) { + if (byProvider[d.id]) byProvider[d.id].cost = d.cost + } + return byProvider +} + +const base = load(baseDir) +const up = load(upDir) +const providers = [...new Set([...Object.keys(base), ...Object.keys(up)])].sort() + +const failures = [] +const notes = [] +const rows = [] + +const relDiff = (a, b) => (a === 0 && b === 0 ? 0 : Math.abs(b - a) / Math.max(Math.abs(a), Math.abs(b))) +const fmt = n => (Number.isInteger(n) ? String(n) : n.toFixed(6)) + +for (const name of providers) { + const b = base[name] + const u = up[name] + let verdict + + if (NEW_IN_THIS_RELEASE.includes(name)) { + if (b) failures.push(`${name}: expected to be absent from the 0.9.20 baseline, but it reported ${b.calls} calls`) + else if (!u || u.calls === 0) failures.push(`${name}: new in this release but the upgraded run reported nothing`) + verdict = 'new (expected)' + } else if (!b || !u) { + failures.push(`${name}: present in ${b ? 'baseline' : 'upgraded'} only`) + verdict = 'MISSING' + } else if (CHANGED_BY_DESIGN.includes(name)) { + const bt = TOKEN_FIELDS.reduce((s, f) => s + b[f], 0) + const ut = TOKEN_FIELDS.reduce((s, f) => s + u[f], 0) + if (b.calls !== u.calls) failures.push(`${name}: usage accounting changed in #1015, but the session/call COUNT should not have: ${b.calls} != ${u.calls}`) + verdict = 'changed by design' + notes.push(`${name}: tokens ${bt} -> ${ut}, cost ${fmt(b.cost)} -> ${fmt(u.cost)} (#1015, expected; magnitude here is a property of the fixture, not evidence)`) + } else { + const diffs = [] + if (b.calls !== u.calls) diffs.push(`calls ${b.calls} != ${u.calls}`) + for (const f of TOKEN_FIELDS) if (b[f] !== u[f]) diffs.push(`${f} ${b[f]} != ${u[f]}`) + const costDrift = relDiff(b.cost, u.cost) + if (costDrift > COST_TOLERANCE) diffs.push(`cost ${fmt(b.cost)} != ${fmt(u.cost)} (${(costDrift * 100).toFixed(3)}% > ${(COST_TOLERANCE * 100).toFixed(1)}%)`) + if (!EXACT.includes(name)) { + notes.push(`${name}: no expectation declared in compare.mjs; ${diffs.length ? diffs.join(', ') : 'identical'}`) + verdict = diffs.length ? 'differs (unclassified)' : 'identical' + } else if (diffs.length) { + failures.push(`${name}: ${diffs.join(', ')}`) + verdict = 'DIFFERS' + } else { + verdict = costDrift === 0 ? 'identical' : `identical (cost ${(costDrift * 100).toFixed(3)}% drift)` + } + } + + rows.push({ + provider: name, + calls: `${b?.calls ?? '-'} -> ${u?.calls ?? '-'}`, + tokens: `${b ? TOKEN_FIELDS.reduce((s, f) => s + b[f], 0) : '-'} -> ${u ? TOKEN_FIELDS.reduce((s, f) => s + u[f], 0) : '-'}`, + cost: `${b ? fmt(b.cost) : '-'} -> ${u ? fmt(u.cost) : '-'}`, + verdict, + }) +} + +const cols = ['provider', 'calls', 'tokens', 'cost', 'verdict'] +const width = Object.fromEntries(cols.map(c => [c, Math.max(c.length, ...rows.map(r => r[c].length))])) +const line = r => cols.map(c => String(r[c]).padEnd(width[c])).join(' ') +console.log('') +console.log(line(Object.fromEntries(cols.map(c => [c, c.toUpperCase()])))) +console.log(cols.map(c => '-'.repeat(width[c])).join(' ')) +for (const r of rows) console.log(line(r)) +console.log('') +for (const n of notes) console.log(`note: ${n}`) +for (const f of failures) console.log(`FAIL: ${f}`) +console.log(failures.length ? `\nparity: ${failures.length} unexpected difference(s)` : '\nparity: ok') +process.exit(failures.length ? 1 : 0) diff --git a/scripts/upgrade-path/gen-corpus.mjs b/scripts/upgrade-path/gen-corpus.mjs new file mode 100644 index 00000000..a4ccbdb6 --- /dev/null +++ b/scripts/upgrade-path/gen-corpus.mjs @@ -0,0 +1,377 @@ +// Deterministic multi-provider fixture corpus for the upgrade-path check. +// +// node scripts/upgrade-path/gen-corpus.mjs +// +// Lays sessions out at each provider's DEFAULT path under , so the run +// only has to set HOME/USERPROFILE and no per-provider override var. Everything +// is seeded off a fixed constant: two invocations against the same day produce +// byte-identical files, which is what makes the worker-determinism and +// 0.9.20-vs-main payload comparisons meaningful. +// +// Day anchoring is the one thing that moves: sessions are dated relative to +// today so they land inside the daily cache's backfill window. That is fine — +// every comparison this corpus feeds happens inside a single run. + +import { mkdirSync, writeFileSync, rmSync, existsSync } from 'node:fs' +import { join } from 'node:path' +import { createRequire } from 'node:module' + +const require_ = createRequire(import.meta.url) + +const HOME = process.argv[2] +if (!HOME) { + console.error('usage: gen-corpus.mjs ') + process.exit(2) +} + +// mulberry32 — same seed, same corpus. +let seedState = 0x9e3779b9 +function rnd() { + seedState = (seedState + 0x6d2b79f5) | 0 + let t = seedState + t = Math.imul(t ^ (t >>> 15), t | 1) + t ^= t + Math.imul(t ^ (t >>> 7), t | 61) + return ((t ^ (t >>> 14)) >>> 0) / 4294967296 +} +const pick = arr => arr[Math.floor(rnd() * arr.length)] +const between = (lo, hi) => lo + Math.floor(rnd() * (hi - lo)) + +// Day 0 = 92 days ago (UTC midnight); the corpus spans day 0..91. +const DAY_MS = 86_400_000 +const SPAN_DAYS = 92 +const day0 = Math.floor(Date.now() / DAY_MS) * DAY_MS - (SPAN_DAYS - 1) * DAY_MS +const at = (day, hour, min = 0, sec = 0) => + new Date(day0 + day * DAY_MS + hour * 3_600_000 + min * 60_000 + sec * 1000) +const iso = d => d.toISOString() + +const write = (path, body) => { + mkdirSync(join(path, '..'), { recursive: true }) + writeFileSync(path, body) +} +const writeLines = (path, lines) => write(path, lines.join('\n') + '\n') + +const PROJECTS = ['/work/api-gateway', '/work/billing', '/work/web app', '/work/infra'] + +// ── claude ─────────────────────────────────────────────────────────────────── +// 204 transcripts across the span: 200 plain, 2 parent/sidechain pairs. One +// plain transcript carries a single line over 32 KB (the large-line scanner +// path); the parent/sidechain pairs exercise the v7 spawn-link capture that the +// migration has to carry forward. + +const CLAUDE_MODELS = ['claude-sonnet-4-5', 'claude-opus-4-8', 'claude-haiku-4-5'] + +function claudeUser(sessionId, ts, cwd, text) { + return JSON.stringify({ type: 'user', sessionId, timestamp: iso(ts), cwd, gitBranch: 'main', message: { role: 'user', content: text } }) +} + +function claudeAssistant(sessionId, ts, cwd, msgId, model, usage, content) { + return JSON.stringify({ + type: 'assistant', sessionId, timestamp: iso(ts), cwd, gitBranch: 'main', + message: { id: msgId, type: 'message', role: 'assistant', model, content, usage }, + }) +} + +function claudeSession(sessionId, day, cwd, turns, opts = {}) { + const lines = [] + for (let t = 0; t < turns; t++) { + const ts = at(day, 9 + (t % 8), (t * 7) % 60) + lines.push(claudeUser(sessionId, ts, cwd, `task ${t} for ${sessionId}`)) + const content = [ + { type: 'text', text: `step ${t}` }, + { type: 'tool_use', id: `tu-${sessionId}-${t}`, name: t % 3 === 0 ? 'Edit' : 'Read', input: { file_path: `${cwd}/src/f${t}.ts` } }, + ] + // One line north of 32 KB, on the file the caller asked for it on. + if (opts.hugeLineAtTurn === t) content.push({ type: 'text', text: 'y'.repeat(40 * 1024) }) + lines.push(claudeAssistant(sessionId, at(day, 9 + (t % 8), (t * 7) % 60, 30), cwd, `msg-${sessionId}-${t}`, pick(CLAUDE_MODELS), { + input_tokens: between(400, 4000), + output_tokens: between(40, 900), + cache_read_input_tokens: between(0, 20000), + cache_creation_input_tokens: between(0, 3000), + }, content)) + } + return lines +} + +function genClaude() { + const projectsDir = join(HOME, '.claude', 'projects') + let files = 0 + for (let i = 0; i < 200; i++) { + const cwd = PROJECTS[i % PROJECTS.length] + const day = (i * 7) % SPAN_DAYS + const sid = `c-${String(i).padStart(4, '0')}` + const dirName = cwd.replace(/[/ ]/g, '-') + writeLines(join(projectsDir, dirName, `${sid}.jsonl`), claudeSession(sid, day, cwd, between(4, 14), i === 137 ? { hugeLineAtTurn: 2 } : {})) + files++ + } + + // Two parent transcripts, each spawning one subagent whose transcript lives + // under /subagents/agent-.jsonl and is marked isSidechain. + for (let p = 0; p < 2; p++) { + const cwd = PROJECTS[p] + const dirName = cwd.replace(/[/ ]/g, '-') + const parent = `p-000${p}` + const agent = `a-000${p}` + const day = 40 + p * 10 + const parentLines = claudeSession(parent, day, cwd, 5) + parentLines.push(JSON.stringify({ + type: 'assistant', sessionId: parent, timestamp: iso(at(day, 12)), cwd, + message: { id: `m-spawn-${p}`, type: 'message', role: 'assistant', model: 'claude-sonnet-4-5', content: [{ type: 'tool_use', id: `toolu_spawn_${p}`, name: 'Agent', input: {} }], usage: { input_tokens: 120, output_tokens: 30 } }, + })) + parentLines.push(JSON.stringify({ + type: 'user', sessionId: parent, timestamp: iso(at(day, 12, 1)), cwd, + message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: `toolu_spawn_${p}`, content: 'subagent done' }] }, + toolUseResult: { status: 'completed', agentId: agent, content: 'subagent done' }, + })) + parentLines.push(JSON.stringify({ type: 'pr-link', sessionId: parent, timestamp: iso(at(day, 12, 2)), cwd, prUrl: `https://github.com/acme/repo/pull/${100 + p}` })) + writeLines(join(projectsDir, dirName, `${parent}.jsonl`), parentLines) + files++ + + const side = [] + for (let t = 0; t < 4; t++) { + side.push(JSON.stringify({ type: 'user', isSidechain: true, sessionId: parent, agentId: agent, timestamp: iso(at(day, 12, 3 + t)), cwd, message: { role: 'user', content: `sub task ${t}` } })) + side.push(JSON.stringify({ + type: 'assistant', isSidechain: true, sessionId: parent, agentId: agent, timestamp: iso(at(day, 12, 3 + t, 20)), cwd, + message: { id: `sub-${p}-${t}`, type: 'message', role: 'assistant', model: 'claude-opus-4-8', content: [{ type: 'text', text: 'ok' }], usage: { input_tokens: between(800, 2000), output_tokens: between(100, 400), cache_read_input_tokens: between(0, 5000) } }, + })) + } + writeLines(join(projectsDir, dirName, parent, 'subagents', `agent-${agent}.jsonl`), side) + write(join(projectsDir, dirName, parent, 'subagents', `agent-${agent}.meta.json`), JSON.stringify({ agentType: 'reviewer' })) + files++ + } + return files +} + +// ── codex ──────────────────────────────────────────────────────────────────── +// token_count carries a CUMULATIVE total_token_usage; the parser diffs +// consecutive events, so the running totals below must only ever grow. + +function genCodex() { + const root = join(HOME, '.codex', 'sessions') + let files = 0 + for (let i = 0; i < 24; i++) { + const day = (i * 4) % SPAN_DAYS + const d = new Date(day0 + day * DAY_MS) + const cwd = PROJECTS[i % PROJECTS.length] + const sid = `codex-${String(i).padStart(3, '0')}` + const lines = [JSON.stringify({ type: 'session_meta', timestamp: iso(at(day, 10)), payload: { cwd, originator: 'codex-cli', session_id: sid, model: 'gpt-5.3-codex' } })] + const total = { input_tokens: 0, cached_input_tokens: 0, output_tokens: 0, reasoning_output_tokens: 0, total_tokens: 0 } + for (let t = 0; t < between(3, 9); t++) { + const ts = iso(at(day, 10, t * 5)) + const last = { input_tokens: between(500, 6000), cached_input_tokens: between(0, 2000), output_tokens: between(50, 800), reasoning_output_tokens: between(0, 300), total_tokens: 0 } + last.total_tokens = last.input_tokens + last.output_tokens + for (const k of Object.keys(total)) total[k] += last[k] + lines.push(JSON.stringify({ type: 'event_msg', timestamp: ts, payload: { type: 'task_started' } })) + lines.push(JSON.stringify({ type: 'response_item', timestamp: ts, payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: `task ${t}` }] } })) + lines.push(JSON.stringify({ type: 'response_item', timestamp: ts, payload: { type: 'function_call', name: 'shell', call_id: `c${t}`, arguments: JSON.stringify({ command: 'ls' }) } })) + lines.push(JSON.stringify({ type: 'response_item', timestamp: ts, payload: { type: 'function_call_output', call_id: `c${t}` } })) + lines.push(JSON.stringify({ type: 'response_item', timestamp: ts, payload: { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'done' }] } })) + lines.push(JSON.stringify({ type: 'event_msg', timestamp: ts, payload: { type: 'token_count', info: { last_token_usage: last, total_token_usage: { ...total } } } })) + lines.push(JSON.stringify({ type: 'event_msg', timestamp: ts, payload: { type: 'task_complete', duration_ms: 4000 } })) + } + const dir = join(root, String(d.getUTCFullYear()), String(d.getUTCMonth() + 1).padStart(2, '0'), String(d.getUTCDate()).padStart(2, '0')) + writeLines(join(dir, `rollout-${sid}.jsonl`), lines) + files++ + } + return files +} + +// ── gemini ─────────────────────────────────────────────────────────────────── + +function genGemini() { + const messages = [] + for (let t = 0; t < 12; t++) { + messages.push({ id: `u${t}`, timestamp: iso(at(60, 9, t * 3)), type: 'user', content: `inspect ${t}` }) + messages.push({ + id: `g${t}`, timestamp: iso(at(60, 9, t * 3, 20)), type: 'gemini', content: 'reading files', + model: 'gemini-3.1-pro-preview', + tokens: { input: between(200, 3000), cached: between(0, 1000), output: between(30, 400), thoughts: between(0, 200) }, + toolCalls: [{ id: `t${t}`, name: 'read_file', args: { path: 'src/index.ts' } }], + }) + } + write(join(HOME, '.gemini', 'tmp', 'api-gateway', 'chats', 'session-upgrade-1.json'), + JSON.stringify({ sessionId: 'gemini-session-1', startTime: iso(at(60, 9)), messages })) + return 1 +} + +// ── kiro ───────────────────────────────────────────────────────────────────── + +function genKiro() { + const dir = join(HOME, '.kiro', 'sessions', 'cli') + const id = 'kiro-upgrade-1' + const lines = [] + for (let t = 0; t < 6; t++) { + lines.push(JSON.stringify({ kind: 'Prompt', data: { content: [{ kind: 'text', data: `add feature ${t}` }] } })) + lines.push(JSON.stringify({ kind: 'AssistantMessage', data: { content: [{ kind: 'text', data: `Done — added feature ${t} and its tests.` }] } })) + } + writeLines(join(dir, `${id}.jsonl`), lines) + write(join(dir, `${id}.json`), JSON.stringify({ + session_id: id, cwd: '/work/billing', + created_at: iso(at(70, 10)), updated_at: iso(at(70, 11)), + session_state: { + rts_model_state: { model_info: { model_id: 'auto' } }, + conversation_metadata: { user_turn_metadatas: [{ end_timestamp: iso(at(70, 11)), metering_usage: [] }] }, + }, + })) + return 2 +} + +// ── dsh ────────────────────────────────────────────────────────────────────── +// Written UNCOMPRESSED on purpose: node:zlib gained zstd in 22.15 and the +// package floor is 22.13, so the .zstd variant would silently drop out of the +// floor matrix leg and the two legs would not be comparable. + +function genDsh() { + const cwd = '/work/api-gateway' + const encoded = `--${cwd.replace(/[/\\]/g, '-')}--` + const dir = join(HOME, '.dsh', 'sessions', encoded, 'session-upgrade-0001') + const lines = [ + JSON.stringify({ type: 'session', version: 0, id: 'session-upgrade-0001', createdAt: at(75, 10).getTime(), cwd, delegationDepth: 0, agentPreset: 'cordis' }), + JSON.stringify({ type: 'request/header', seq: 1, time: at(75, 10).getTime(), data: { header: { config: { provider: 'deepseek-official', model: 'deepseek-v3.2', reasoningEffort: 'max', maxTokens: 256000 } } } }), + ] + let seq = 2 + for (let turn = 1; turn <= 8; turn++) { + const base = at(75, 10, turn * 5).getTime() + lines.push(JSON.stringify({ type: 'turn/start', seq: seq++, time: base, data: { turn } })) + lines.push(JSON.stringify({ type: 'user/message', seq: seq++, time: base + 100, data: { content: [{ type: 'text', text: `build ${turn}` }], source: { kind: 'user' }, role: 'user', id: `msg-${turn}` } })) + lines.push(JSON.stringify({ type: 'tool/call', seq: seq++, time: base + 200, data: { turn, step: 1, callId: `call_${turn}`, name: 'bash', arguments: JSON.stringify({ command: 'git status' }) } })) + lines.push(JSON.stringify({ + type: 'assistant/message', seq: seq++, time: base + 900, + data: { turn, step: 1, message: { role: 'assistant', content: [{ type: 'text', text: 'done' }] }, usage: { inputTokens: between(2000, 20000), outputTokens: between(100, 900), cacheReadTokens: between(0, 5000), reasoningTokens: between(0, 600) } }, + })) + } + writeLines(join(dir, 'session.jsonl'), lines) + return 1 +} + +// ── grok ───────────────────────────────────────────────────────────────────── +// Uses the authoritative `turn_completed.usage` records that #1015 switched to. + +function genGrok() { + const cwd = '/work/infra' + const root = join(HOME, '.grok', 'sessions', encodeURIComponent(cwd)) + let files = 0 + for (let i = 0; i < 3; i++) { + const id = `019edf9c-0000-7000-8000-00000000000${i + 1}` + const day = 80 + i + const dir = join(root, id) + write(join(dir, 'summary.json'), JSON.stringify({ + info: { id, cwd }, created_at: iso(at(day, 11)), updated_at: iso(at(day, 12)), last_active_at: iso(at(day, 12)), + num_messages: 12, current_model_id: 'grok-build', session_summary: 'repo work', generated_title: 'repo work', + })) + write(join(dir, 'signals.json'), JSON.stringify({ + primaryModelId: 'grok-build', modelsUsed: ['grok-build'], toolsUsed: ['read_file', 'grep'], + contextTokensUsed: 40000, contextWindowTokens: 512000, + })) + const updates = [] + let running = 0 + for (let t = 0; t < 5; t++) { + // Streamed chunk carrying the running context counter. This is all the + // published CLI can see, and what it estimates from; main ignores it in + // favour of the turn_completed record below. Both are present in a real + // session, so the corpus carries both and the two versions have something + // to disagree about. + running += between(3000, 12000) + updates.push(JSON.stringify({ + timestamp: iso(at(day, 11, t * 5)), method: 'session/update', + params: { sessionId: id, update: { sessionUpdate: 'agent_message_chunk', content: { type: 'text', text: `chunk ${t}` } }, _meta: { totalTokens: running, promptId: `p${t}`, updateType: 'AgentMessageChunk', modelId: 'grok-build' } }, + })) + const usage = { + inputTokens: between(1000, 9000), outputTokens: between(80, 700), totalTokens: 0, + cachedReadTokens: between(0, 40000), cacheCreationTokens: between(0, 2000), reasoningTokens: between(0, 300), + modelCalls: 1, apiDurationMs: 1000, costUsdTicks: 125117780000, numTurns: 1, + } + usage.totalTokens = usage.inputTokens + usage.outputTokens + usage.modelUsage = { 'grok-4.6-build': { ...usage } } + updates.push(JSON.stringify({ + timestamp: Math.floor(at(day, 11, t * 5).getTime() / 1000), method: '_x.ai/session/update', + params: { sessionId: id, update: { sessionUpdate: 'turn_completed', prompt_id: `p${t}`, usage }, _meta: { eventId: `event-${t}`, agentTimestampMs: at(day, 11, t * 5).getTime() } }, + })) + } + writeLines(join(dir, 'updates.jsonl'), updates) + files += 3 + } + return files +} + +// ── cursor (WAL-mode SQLite) ───────────────────────────────────────────────── +// Left with an un-checkpointed -wal sidecar and NO -shm: that is what a live +// Cursor database looks like on disk, and it is the shape that made +// read-only opens fail before #1017. + +function genCursor() { + let DatabaseSync + try { ({ DatabaseSync } = require_('node:sqlite')) } catch { return 0 } + + const userDir = process.platform === 'darwin' + ? join(HOME, 'Library', 'Application Support', 'Cursor', 'User') + : process.platform === 'win32' + ? join(HOME, 'AppData', 'Roaming', 'Cursor', 'User') + : join(HOME, '.config', 'Cursor', 'User') + + const globalDir = join(userDir, 'globalStorage') + mkdirSync(globalDir, { recursive: true }) + const dbPath = join(globalDir, 'state.vscdb') + for (const suffix of ['', '-wal', '-shm']) rmSync(dbPath + suffix, { force: true }) + const db = new DatabaseSync(dbPath) + db.exec('PRAGMA journal_mode=WAL') + db.exec('CREATE TABLE cursorDiskKV (key TEXT PRIMARY KEY, value BLOB)') + db.exec('CREATE TABLE ItemTable (key TEXT UNIQUE, value BLOB)') + const ins = db.prepare('INSERT INTO cursorDiskKV (key, value) VALUES (?, ?)') + + const composers = [] + for (let c = 0; c < 6; c++) { + const composerId = `composer-${c}` + composers.push({ composerId, name: `session-${c}`, unifiedMode: 'agent' }) + ins.run(`composerData:${composerId}`, JSON.stringify({ + promptTokenBreakdown: { totalUsedTokens: between(10000, 90000) }, + createdAt: at(85, 9 + c).getTime(), + })) + for (let b = 0; b < 8; b++) { + const createdAt = iso(at(85, 9 + c, b * 4)) + ins.run(`bubbleId:${composerId}:u${b}`, JSON.stringify({ type: 1, conversationId: composerId, createdAt, text: `ask ${b}`, codeBlocks: '[]' })) + ins.run(`bubbleId:${composerId}:a${b}`, JSON.stringify({ + type: 2, conversationId: composerId, createdAt, text: `reply ${b}`, codeBlocks: '[]', + tokenCount: { inputTokens: between(300, 4000), outputTokens: between(40, 500) }, + modelInfo: { modelName: 'claude-4.6-sonnet' }, + requestId: `req-${c}-${b}`, + })) + } + } + // Checkpoint what is written so far, then stop auto-checkpointing and append + // more: the tail rows live only in the -wal the copy below carries. + db.exec('PRAGMA wal_checkpoint(TRUNCATE)') + db.exec('PRAGMA wal_autocheckpoint=0') + ins.run('composerData:composer-tail', JSON.stringify({ promptTokenBreakdown: { totalUsedTokens: 12345 }, createdAt: at(86, 9).getTime() })) + ins.run('bubbleId:composer-tail:a0', JSON.stringify({ + type: 2, conversationId: 'composer-tail', createdAt: iso(at(86, 9)), text: 'tail reply', codeBlocks: '[]', + tokenCount: { inputTokens: 2222, outputTokens: 333 }, modelInfo: { modelName: 'claude-4.6-sonnet' }, + })) + composers.push({ composerId: 'composer-tail', name: 'session-tail', unifiedMode: 'agent' }) + db.close() + + // Per-workspace DB naming the composers, plus the workspace.json that gives + // the project its name. + const wsDir = join(userDir, 'workspaceStorage', 'ws0000000000000000000000000000000') + mkdirSync(wsDir, { recursive: true }) + for (const suffix of ['', '-wal', '-shm']) rmSync(join(wsDir, 'state.vscdb' + suffix), { force: true }) + const wsDb = new DatabaseSync(join(wsDir, 'state.vscdb')) + wsDb.exec('CREATE TABLE ItemTable (key TEXT UNIQUE, value BLOB)') + wsDb.prepare('INSERT INTO ItemTable (key, value) VALUES (?, ?)').run('composer.composerData', JSON.stringify({ allComposers: composers })) + wsDb.close() + write(join(wsDir, 'workspace.json'), JSON.stringify({ folder: 'file:///work/billing' })) + + return existsSync(dbPath + '-wal') ? 3 : 2 +} + +// ── run ────────────────────────────────────────────────────────────────────── + +const counts = { + claude: genClaude(), + codex: genCodex(), + gemini: genGemini(), + kiro: genKiro(), + dsh: genDsh(), + grok: genGrok(), + cursor: genCursor(), +} +console.log(JSON.stringify(counts)) diff --git a/scripts/upgrade-path/run.mjs b/scripts/upgrade-path/run.mjs new file mode 100644 index 00000000..3bd66395 --- /dev/null +++ b/scripts/upgrade-path/run.mjs @@ -0,0 +1,422 @@ +// Upgrade-path verification: prove that a cache written by the last PUBLISHED +// CLI survives this build's first run, on this platform, with this Node. +// +// npm run verify:upgrade +// +// What it does, in order: +// 1. generates a deterministic multi-provider corpus into an isolated HOME +// (whose path contains a space, because a real Windows HOME usually does) +// 2. installs codeburn@0.9.20 into an isolated global prefix and runs it, +// producing a genuine session-cache.v7 + daily-cache.v17 +// 3. installs THIS build the same way and runs it against the SAME cache dir, +// through the npm bin shim rather than `node dist/cli.js`, so +// dist/parse-worker.js has to resolve from a symlinked entry point +// 4. asserts the migration landed and compares payloads per provider +// 5. serve --stdio smoke, worker determinism, warm-run stability +// +// Env: UPGRADE_PATH_WORK (work dir), UPGRADE_PATH_OLD (published version to +// upgrade from), UPGRADE_PATH_KEEP=1 to leave the work dir behind. + +import { spawnSync, spawn } from 'node:child_process' +import { mkdirSync, rmSync, existsSync, readdirSync, statSync, readFileSync, writeFileSync, copyFileSync } from 'node:fs' +import { join, dirname } from 'node:path' +import { fileURLToPath } from 'node:url' +import { tmpdir } from 'node:os' + +const HERE = dirname(fileURLToPath(import.meta.url)) +const REPO = join(HERE, '..', '..') +const OLD_VERSION = process.env['UPGRADE_PATH_OLD'] || '0.9.20' +const WORK = process.env['UPGRADE_PATH_WORK'] || join(tmpdir(), 'codeburn upgrade path') + +// The published binary's cache versions. If a future baseline writes something +// else these two are the knobs to move, and the assertions below will say so. +const OLD_SESSION_CACHE = 'session-cache.v7.json' +const OLD_DAILY_CACHE = 'daily-cache.v17.json' +const NEW_SESSION_CACHE_DIR = 'session-cache.v9' +const NEW_DAILY_CACHE = 'daily-cache.v19.json' + +const HOME = join(WORK, 'user home') +const PAYLOADS = join(WORK, 'payloads') +const CACHES = join(WORK, 'caches') +const npmCmd = process.platform === 'win32' ? 'npm.cmd' : 'npm' + +let failures = 0 +let skipped = 0 +const step = msg => console.log(`\n=== ${msg}`) +const ok = msg => console.log(` ok ${msg}`) +const fail = msg => { failures++; console.log(` FAIL ${msg}`) } +const skip = msg => { skipped++; console.log(` skip ${msg}`) } +const check = (cond, msg) => (cond ? ok(msg) : fail(msg)) + +function run(cmd, args, opts = {}) { + const r = spawnSync(cmd, args, { encoding: 'utf8', maxBuffer: 1 << 29, shell: false, ...opts }) + if (r.error) throw new Error(`${cmd} ${args.join(' ')}: ${r.error.message}`) + return r +} + +// Node refuses to spawn a .cmd/.bat without a shell, and a shell spawn quotes +// nothing for you — so on Windows anything holding a space (every path here +// does, by design) has to be quoted by hand. +const quoteForShell = s => (process.platform === 'win32' && /[\s&|^]/.test(s) ? `"${s}"` : s) +function runShell(cmd, args, opts = {}) { + if (process.platform !== 'win32') return run(cmd, args, opts) + return run(quoteForShell(cmd), args.map(quoteForShell), { ...opts, shell: true }) +} + +function cliEnv(cacheDir, extra = {}) { + // Deliberately minimal: no provider override vars, so every provider resolves + // its own default path under the isolated HOME. APPDATA/LOCALAPPDATA are + // pinned under it too — several providers read them on Windows, and inheriting + // the runner's would let real (or leftover) data into the comparison. + const passthrough = {} + for (const k of ['PATH', 'PATHEXT', 'SystemRoot', 'ComSpec', 'windir', 'TEMP', 'TMP', 'NUMBER_OF_PROCESSORS']) { + if (process.env[k] !== undefined) passthrough[k] = process.env[k] + } + return { + ...passthrough, + HOME, USERPROFILE: HOME, TZ: 'UTC', CODEBURN_CACHE_DIR: cacheDir, + APPDATA: join(HOME, 'AppData', 'Roaming'), LOCALAPPDATA: join(HOME, 'AppData', 'Local'), + ...extra, + } +} + +function cli(bin, args, cacheDir, extra = {}) { + const r = run(bin.cmd, [...bin.args, ...args], { env: cliEnv(cacheDir, extra), cwd: WORK }) + if (r.status !== 0) throw new Error(`${args.join(' ')} exited ${r.status}\n${r.stderr?.slice(0, 4000)}`) + return r.stdout +} + +// Install into an isolated global prefix, so the CLI runs from a location that +// has nothing to do with this checkout — which is what makes dist/parse-worker.js +// resolution worth testing. On POSIX npm's bin is a symlink INTO the package and +// we drive that directly. On Windows it is a .cmd shim, which Node will not +// spawn without a shell; the payload captures go through the installed +// dist/cli.js there and the shim itself is smoke-tested once, separately. +function installGlobal(prefix, spec) { + const r = runShell(npmCmd, ['install', '-g', '--prefix', prefix, spec, '--no-audit', '--no-fund', '--loglevel', 'error'], { cwd: WORK }) + if (r.status !== 0) throw new Error(`npm install -g ${spec} exited ${r.status}\n${r.stdout}\n${r.stderr}`) + const symlink = join(prefix, 'bin', 'codeburn') + if (existsSync(symlink)) return { cmd: process.execPath, args: [symlink], shim: null } + const winCmd = join(prefix, 'codeburn.cmd') + const entry = join(prefix, 'node_modules', 'codeburn', 'dist', 'cli.js') + if (existsSync(entry)) return { cmd: process.execPath, args: [entry], shim: existsSync(winCmd) ? winCmd : null } + throw new Error(`no codeburn bin under ${prefix}`) +} + +// The npm shim, exercised once. Needs a shell on Windows, so nothing with a +// space in it is passed through here. +function checkShim(bin, cacheDir) { + if (!bin.shim) { ok("CLI invoked through npm's bin symlink"); return } + const r = runShell(bin.shim, ['--version'], { env: cliEnv(cacheDir), cwd: WORK }) + check(r.status === 0 && r.stdout.trim().length > 0, `npm .cmd shim runs: ${r.stdout.trim() || r.stderr?.slice(0, 200)}`) +} + +// Captured payloads. `export` is the token/call source, `menubar-json` the +// unrounded per-provider cost; both are stable given a fixed corpus. --period all +// so the whole three-month corpus is in scope on both sides. +function capture(bin, cacheDir, outDir, extra = {}) { + mkdirSync(outDir, { recursive: true }) + cli(bin, ['export', '--format', 'json', '--from', '2000-01-01', '--to', '2999-12-31', '-o', join(outDir, 'export.json')], cacheDir, extra) + const menubar = cli(bin, ['status', '--format', 'menubar-json', '--period', 'all', '--no-optimize', '--no-timeline'], cacheDir, extra) + writeFileSync(join(outDir, 'menubar.json'), menubar) + const status = cli(bin, ['status', '--format', 'json', '--period', 'all'], cacheDir, extra) + writeFileSync(join(outDir, 'status.json'), status) + return { menubar: JSON.parse(menubar), status: JSON.parse(status) } +} + +// The one field that moves between two runs of the same payload. +const stripGenerated = obj => JSON.parse(JSON.stringify(obj, (k, v) => (k.startsWith('generated') ? undefined : v))) + +// Shards carry real stat data and are published under a random filename, so +// "identical" means identical after normalizing both away. +function shardSnapshot(cacheDir) { + const dir = join(cacheDir, NEW_SESSION_CACHE_DIR) + if (!existsSync(dir)) return null + const out = {} + for (const name of readdirSync(dir).sort()) { + if (name === 'envelope.json') continue + const body = JSON.parse(readFileSync(join(dir, name), 'utf8')) + for (const entry of Object.values(body)) { + if (entry && typeof entry === 'object' && entry.fingerprint) { + delete entry.fingerprint.dev + delete entry.fingerprint.ino + delete entry.fingerprint.mtimeMs + } + } + // Key the bucket off the shard's provider.month prefix, dropping the nonce. + out[name.replace(/\.[0-9a-f]{16}\.json$/, '')] = sortDeep(body) + } + return out +} + +function sortDeep(v) { + if (Array.isArray(v)) return v.map(sortDeep) + if (v && typeof v === 'object') return Object.fromEntries(Object.keys(v).sort().map(k => [k, sortDeep(v[k])])) + return v +} + +function shardMtimes(cacheDir) { + const dir = join(cacheDir, NEW_SESSION_CACHE_DIR) + return Object.fromEntries(readdirSync(dir).sort().map(n => [n, statSync(join(dir, n)).mtimeMs])) +} + +// ── 1. corpus ──────────────────────────────────────────────────────────────── + +step(`work dir: ${WORK}`) +try { rmSync(WORK, { recursive: true, force: true }) } catch (err) { console.log(` note could not clear the work dir (${err.code}); reusing it`) } +mkdirSync(HOME, { recursive: true }) +mkdirSync(PAYLOADS, { recursive: true }) + +const gen = run(process.execPath, [join(HERE, 'gen-corpus.mjs'), HOME]) +if (gen.status !== 0) { console.log(gen.stderr); process.exit(1) } +ok(`corpus generated: ${gen.stdout.trim()}`) + +const upgradeCache = join(CACHES, 'upgrade') +mkdirSync(upgradeCache, { recursive: true }) + +// ── 2. baseline: the last published CLI ────────────────────────────────────── + +step(`baseline: codeburn@${OLD_VERSION}`) +const oldBin = installGlobal(join(WORK, 'old'), `codeburn@${OLD_VERSION}`) +const oldVersion = cli(oldBin, ['--version'], upgradeCache).trim() +check(oldVersion === OLD_VERSION, `installed baseline reports ${oldVersion}`) + +const baseline = capture(oldBin, upgradeCache, join(PAYLOADS, 'baseline')) +check(baseline.menubar.current.calls > 0, `baseline counted ${baseline.menubar.current.calls} calls across ${baseline.menubar.current.sessions} sessions`) +check(existsSync(join(upgradeCache, OLD_SESSION_CACHE)), `${OLD_VERSION} wrote ${OLD_SESSION_CACHE}`) +check(existsSync(join(upgradeCache, OLD_DAILY_CACHE)), `${OLD_VERSION} wrote ${OLD_DAILY_CACHE}`) + +// ── 3. upgrade: this build, same cache dir, through the npm bin shim ───────── + +step('upgrade: this build against the same cache dir') +const pack = runShell(npmCmd, ['pack', '--ignore-scripts', '--pack-destination', WORK, '--loglevel', 'error'], { cwd: REPO }) +if (pack.status !== 0) { console.log(pack.stdout, pack.stderr); process.exit(1) } +const tarball = join(WORK, pack.stdout.trim().split('\n').pop().trim()) +const newBin = installGlobal(join(WORK, 'new'), tarball) +ok(`this build installed as ${newBin.args[0] ?? newBin.cmd}`) +checkShim(newBin, upgradeCache) + +const upgraded = capture(newBin, upgradeCache, join(PAYLOADS, 'upgraded')) + +check(!existsSync(join(upgradeCache, OLD_SESSION_CACHE)), `${OLD_SESSION_CACHE} removed after the re-layout`) +check(existsSync(join(upgradeCache, NEW_SESSION_CACHE_DIR)), `${NEW_SESSION_CACHE_DIR}/ present`) +check(existsSync(join(upgradeCache, NEW_SESSION_CACHE_DIR, 'envelope.json')), `${NEW_SESSION_CACHE_DIR}/envelope.json present`) +const envelope = JSON.parse(readFileSync(join(upgradeCache, NEW_SESSION_CACHE_DIR, 'envelope.json'), 'utf8')) +check(envelope.version === 9 && Object.keys(envelope.providers ?? {}).length > 0, + `envelope at version ${envelope.version} with ${Object.keys(envelope.providers ?? {}).length} providers`) +check(readdirSync(join(upgradeCache, NEW_SESSION_CACHE_DIR)).some(n => n !== 'envelope.json'), 'shards published alongside the envelope') +check(existsSync(join(upgradeCache, NEW_DAILY_CACHE)), `${NEW_DAILY_CACHE} re-derived`) +check(existsSync(join(upgradeCache, OLD_DAILY_CACHE)), `${OLD_DAILY_CACHE} kept as the carry-forward baseline`) +const oldDays = JSON.parse(readFileSync(join(upgradeCache, OLD_DAILY_CACHE), 'utf8')).days.length +const newDays = JSON.parse(readFileSync(join(upgradeCache, NEW_DAILY_CACHE), 'utf8')).days.length +check(newDays >= oldDays, `daily history did not shrink: ${oldDays} -> ${newDays} days`) + +// ── 4. payload parity ──────────────────────────────────────────────────────── + +step('payload parity vs the baseline') +const cmp = run(process.execPath, [join(HERE, 'compare.mjs'), join(PAYLOADS, 'baseline'), join(PAYLOADS, 'upgraded')], { stdio: 'inherit' }) +if (cmp.status !== 0) failures++ + +// ── 5. serve smoke ─────────────────────────────────────────────────────────── + +step('serve --stdio') +const serveFrames = await serveSmoke() +if (serveFrames) { + for (const [name, args] of [['menubar-json', ['status', '--format', 'menubar-json', '--period', 'all', '--no-optimize', '--no-timeline']], ['models', ['models', '--format', 'json']]]) { + const frame = serveFrames.get(name) + if (!frame?.ok) { fail(`serve returned no ok frame for ${name}: ${JSON.stringify(frame)}`); continue } + ok(`serve ok frame for ${name}`) + const oneShot = cli(newBin, args, upgradeCache) + const a = JSON.stringify(stripGenerated(JSON.parse(frame.output))) + const b = JSON.stringify(stripGenerated(JSON.parse(oneShot))) + check(a === b, `serve ${name} matches the one-shot payload (ignoring generated*)`) + } +} + +async function serveSmoke() { + const child = spawn(newBin.cmd, [...newBin.args, 'serve', '--stdio'], { env: cliEnv(upgradeCache), cwd: WORK, stdio: ['pipe', 'pipe', 'pipe'] }) + const frames = new Map() + let buf = '' + let ready = false + const done = new Promise(resolve => { + child.stdout.on('data', d => { + buf += d + let nl + while ((nl = buf.indexOf('\n')) >= 0) { + const line = buf.slice(0, nl); buf = buf.slice(nl + 1) + if (!line.trim()) continue + let msg + try { msg = JSON.parse(line) } catch { continue } + if (msg.ready) { ready = true; continue } + if (msg.progress !== undefined) continue + if (msg.id === 1) frames.set('menubar-json', msg) + if (msg.id === 2) frames.set('models', msg) + if (frames.size === 2) resolve() + } + }) + }) + child.stdin.write(JSON.stringify({ id: 1, args: ['status', '--format', 'menubar-json', '--period', 'all', '--no-optimize', '--no-timeline'] }) + '\n') + child.stdin.write(JSON.stringify({ id: 2, args: ['models', '--format', 'json'] }) + '\n') + const timeout = new Promise(r => setTimeout(() => r('timeout'), 240_000)) + if ((await Promise.race([done, timeout])) === 'timeout') { child.kill(); fail('serve did not answer both requests within 240s'); return null } + check(ready, 'serve announced itself with a ready frame') + + // Closing stdin is the documented shutdown: the child must exit on its own. + const exited = new Promise(r => child.once('exit', code => r(code))) + child.stdin.end() + const exitCode = await Promise.race([exited, new Promise(r => setTimeout(() => r('hung'), 30_000))]) + if (exitCode === 'hung') { child.kill(); fail('serve did not exit when stdin closed') } + else ok(`serve exited on stdin close (code ${exitCode})`) + return frames +} + +// ── 6. worker determinism ──────────────────────────────────────────────────── + +step('parse-worker determinism (CODEBURN_PARSE_WORKERS 0 vs 3)') +const serialCache = join(CACHES, 'workers-0') +const parallelCache = join(CACHES, 'workers-3') +for (const dir of [serialCache, parallelCache]) { + mkdirSync(dir, { recursive: true }) + // Seed the shared price table so the two runs cannot be priced differently by + // a cache expiring between them. Parsing is unaffected either way. + const priced = join(upgradeCache, 'litellm-pricing.json') + if (existsSync(priced)) copyFileSync(priced, join(dir, 'litellm-pricing.json')) +} +const serialOut = capture(newBin, serialCache, join(PAYLOADS, 'workers-0'), { CODEBURN_PARSE_WORKERS: '0', CODEBURN_VERBOSE: '1' }) +const parallelOut = capture(newBin, parallelCache, join(PAYLOADS, 'workers-3'), { CODEBURN_PARSE_WORKERS: '3', CODEBURN_VERBOSE: '1' }) +check(JSON.stringify(stripGenerated(serialOut.menubar)) === JSON.stringify(stripGenerated(parallelOut.menubar)), + 'menubar-json payload identical with and without workers') +const readExport = dir => stripGenerated(JSON.parse(readFileSync(join(PAYLOADS, dir, 'export.json'), 'utf8'))) +check(JSON.stringify(readExport('workers-0')) === JSON.stringify(readExport('workers-3')), + 'export payload identical with and without workers') +check(JSON.stringify(shardSnapshot(serialCache)) === JSON.stringify(shardSnapshot(parallelCache)), + 'shard bodies identical with and without workers (fingerprint stat data and shard nonces normalized)') + +// A forced pool that never actually spawned would make the check above vacuous. +const verbose = run(newBin.cmd, [...newBin.args, 'status', '--format', 'json', '--period', 'all'], { + env: cliEnv(join(CACHES, 'workers-probe'), { CODEBURN_PARSE_WORKERS: '3', CODEBURN_VERBOSE: '1' }), cwd: WORK, +}) +const decision = (verbose.stderr || '').split('\n').filter(l => l.includes('parse workers=')) +if (decision.length === 0) skip('no "parse workers=" line on stderr; cannot confirm the pool was forced') +else check(decision.some(l => /parse workers=[1-9]/.test(l)), `worker pool engaged: ${decision.map(l => l.trim()).join(' | ')}`) + +// ── 7. second run is warm ──────────────────────────────────────────────────── + +step('second run is warm') +const beforeBodies = shardSnapshot(upgradeCache) +const beforeMtimes = shardMtimes(upgradeCache) +const warm = capture(newBin, upgradeCache, join(PAYLOADS, 'warm')) + +// The direct no-re-parse signal: the worker gate prints how many whole-file +// re-parses are pending. On an unchanged corpus that must be zero for the two +// providers big enough to be gated. +const warmVerbose = run(newBin.cmd, [...newBin.args, 'status', '--format', 'menubar-json', '--period', 'all', '--no-optimize', '--no-timeline'], { + env: cliEnv(upgradeCache, { CODEBURN_VERBOSE: '1' }), cwd: WORK, +}) +const pending = (warmVerbose.stderr || '').split('\n').filter(l => l.includes('parse workers=')) +if (pending.length === 0) skip('no "parse workers=" line on a warm run; cannot confirm nothing re-parsed') +else check(pending.every(l => /0 pending files|no full parses pending/.test(l)), + `nothing re-parsed on the warm run: ${pending.map(l => l.replace(/^codeburn: /, '').trim()).join(' | ')}`) + +check(JSON.stringify(shardSnapshot(upgradeCache)) === JSON.stringify(beforeBodies), 'warm run left every shard body unchanged') +check(JSON.stringify(stripGenerated(warm.menubar)) === JSON.stringify(stripGenerated(upgraded.menubar)), + 'warm run reports the same payload as the run that migrated the cache') + +// Republication without a content change is wasted I/O, not a correctness +// problem, so it is reported rather than failed. It is real, and it has a +// follow-up issue: a date-RANGED query (`status --format json`, the +// statusline/menubar fast path) republishes the month shards its range skipped, +// on every run, even when nothing changed — partially defeating #1007's +// "a warm launch rewrites only the month that changed". The identical-bodies +// check above is what proves the content survives it. +const afterMtimes = shardMtimes(upgradeCache) +const republished = Object.keys(afterMtimes).filter(n => n !== 'envelope.json' && beforeMtimes[n] !== afterMtimes[n]) +const retired = Object.keys(beforeMtimes).filter(n => n !== 'envelope.json' && !(n in afterMtimes)) +const churnNote = 'known defect, see #1032: a date-ranged run republishes the month shards it skipped' +if (retired.length) console.log(` note ${retired.length} shard(s) republished under a new name with identical content (${churnNote}): ${retired.join(', ')}`) +else if (republished.length) console.log(` note ${republished.length} shard(s) rewritten in place (${churnNote}): ${republished.join(', ')}`) +else ok('no shard republished on an unchanged corpus') + +// ── 8. partial source aging (release blocker, track C) ─────────────────────── +// Claude Code deletes its transcripts after ~30 days, so between one run and the +// next a day can go from fully sourced to PARTIALLY sourced. The daily cache's +// never-lose contract says a schema bump re-derives what it can and carries +// forward what it cannot — but on a partially-sourced day the re-derivation +// produces a smaller slice from the surviving files and that slice REPLACES the +// baseline one instead of being unioned with it, so the aged-out portion is lost. +// A day that aged out completely is carried forward correctly, which is what +// makes the partial case a hole rather than a missing feature. +// +// Runs last, and on its own cache dir, so mutating the corpus cannot disturb the +// parity comparison above. + +step('never-lose across partial source aging') +const agingCache = join(CACHES, 'aging') +mkdirSync(agingCache, { recursive: true }) + +// A fresh 0.9.20 cache, taken while every transcript still exists. +capture(oldBin, agingCache, join(PAYLOADS, 'aging-baseline')) +const baseDaily = JSON.parse(readFileSync(join(agingCache, OLD_DAILY_CACHE), 'utf8')) +const sliceOf = (cache, date) => cache.days.find(d => d.date === date)?.providers?.claude + +// Group transcripts by the day their turns land on. Sidechain files are left out: +// deleting a parent's subagent transcript entangles this with spawn-link +// carry-forward, which is a different contract. +const projectsDir = join(HOME, '.claude', 'projects') +const byDay = new Map() +for (const rel of readdirSync(projectsDir, { recursive: true })) { + const relPath = String(rel) + if (!relPath.endsWith('.jsonl') || relPath.includes('subagents')) continue + const full = join(projectsDir, relPath) + const first = readFileSync(full, 'utf8').split('\n', 1)[0] + const date = JSON.parse(first).timestamp?.slice(0, 10) + if (!date || !sliceOf(baseDaily, date)) continue + if (!byDay.has(date)) byDay.set(date, []) + byDay.get(date).push(full) +} + +// Densest days first, oldest among equals — the ones a retention window reaches +// first, and the ones where losing the aged-out portion shows up largest. +const candidates = [...byDay.entries()].filter(([, files]) => files.length >= 2) + .sort((a, b) => (b[1].length - a[1].length) || a[0].localeCompare(b[0])) + +let aged = [] +if (candidates.length < 3) fail(`need 3 multi-transcript claude days to age out, found ${candidates.length}`) +else { + for (const [date, files] of candidates.slice(0, 2)) { + // Keep exactly one file, so the day is still sourced — just not fully. + for (const f of files.slice(1)) rmSync(f) + aged.push({ date, kind: 'partially sourceless', kept: 1, removed: files.length - 1 }) + } + const [goneDate, goneFiles] = candidates[2] + for (const f of goneFiles) rmSync(f) + aged.push({ date: goneDate, kind: 'fully sourceless', kept: 0, removed: goneFiles.length }) + for (const a of aged) ok(`${a.date}: ${a.kind} (removed ${a.removed} of ${a.removed + a.kept} transcripts)`) + + capture(newBin, agingCache, join(PAYLOADS, 'aging-upgraded')) + const upDaily = JSON.parse(readFileSync(join(agingCache, NEW_DAILY_CACHE), 'utf8')) + const usd = n => `$${n.toFixed(6)}` + + for (const a of aged) { + const b = sliceOf(baseDaily, a.date) + const u = sliceOf(upDaily, a.date) + if (!u) { fail(`${a.date} (${a.kind}): the claude slice is gone entirely; baseline had ${usd(b.cost)} over ${b.calls} calls`); continue } + // A fully sourceless day has nothing to re-derive, so it must come back + // EXACTLY. A partially sourceless one may legitimately grow (a re-parse + // under new accounting), but must never shrink. + const exact = a.kind === 'fully sourceless' + const costOk = exact ? Math.abs(u.cost - b.cost) < 1e-9 : u.cost >= b.cost - 1e-9 + const callsOk = exact ? u.calls === b.calls : u.calls >= b.calls + const loss = costOk && callsOk ? '' : + ` — LOST ${usd(b.cost - u.cost)} (${(100 * (b.cost - u.cost) / b.cost).toFixed(1)}%) and ${b.calls - u.calls} calls` + check(costOk && callsOk, + `${a.date} (${a.kind}): cost ${usd(b.cost)} -> ${usd(u.cost)}, calls ${b.calls} -> ${u.calls}${loss}`) + } +} + +// ── done ───────────────────────────────────────────────────────────────────── + +console.log(`\n${failures ? `FAILED: ${failures} check(s)` : 'PASSED'}${skipped ? ` (${skipped} skipped)` : ''}`) +if (failures && process.env['UPGRADE_PATH_KEEP'] !== '0') console.log(`payloads left in: ${PAYLOADS}`) +else if (process.env['UPGRADE_PATH_KEEP'] !== '1') { try { rmSync(WORK, { recursive: true, force: true }) } catch { /* windows file locks */ } } +process.exit(failures ? 1 : 0)