mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-08-21 06:24:32 +00:00
Merge remote-tracking branch 'origin/main' into pr1040-rebase
This commit is contained in:
iamtoruk
commit
952a301877
5 changed files with 983 additions and 0 deletions
53
.github/workflows/upgrade-path.yml
vendored
Normal file
53
.github/workflows/upgrade-path.yml
vendored
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
name: Upgrade path
|
||||
|
||||
# Every existing user upgrading from the last published CLI (0.9.20) crosses the
|
||||
# session-cache v7 -> v9 re-layout and the daily-cache v17 -> v19 re-derivation on
|
||||
# their first run. Unit tests cover the migration in isolation on one platform; this
|
||||
# job proves it against a cache that the REAL 0.9.20 binary wrote, on all three
|
||||
# platforms, at both the package floor and the newest 22.x.
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'src/**'
|
||||
- 'scripts/upgrade-path/**'
|
||||
- '.github/workflows/upgrade-path.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
upgrade-path:
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-latest]
|
||||
# Package floor, and the newest 22.x. The floor matters here: node:zlib
|
||||
# gained zstd in 22.15, so dsh degrades below it (the corpus writes the
|
||||
# uncompressed dsh variant so both legs still count the same numbers).
|
||||
node-version: [22.13.0, 22]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ matrix.node-version }}
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
# dist/cli.js + dist/parse-worker.js. The dash bundle is not exercised by
|
||||
# any command this job runs, so the full `npm run build` is not paid for.
|
||||
- run: npm run build:cli
|
||||
|
||||
- name: Upgrade path from codeburn@0.9.20
|
||||
run: npm run verify:upgrade
|
||||
env:
|
||||
# Under runner.temp so the artifact step below can find it. The space
|
||||
# is deliberate: a real Windows HOME almost always has one.
|
||||
UPGRADE_PATH_WORK: ${{ runner.temp }}/codeburn upgrade path
|
||||
|
||||
- name: Upload payloads on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: upgrade-path-${{ matrix.os }}-node${{ matrix.node-version }}
|
||||
path: ${{ runner.temp }}/codeburn upgrade path/payloads
|
||||
if-no-files-found: ignore
|
||||
|
|
@ -21,6 +21,7 @@
|
|||
"test": "vitest run tests --exclude \"tests/cache-refresh-lock*\"",
|
||||
"test:locks": "vitest run tests/cache-refresh-lock.test.ts tests/cache-refresh-lock-corrupt-body.test.ts tests/cache-refresh-lock-process.test.ts --poolOptions.forks.singleFork=true",
|
||||
"test:watch": "vitest tests --exclude \"tests/cache-refresh-lock*\"",
|
||||
"verify:upgrade": "node scripts/upgrade-path/run.mjs",
|
||||
"prepublishOnly": "npm run build"
|
||||
},
|
||||
"keywords": [
|
||||
|
|
|
|||
130
scripts/upgrade-path/compare.mjs
Normal file
130
scripts/upgrade-path/compare.mjs
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
// Per-provider payload parity between the published CLI and this build.
|
||||
//
|
||||
// node scripts/upgrade-path/compare.mjs <baselineDir> <upgradedDir>
|
||||
//
|
||||
// Each dir holds the payloads run.mjs captured: `export.json` (per-call records,
|
||||
// the token/call source) and `menubar.json` (the unrounded per-provider cost).
|
||||
// Prints one row per provider and exits non-zero on a diff that is not expected.
|
||||
//
|
||||
// Expectations, and why:
|
||||
// claude, codex, gemini, kiro, cursor parse identically either side of the
|
||||
// upgrade. Calls and every token field must match EXACTLY; cost is allowed
|
||||
// COST_TOLERANCE of drift because the two binaries carry different bundled
|
||||
// LiteLLM price snapshots and only agree when the shared pricing cache in
|
||||
// CODEBURN_CACHE_DIR is warm (which it is, unless the runner is offline).
|
||||
// grok changed by design in #1015: usage now comes from the CLI's own
|
||||
// turn_completed records instead of a context-curve estimate. The change
|
||||
// is REPORTED, never asserted — not even directionally. On real corpora
|
||||
// the changelog documents totals rising, but that is a property of real
|
||||
// Grok sessions, and the direction here would only reflect how the
|
||||
// generator happened to size its synthetic context curve against its
|
||||
// synthetic usage records. Both sides must still count the same SESSIONS,
|
||||
// which is the part the corpus can honestly establish.
|
||||
// dsh did not exist in the published CLI. Reported; required to be absent
|
||||
// in the baseline and present after the upgrade.
|
||||
const EXACT = ['claude', 'codex', 'gemini', 'kiro', 'cursor']
|
||||
const CHANGED_BY_DESIGN = ['grok']
|
||||
const NEW_IN_THIS_RELEASE = ['dsh']
|
||||
|
||||
const COST_TOLERANCE = 0.005 // 0.5% relative
|
||||
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
|
||||
const [baseDir, upDir] = process.argv.slice(2)
|
||||
if (!baseDir || !upDir) {
|
||||
console.error('usage: compare.mjs <baselineDir> <upgradedDir>')
|
||||
process.exit(2)
|
||||
}
|
||||
|
||||
const TOKEN_FIELDS = ['inputTokens', 'outputTokens', 'reasoningTokens', 'cacheWriteTokens', 'cacheReadTokens']
|
||||
|
||||
function load(dir) {
|
||||
const exported = JSON.parse(readFileSync(join(dir, 'export.json'), 'utf8'))
|
||||
const menubar = JSON.parse(readFileSync(join(dir, 'menubar.json'), 'utf8'))
|
||||
const byProvider = {}
|
||||
for (const r of exported.records ?? []) {
|
||||
const p = r.provider || 'unknown'
|
||||
const acc = (byProvider[p] ??= { calls: 0, cost: 0, ...Object.fromEntries(TOKEN_FIELDS.map(f => [f, 0])) })
|
||||
acc.calls++
|
||||
acc.cost += r.cost ?? 0
|
||||
for (const f of TOKEN_FIELDS) acc[f] += r[f] ?? 0
|
||||
}
|
||||
// Prefer the unrounded cost, keyed by the provider's internal id.
|
||||
// `providerDetails` is the only place that pairing exists — the sibling
|
||||
// `providers` map is keyed by lowercased display name. The per-record sum
|
||||
// above stands in when a binary predates providerDetails; it is rounded per
|
||||
// record, so it is the coarser of the two.
|
||||
for (const d of menubar.current?.providerDetails ?? []) {
|
||||
if (byProvider[d.id]) byProvider[d.id].cost = d.cost
|
||||
}
|
||||
return byProvider
|
||||
}
|
||||
|
||||
const base = load(baseDir)
|
||||
const up = load(upDir)
|
||||
const providers = [...new Set([...Object.keys(base), ...Object.keys(up)])].sort()
|
||||
|
||||
const failures = []
|
||||
const notes = []
|
||||
const rows = []
|
||||
|
||||
const relDiff = (a, b) => (a === 0 && b === 0 ? 0 : Math.abs(b - a) / Math.max(Math.abs(a), Math.abs(b)))
|
||||
const fmt = n => (Number.isInteger(n) ? String(n) : n.toFixed(6))
|
||||
|
||||
for (const name of providers) {
|
||||
const b = base[name]
|
||||
const u = up[name]
|
||||
let verdict
|
||||
|
||||
if (NEW_IN_THIS_RELEASE.includes(name)) {
|
||||
if (b) failures.push(`${name}: expected to be absent from the 0.9.20 baseline, but it reported ${b.calls} calls`)
|
||||
else if (!u || u.calls === 0) failures.push(`${name}: new in this release but the upgraded run reported nothing`)
|
||||
verdict = 'new (expected)'
|
||||
} else if (!b || !u) {
|
||||
failures.push(`${name}: present in ${b ? 'baseline' : 'upgraded'} only`)
|
||||
verdict = 'MISSING'
|
||||
} else if (CHANGED_BY_DESIGN.includes(name)) {
|
||||
const bt = TOKEN_FIELDS.reduce((s, f) => s + b[f], 0)
|
||||
const ut = TOKEN_FIELDS.reduce((s, f) => s + u[f], 0)
|
||||
if (b.calls !== u.calls) failures.push(`${name}: usage accounting changed in #1015, but the session/call COUNT should not have: ${b.calls} != ${u.calls}`)
|
||||
verdict = 'changed by design'
|
||||
notes.push(`${name}: tokens ${bt} -> ${ut}, cost ${fmt(b.cost)} -> ${fmt(u.cost)} (#1015, expected; magnitude here is a property of the fixture, not evidence)`)
|
||||
} else {
|
||||
const diffs = []
|
||||
if (b.calls !== u.calls) diffs.push(`calls ${b.calls} != ${u.calls}`)
|
||||
for (const f of TOKEN_FIELDS) if (b[f] !== u[f]) diffs.push(`${f} ${b[f]} != ${u[f]}`)
|
||||
const costDrift = relDiff(b.cost, u.cost)
|
||||
if (costDrift > COST_TOLERANCE) diffs.push(`cost ${fmt(b.cost)} != ${fmt(u.cost)} (${(costDrift * 100).toFixed(3)}% > ${(COST_TOLERANCE * 100).toFixed(1)}%)`)
|
||||
if (!EXACT.includes(name)) {
|
||||
notes.push(`${name}: no expectation declared in compare.mjs; ${diffs.length ? diffs.join(', ') : 'identical'}`)
|
||||
verdict = diffs.length ? 'differs (unclassified)' : 'identical'
|
||||
} else if (diffs.length) {
|
||||
failures.push(`${name}: ${diffs.join(', ')}`)
|
||||
verdict = 'DIFFERS'
|
||||
} else {
|
||||
verdict = costDrift === 0 ? 'identical' : `identical (cost ${(costDrift * 100).toFixed(3)}% drift)`
|
||||
}
|
||||
}
|
||||
|
||||
rows.push({
|
||||
provider: name,
|
||||
calls: `${b?.calls ?? '-'} -> ${u?.calls ?? '-'}`,
|
||||
tokens: `${b ? TOKEN_FIELDS.reduce((s, f) => s + b[f], 0) : '-'} -> ${u ? TOKEN_FIELDS.reduce((s, f) => s + u[f], 0) : '-'}`,
|
||||
cost: `${b ? fmt(b.cost) : '-'} -> ${u ? fmt(u.cost) : '-'}`,
|
||||
verdict,
|
||||
})
|
||||
}
|
||||
|
||||
const cols = ['provider', 'calls', 'tokens', 'cost', 'verdict']
|
||||
const width = Object.fromEntries(cols.map(c => [c, Math.max(c.length, ...rows.map(r => r[c].length))]))
|
||||
const line = r => cols.map(c => String(r[c]).padEnd(width[c])).join(' ')
|
||||
console.log('')
|
||||
console.log(line(Object.fromEntries(cols.map(c => [c, c.toUpperCase()]))))
|
||||
console.log(cols.map(c => '-'.repeat(width[c])).join(' '))
|
||||
for (const r of rows) console.log(line(r))
|
||||
console.log('')
|
||||
for (const n of notes) console.log(`note: ${n}`)
|
||||
for (const f of failures) console.log(`FAIL: ${f}`)
|
||||
console.log(failures.length ? `\nparity: ${failures.length} unexpected difference(s)` : '\nparity: ok')
|
||||
process.exit(failures.length ? 1 : 0)
|
||||
377
scripts/upgrade-path/gen-corpus.mjs
Normal file
377
scripts/upgrade-path/gen-corpus.mjs
Normal file
|
|
@ -0,0 +1,377 @@
|
|||
// Deterministic multi-provider fixture corpus for the upgrade-path check.
|
||||
//
|
||||
// node scripts/upgrade-path/gen-corpus.mjs <homeDir>
|
||||
//
|
||||
// Lays sessions out at each provider's DEFAULT path under <homeDir>, so the run
|
||||
// only has to set HOME/USERPROFILE and no per-provider override var. Everything
|
||||
// is seeded off a fixed constant: two invocations against the same day produce
|
||||
// byte-identical files, which is what makes the worker-determinism and
|
||||
// 0.9.20-vs-main payload comparisons meaningful.
|
||||
//
|
||||
// Day anchoring is the one thing that moves: sessions are dated relative to
|
||||
// today so they land inside the daily cache's backfill window. That is fine —
|
||||
// every comparison this corpus feeds happens inside a single run.
|
||||
|
||||
import { mkdirSync, writeFileSync, rmSync, existsSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { createRequire } from 'node:module'
|
||||
|
||||
const require_ = createRequire(import.meta.url)
|
||||
|
||||
const HOME = process.argv[2]
|
||||
if (!HOME) {
|
||||
console.error('usage: gen-corpus.mjs <homeDir>')
|
||||
process.exit(2)
|
||||
}
|
||||
|
||||
// mulberry32 — same seed, same corpus.
|
||||
let seedState = 0x9e3779b9
|
||||
function rnd() {
|
||||
seedState = (seedState + 0x6d2b79f5) | 0
|
||||
let t = seedState
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1)
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61)
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296
|
||||
}
|
||||
const pick = arr => arr[Math.floor(rnd() * arr.length)]
|
||||
const between = (lo, hi) => lo + Math.floor(rnd() * (hi - lo))
|
||||
|
||||
// Day 0 = 92 days ago (UTC midnight); the corpus spans day 0..91.
|
||||
const DAY_MS = 86_400_000
|
||||
const SPAN_DAYS = 92
|
||||
const day0 = Math.floor(Date.now() / DAY_MS) * DAY_MS - (SPAN_DAYS - 1) * DAY_MS
|
||||
const at = (day, hour, min = 0, sec = 0) =>
|
||||
new Date(day0 + day * DAY_MS + hour * 3_600_000 + min * 60_000 + sec * 1000)
|
||||
const iso = d => d.toISOString()
|
||||
|
||||
const write = (path, body) => {
|
||||
mkdirSync(join(path, '..'), { recursive: true })
|
||||
writeFileSync(path, body)
|
||||
}
|
||||
const writeLines = (path, lines) => write(path, lines.join('\n') + '\n')
|
||||
|
||||
const PROJECTS = ['/work/api-gateway', '/work/billing', '/work/web app', '/work/infra']
|
||||
|
||||
// ── claude ───────────────────────────────────────────────────────────────────
|
||||
// 204 transcripts across the span: 200 plain, 2 parent/sidechain pairs. One
|
||||
// plain transcript carries a single line over 32 KB (the large-line scanner
|
||||
// path); the parent/sidechain pairs exercise the v7 spawn-link capture that the
|
||||
// migration has to carry forward.
|
||||
|
||||
const CLAUDE_MODELS = ['claude-sonnet-4-5', 'claude-opus-4-8', 'claude-haiku-4-5']
|
||||
|
||||
function claudeUser(sessionId, ts, cwd, text) {
|
||||
return JSON.stringify({ type: 'user', sessionId, timestamp: iso(ts), cwd, gitBranch: 'main', message: { role: 'user', content: text } })
|
||||
}
|
||||
|
||||
function claudeAssistant(sessionId, ts, cwd, msgId, model, usage, content) {
|
||||
return JSON.stringify({
|
||||
type: 'assistant', sessionId, timestamp: iso(ts), cwd, gitBranch: 'main',
|
||||
message: { id: msgId, type: 'message', role: 'assistant', model, content, usage },
|
||||
})
|
||||
}
|
||||
|
||||
function claudeSession(sessionId, day, cwd, turns, opts = {}) {
|
||||
const lines = []
|
||||
for (let t = 0; t < turns; t++) {
|
||||
const ts = at(day, 9 + (t % 8), (t * 7) % 60)
|
||||
lines.push(claudeUser(sessionId, ts, cwd, `task ${t} for ${sessionId}`))
|
||||
const content = [
|
||||
{ type: 'text', text: `step ${t}` },
|
||||
{ type: 'tool_use', id: `tu-${sessionId}-${t}`, name: t % 3 === 0 ? 'Edit' : 'Read', input: { file_path: `${cwd}/src/f${t}.ts` } },
|
||||
]
|
||||
// One line north of 32 KB, on the file the caller asked for it on.
|
||||
if (opts.hugeLineAtTurn === t) content.push({ type: 'text', text: 'y'.repeat(40 * 1024) })
|
||||
lines.push(claudeAssistant(sessionId, at(day, 9 + (t % 8), (t * 7) % 60, 30), cwd, `msg-${sessionId}-${t}`, pick(CLAUDE_MODELS), {
|
||||
input_tokens: between(400, 4000),
|
||||
output_tokens: between(40, 900),
|
||||
cache_read_input_tokens: between(0, 20000),
|
||||
cache_creation_input_tokens: between(0, 3000),
|
||||
}, content))
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
function genClaude() {
|
||||
const projectsDir = join(HOME, '.claude', 'projects')
|
||||
let files = 0
|
||||
for (let i = 0; i < 200; i++) {
|
||||
const cwd = PROJECTS[i % PROJECTS.length]
|
||||
const day = (i * 7) % SPAN_DAYS
|
||||
const sid = `c-${String(i).padStart(4, '0')}`
|
||||
const dirName = cwd.replace(/[/ ]/g, '-')
|
||||
writeLines(join(projectsDir, dirName, `${sid}.jsonl`), claudeSession(sid, day, cwd, between(4, 14), i === 137 ? { hugeLineAtTurn: 2 } : {}))
|
||||
files++
|
||||
}
|
||||
|
||||
// Two parent transcripts, each spawning one subagent whose transcript lives
|
||||
// under <parent-uuid>/subagents/agent-<id>.jsonl and is marked isSidechain.
|
||||
for (let p = 0; p < 2; p++) {
|
||||
const cwd = PROJECTS[p]
|
||||
const dirName = cwd.replace(/[/ ]/g, '-')
|
||||
const parent = `p-000${p}`
|
||||
const agent = `a-000${p}`
|
||||
const day = 40 + p * 10
|
||||
const parentLines = claudeSession(parent, day, cwd, 5)
|
||||
parentLines.push(JSON.stringify({
|
||||
type: 'assistant', sessionId: parent, timestamp: iso(at(day, 12)), cwd,
|
||||
message: { id: `m-spawn-${p}`, type: 'message', role: 'assistant', model: 'claude-sonnet-4-5', content: [{ type: 'tool_use', id: `toolu_spawn_${p}`, name: 'Agent', input: {} }], usage: { input_tokens: 120, output_tokens: 30 } },
|
||||
}))
|
||||
parentLines.push(JSON.stringify({
|
||||
type: 'user', sessionId: parent, timestamp: iso(at(day, 12, 1)), cwd,
|
||||
message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: `toolu_spawn_${p}`, content: 'subagent done' }] },
|
||||
toolUseResult: { status: 'completed', agentId: agent, content: 'subagent done' },
|
||||
}))
|
||||
parentLines.push(JSON.stringify({ type: 'pr-link', sessionId: parent, timestamp: iso(at(day, 12, 2)), cwd, prUrl: `https://github.com/acme/repo/pull/${100 + p}` }))
|
||||
writeLines(join(projectsDir, dirName, `${parent}.jsonl`), parentLines)
|
||||
files++
|
||||
|
||||
const side = []
|
||||
for (let t = 0; t < 4; t++) {
|
||||
side.push(JSON.stringify({ type: 'user', isSidechain: true, sessionId: parent, agentId: agent, timestamp: iso(at(day, 12, 3 + t)), cwd, message: { role: 'user', content: `sub task ${t}` } }))
|
||||
side.push(JSON.stringify({
|
||||
type: 'assistant', isSidechain: true, sessionId: parent, agentId: agent, timestamp: iso(at(day, 12, 3 + t, 20)), cwd,
|
||||
message: { id: `sub-${p}-${t}`, type: 'message', role: 'assistant', model: 'claude-opus-4-8', content: [{ type: 'text', text: 'ok' }], usage: { input_tokens: between(800, 2000), output_tokens: between(100, 400), cache_read_input_tokens: between(0, 5000) } },
|
||||
}))
|
||||
}
|
||||
writeLines(join(projectsDir, dirName, parent, 'subagents', `agent-${agent}.jsonl`), side)
|
||||
write(join(projectsDir, dirName, parent, 'subagents', `agent-${agent}.meta.json`), JSON.stringify({ agentType: 'reviewer' }))
|
||||
files++
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
// ── codex ────────────────────────────────────────────────────────────────────
|
||||
// token_count carries a CUMULATIVE total_token_usage; the parser diffs
|
||||
// consecutive events, so the running totals below must only ever grow.
|
||||
|
||||
function genCodex() {
|
||||
const root = join(HOME, '.codex', 'sessions')
|
||||
let files = 0
|
||||
for (let i = 0; i < 24; i++) {
|
||||
const day = (i * 4) % SPAN_DAYS
|
||||
const d = new Date(day0 + day * DAY_MS)
|
||||
const cwd = PROJECTS[i % PROJECTS.length]
|
||||
const sid = `codex-${String(i).padStart(3, '0')}`
|
||||
const lines = [JSON.stringify({ type: 'session_meta', timestamp: iso(at(day, 10)), payload: { cwd, originator: 'codex-cli', session_id: sid, model: 'gpt-5.3-codex' } })]
|
||||
const total = { input_tokens: 0, cached_input_tokens: 0, output_tokens: 0, reasoning_output_tokens: 0, total_tokens: 0 }
|
||||
for (let t = 0; t < between(3, 9); t++) {
|
||||
const ts = iso(at(day, 10, t * 5))
|
||||
const last = { input_tokens: between(500, 6000), cached_input_tokens: between(0, 2000), output_tokens: between(50, 800), reasoning_output_tokens: between(0, 300), total_tokens: 0 }
|
||||
last.total_tokens = last.input_tokens + last.output_tokens
|
||||
for (const k of Object.keys(total)) total[k] += last[k]
|
||||
lines.push(JSON.stringify({ type: 'event_msg', timestamp: ts, payload: { type: 'task_started' } }))
|
||||
lines.push(JSON.stringify({ type: 'response_item', timestamp: ts, payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: `task ${t}` }] } }))
|
||||
lines.push(JSON.stringify({ type: 'response_item', timestamp: ts, payload: { type: 'function_call', name: 'shell', call_id: `c${t}`, arguments: JSON.stringify({ command: 'ls' }) } }))
|
||||
lines.push(JSON.stringify({ type: 'response_item', timestamp: ts, payload: { type: 'function_call_output', call_id: `c${t}` } }))
|
||||
lines.push(JSON.stringify({ type: 'response_item', timestamp: ts, payload: { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'done' }] } }))
|
||||
lines.push(JSON.stringify({ type: 'event_msg', timestamp: ts, payload: { type: 'token_count', info: { last_token_usage: last, total_token_usage: { ...total } } } }))
|
||||
lines.push(JSON.stringify({ type: 'event_msg', timestamp: ts, payload: { type: 'task_complete', duration_ms: 4000 } }))
|
||||
}
|
||||
const dir = join(root, String(d.getUTCFullYear()), String(d.getUTCMonth() + 1).padStart(2, '0'), String(d.getUTCDate()).padStart(2, '0'))
|
||||
writeLines(join(dir, `rollout-${sid}.jsonl`), lines)
|
||||
files++
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
// ── gemini ───────────────────────────────────────────────────────────────────
|
||||
|
||||
function genGemini() {
|
||||
const messages = []
|
||||
for (let t = 0; t < 12; t++) {
|
||||
messages.push({ id: `u${t}`, timestamp: iso(at(60, 9, t * 3)), type: 'user', content: `inspect ${t}` })
|
||||
messages.push({
|
||||
id: `g${t}`, timestamp: iso(at(60, 9, t * 3, 20)), type: 'gemini', content: 'reading files',
|
||||
model: 'gemini-3.1-pro-preview',
|
||||
tokens: { input: between(200, 3000), cached: between(0, 1000), output: between(30, 400), thoughts: between(0, 200) },
|
||||
toolCalls: [{ id: `t${t}`, name: 'read_file', args: { path: 'src/index.ts' } }],
|
||||
})
|
||||
}
|
||||
write(join(HOME, '.gemini', 'tmp', 'api-gateway', 'chats', 'session-upgrade-1.json'),
|
||||
JSON.stringify({ sessionId: 'gemini-session-1', startTime: iso(at(60, 9)), messages }))
|
||||
return 1
|
||||
}
|
||||
|
||||
// ── kiro ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
function genKiro() {
|
||||
const dir = join(HOME, '.kiro', 'sessions', 'cli')
|
||||
const id = 'kiro-upgrade-1'
|
||||
const lines = []
|
||||
for (let t = 0; t < 6; t++) {
|
||||
lines.push(JSON.stringify({ kind: 'Prompt', data: { content: [{ kind: 'text', data: `add feature ${t}` }] } }))
|
||||
lines.push(JSON.stringify({ kind: 'AssistantMessage', data: { content: [{ kind: 'text', data: `Done — added feature ${t} and its tests.` }] } }))
|
||||
}
|
||||
writeLines(join(dir, `${id}.jsonl`), lines)
|
||||
write(join(dir, `${id}.json`), JSON.stringify({
|
||||
session_id: id, cwd: '/work/billing',
|
||||
created_at: iso(at(70, 10)), updated_at: iso(at(70, 11)),
|
||||
session_state: {
|
||||
rts_model_state: { model_info: { model_id: 'auto' } },
|
||||
conversation_metadata: { user_turn_metadatas: [{ end_timestamp: iso(at(70, 11)), metering_usage: [] }] },
|
||||
},
|
||||
}))
|
||||
return 2
|
||||
}
|
||||
|
||||
// ── dsh ──────────────────────────────────────────────────────────────────────
|
||||
// Written UNCOMPRESSED on purpose: node:zlib gained zstd in 22.15 and the
|
||||
// package floor is 22.13, so the .zstd variant would silently drop out of the
|
||||
// floor matrix leg and the two legs would not be comparable.
|
||||
|
||||
function genDsh() {
|
||||
const cwd = '/work/api-gateway'
|
||||
const encoded = `--${cwd.replace(/[/\\]/g, '-')}--`
|
||||
const dir = join(HOME, '.dsh', 'sessions', encoded, 'session-upgrade-0001')
|
||||
const lines = [
|
||||
JSON.stringify({ type: 'session', version: 0, id: 'session-upgrade-0001', createdAt: at(75, 10).getTime(), cwd, delegationDepth: 0, agentPreset: 'cordis' }),
|
||||
JSON.stringify({ type: 'request/header', seq: 1, time: at(75, 10).getTime(), data: { header: { config: { provider: 'deepseek-official', model: 'deepseek-v3.2', reasoningEffort: 'max', maxTokens: 256000 } } } }),
|
||||
]
|
||||
let seq = 2
|
||||
for (let turn = 1; turn <= 8; turn++) {
|
||||
const base = at(75, 10, turn * 5).getTime()
|
||||
lines.push(JSON.stringify({ type: 'turn/start', seq: seq++, time: base, data: { turn } }))
|
||||
lines.push(JSON.stringify({ type: 'user/message', seq: seq++, time: base + 100, data: { content: [{ type: 'text', text: `build ${turn}` }], source: { kind: 'user' }, role: 'user', id: `msg-${turn}` } }))
|
||||
lines.push(JSON.stringify({ type: 'tool/call', seq: seq++, time: base + 200, data: { turn, step: 1, callId: `call_${turn}`, name: 'bash', arguments: JSON.stringify({ command: 'git status' }) } }))
|
||||
lines.push(JSON.stringify({
|
||||
type: 'assistant/message', seq: seq++, time: base + 900,
|
||||
data: { turn, step: 1, message: { role: 'assistant', content: [{ type: 'text', text: 'done' }] }, usage: { inputTokens: between(2000, 20000), outputTokens: between(100, 900), cacheReadTokens: between(0, 5000), reasoningTokens: between(0, 600) } },
|
||||
}))
|
||||
}
|
||||
writeLines(join(dir, 'session.jsonl'), lines)
|
||||
return 1
|
||||
}
|
||||
|
||||
// ── grok ─────────────────────────────────────────────────────────────────────
|
||||
// Uses the authoritative `turn_completed.usage` records that #1015 switched to.
|
||||
|
||||
function genGrok() {
|
||||
const cwd = '/work/infra'
|
||||
const root = join(HOME, '.grok', 'sessions', encodeURIComponent(cwd))
|
||||
let files = 0
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const id = `019edf9c-0000-7000-8000-00000000000${i + 1}`
|
||||
const day = 80 + i
|
||||
const dir = join(root, id)
|
||||
write(join(dir, 'summary.json'), JSON.stringify({
|
||||
info: { id, cwd }, created_at: iso(at(day, 11)), updated_at: iso(at(day, 12)), last_active_at: iso(at(day, 12)),
|
||||
num_messages: 12, current_model_id: 'grok-build', session_summary: 'repo work', generated_title: 'repo work',
|
||||
}))
|
||||
write(join(dir, 'signals.json'), JSON.stringify({
|
||||
primaryModelId: 'grok-build', modelsUsed: ['grok-build'], toolsUsed: ['read_file', 'grep'],
|
||||
contextTokensUsed: 40000, contextWindowTokens: 512000,
|
||||
}))
|
||||
const updates = []
|
||||
let running = 0
|
||||
for (let t = 0; t < 5; t++) {
|
||||
// Streamed chunk carrying the running context counter. This is all the
|
||||
// published CLI can see, and what it estimates from; main ignores it in
|
||||
// favour of the turn_completed record below. Both are present in a real
|
||||
// session, so the corpus carries both and the two versions have something
|
||||
// to disagree about.
|
||||
running += between(3000, 12000)
|
||||
updates.push(JSON.stringify({
|
||||
timestamp: iso(at(day, 11, t * 5)), method: 'session/update',
|
||||
params: { sessionId: id, update: { sessionUpdate: 'agent_message_chunk', content: { type: 'text', text: `chunk ${t}` } }, _meta: { totalTokens: running, promptId: `p${t}`, updateType: 'AgentMessageChunk', modelId: 'grok-build' } },
|
||||
}))
|
||||
const usage = {
|
||||
inputTokens: between(1000, 9000), outputTokens: between(80, 700), totalTokens: 0,
|
||||
cachedReadTokens: between(0, 40000), cacheCreationTokens: between(0, 2000), reasoningTokens: between(0, 300),
|
||||
modelCalls: 1, apiDurationMs: 1000, costUsdTicks: 125117780000, numTurns: 1,
|
||||
}
|
||||
usage.totalTokens = usage.inputTokens + usage.outputTokens
|
||||
usage.modelUsage = { 'grok-4.6-build': { ...usage } }
|
||||
updates.push(JSON.stringify({
|
||||
timestamp: Math.floor(at(day, 11, t * 5).getTime() / 1000), method: '_x.ai/session/update',
|
||||
params: { sessionId: id, update: { sessionUpdate: 'turn_completed', prompt_id: `p${t}`, usage }, _meta: { eventId: `event-${t}`, agentTimestampMs: at(day, 11, t * 5).getTime() } },
|
||||
}))
|
||||
}
|
||||
writeLines(join(dir, 'updates.jsonl'), updates)
|
||||
files += 3
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
// ── cursor (WAL-mode SQLite) ─────────────────────────────────────────────────
|
||||
// Left with an un-checkpointed -wal sidecar and NO -shm: that is what a live
|
||||
// Cursor database looks like on disk, and it is the shape that made
|
||||
// read-only opens fail before #1017.
|
||||
|
||||
function genCursor() {
|
||||
let DatabaseSync
|
||||
try { ({ DatabaseSync } = require_('node:sqlite')) } catch { return 0 }
|
||||
|
||||
const userDir = process.platform === 'darwin'
|
||||
? join(HOME, 'Library', 'Application Support', 'Cursor', 'User')
|
||||
: process.platform === 'win32'
|
||||
? join(HOME, 'AppData', 'Roaming', 'Cursor', 'User')
|
||||
: join(HOME, '.config', 'Cursor', 'User')
|
||||
|
||||
const globalDir = join(userDir, 'globalStorage')
|
||||
mkdirSync(globalDir, { recursive: true })
|
||||
const dbPath = join(globalDir, 'state.vscdb')
|
||||
for (const suffix of ['', '-wal', '-shm']) rmSync(dbPath + suffix, { force: true })
|
||||
const db = new DatabaseSync(dbPath)
|
||||
db.exec('PRAGMA journal_mode=WAL')
|
||||
db.exec('CREATE TABLE cursorDiskKV (key TEXT PRIMARY KEY, value BLOB)')
|
||||
db.exec('CREATE TABLE ItemTable (key TEXT UNIQUE, value BLOB)')
|
||||
const ins = db.prepare('INSERT INTO cursorDiskKV (key, value) VALUES (?, ?)')
|
||||
|
||||
const composers = []
|
||||
for (let c = 0; c < 6; c++) {
|
||||
const composerId = `composer-${c}`
|
||||
composers.push({ composerId, name: `session-${c}`, unifiedMode: 'agent' })
|
||||
ins.run(`composerData:${composerId}`, JSON.stringify({
|
||||
promptTokenBreakdown: { totalUsedTokens: between(10000, 90000) },
|
||||
createdAt: at(85, 9 + c).getTime(),
|
||||
}))
|
||||
for (let b = 0; b < 8; b++) {
|
||||
const createdAt = iso(at(85, 9 + c, b * 4))
|
||||
ins.run(`bubbleId:${composerId}:u${b}`, JSON.stringify({ type: 1, conversationId: composerId, createdAt, text: `ask ${b}`, codeBlocks: '[]' }))
|
||||
ins.run(`bubbleId:${composerId}:a${b}`, JSON.stringify({
|
||||
type: 2, conversationId: composerId, createdAt, text: `reply ${b}`, codeBlocks: '[]',
|
||||
tokenCount: { inputTokens: between(300, 4000), outputTokens: between(40, 500) },
|
||||
modelInfo: { modelName: 'claude-4.6-sonnet' },
|
||||
requestId: `req-${c}-${b}`,
|
||||
}))
|
||||
}
|
||||
}
|
||||
// Checkpoint what is written so far, then stop auto-checkpointing and append
|
||||
// more: the tail rows live only in the -wal the copy below carries.
|
||||
db.exec('PRAGMA wal_checkpoint(TRUNCATE)')
|
||||
db.exec('PRAGMA wal_autocheckpoint=0')
|
||||
ins.run('composerData:composer-tail', JSON.stringify({ promptTokenBreakdown: { totalUsedTokens: 12345 }, createdAt: at(86, 9).getTime() }))
|
||||
ins.run('bubbleId:composer-tail:a0', JSON.stringify({
|
||||
type: 2, conversationId: 'composer-tail', createdAt: iso(at(86, 9)), text: 'tail reply', codeBlocks: '[]',
|
||||
tokenCount: { inputTokens: 2222, outputTokens: 333 }, modelInfo: { modelName: 'claude-4.6-sonnet' },
|
||||
}))
|
||||
composers.push({ composerId: 'composer-tail', name: 'session-tail', unifiedMode: 'agent' })
|
||||
db.close()
|
||||
|
||||
// Per-workspace DB naming the composers, plus the workspace.json that gives
|
||||
// the project its name.
|
||||
const wsDir = join(userDir, 'workspaceStorage', 'ws0000000000000000000000000000000')
|
||||
mkdirSync(wsDir, { recursive: true })
|
||||
for (const suffix of ['', '-wal', '-shm']) rmSync(join(wsDir, 'state.vscdb' + suffix), { force: true })
|
||||
const wsDb = new DatabaseSync(join(wsDir, 'state.vscdb'))
|
||||
wsDb.exec('CREATE TABLE ItemTable (key TEXT UNIQUE, value BLOB)')
|
||||
wsDb.prepare('INSERT INTO ItemTable (key, value) VALUES (?, ?)').run('composer.composerData', JSON.stringify({ allComposers: composers }))
|
||||
wsDb.close()
|
||||
write(join(wsDir, 'workspace.json'), JSON.stringify({ folder: 'file:///work/billing' }))
|
||||
|
||||
return existsSync(dbPath + '-wal') ? 3 : 2
|
||||
}
|
||||
|
||||
// ── run ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
const counts = {
|
||||
claude: genClaude(),
|
||||
codex: genCodex(),
|
||||
gemini: genGemini(),
|
||||
kiro: genKiro(),
|
||||
dsh: genDsh(),
|
||||
grok: genGrok(),
|
||||
cursor: genCursor(),
|
||||
}
|
||||
console.log(JSON.stringify(counts))
|
||||
422
scripts/upgrade-path/run.mjs
Normal file
422
scripts/upgrade-path/run.mjs
Normal file
|
|
@ -0,0 +1,422 @@
|
|||
// Upgrade-path verification: prove that a cache written by the last PUBLISHED
|
||||
// CLI survives this build's first run, on this platform, with this Node.
|
||||
//
|
||||
// npm run verify:upgrade
|
||||
//
|
||||
// What it does, in order:
|
||||
// 1. generates a deterministic multi-provider corpus into an isolated HOME
|
||||
// (whose path contains a space, because a real Windows HOME usually does)
|
||||
// 2. installs codeburn@0.9.20 into an isolated global prefix and runs it,
|
||||
// producing a genuine session-cache.v7 + daily-cache.v17
|
||||
// 3. installs THIS build the same way and runs it against the SAME cache dir,
|
||||
// through the npm bin shim rather than `node dist/cli.js`, so
|
||||
// dist/parse-worker.js has to resolve from a symlinked entry point
|
||||
// 4. asserts the migration landed and compares payloads per provider
|
||||
// 5. serve --stdio smoke, worker determinism, warm-run stability
|
||||
//
|
||||
// Env: UPGRADE_PATH_WORK (work dir), UPGRADE_PATH_OLD (published version to
|
||||
// upgrade from), UPGRADE_PATH_KEEP=1 to leave the work dir behind.
|
||||
|
||||
import { spawnSync, spawn } from 'node:child_process'
|
||||
import { mkdirSync, rmSync, existsSync, readdirSync, statSync, readFileSync, writeFileSync, copyFileSync } from 'node:fs'
|
||||
import { join, dirname } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { tmpdir } from 'node:os'
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url))
|
||||
const REPO = join(HERE, '..', '..')
|
||||
const OLD_VERSION = process.env['UPGRADE_PATH_OLD'] || '0.9.20'
|
||||
const WORK = process.env['UPGRADE_PATH_WORK'] || join(tmpdir(), 'codeburn upgrade path')
|
||||
|
||||
// The published binary's cache versions. If a future baseline writes something
|
||||
// else these two are the knobs to move, and the assertions below will say so.
|
||||
const OLD_SESSION_CACHE = 'session-cache.v7.json'
|
||||
const OLD_DAILY_CACHE = 'daily-cache.v17.json'
|
||||
const NEW_SESSION_CACHE_DIR = 'session-cache.v9'
|
||||
const NEW_DAILY_CACHE = 'daily-cache.v19.json'
|
||||
|
||||
const HOME = join(WORK, 'user home')
|
||||
const PAYLOADS = join(WORK, 'payloads')
|
||||
const CACHES = join(WORK, 'caches')
|
||||
const npmCmd = process.platform === 'win32' ? 'npm.cmd' : 'npm'
|
||||
|
||||
let failures = 0
|
||||
let skipped = 0
|
||||
const step = msg => console.log(`\n=== ${msg}`)
|
||||
const ok = msg => console.log(` ok ${msg}`)
|
||||
const fail = msg => { failures++; console.log(` FAIL ${msg}`) }
|
||||
const skip = msg => { skipped++; console.log(` skip ${msg}`) }
|
||||
const check = (cond, msg) => (cond ? ok(msg) : fail(msg))
|
||||
|
||||
function run(cmd, args, opts = {}) {
|
||||
const r = spawnSync(cmd, args, { encoding: 'utf8', maxBuffer: 1 << 29, shell: false, ...opts })
|
||||
if (r.error) throw new Error(`${cmd} ${args.join(' ')}: ${r.error.message}`)
|
||||
return r
|
||||
}
|
||||
|
||||
// Node refuses to spawn a .cmd/.bat without a shell, and a shell spawn quotes
|
||||
// nothing for you — so on Windows anything holding a space (every path here
|
||||
// does, by design) has to be quoted by hand.
|
||||
const quoteForShell = s => (process.platform === 'win32' && /[\s&|^]/.test(s) ? `"${s}"` : s)
|
||||
function runShell(cmd, args, opts = {}) {
|
||||
if (process.platform !== 'win32') return run(cmd, args, opts)
|
||||
return run(quoteForShell(cmd), args.map(quoteForShell), { ...opts, shell: true })
|
||||
}
|
||||
|
||||
function cliEnv(cacheDir, extra = {}) {
|
||||
// Deliberately minimal: no provider override vars, so every provider resolves
|
||||
// its own default path under the isolated HOME. APPDATA/LOCALAPPDATA are
|
||||
// pinned under it too — several providers read them on Windows, and inheriting
|
||||
// the runner's would let real (or leftover) data into the comparison.
|
||||
const passthrough = {}
|
||||
for (const k of ['PATH', 'PATHEXT', 'SystemRoot', 'ComSpec', 'windir', 'TEMP', 'TMP', 'NUMBER_OF_PROCESSORS']) {
|
||||
if (process.env[k] !== undefined) passthrough[k] = process.env[k]
|
||||
}
|
||||
return {
|
||||
...passthrough,
|
||||
HOME, USERPROFILE: HOME, TZ: 'UTC', CODEBURN_CACHE_DIR: cacheDir,
|
||||
APPDATA: join(HOME, 'AppData', 'Roaming'), LOCALAPPDATA: join(HOME, 'AppData', 'Local'),
|
||||
...extra,
|
||||
}
|
||||
}
|
||||
|
||||
function cli(bin, args, cacheDir, extra = {}) {
|
||||
const r = run(bin.cmd, [...bin.args, ...args], { env: cliEnv(cacheDir, extra), cwd: WORK })
|
||||
if (r.status !== 0) throw new Error(`${args.join(' ')} exited ${r.status}\n${r.stderr?.slice(0, 4000)}`)
|
||||
return r.stdout
|
||||
}
|
||||
|
||||
// Install into an isolated global prefix, so the CLI runs from a location that
|
||||
// has nothing to do with this checkout — which is what makes dist/parse-worker.js
|
||||
// resolution worth testing. On POSIX npm's bin is a symlink INTO the package and
|
||||
// we drive that directly. On Windows it is a .cmd shim, which Node will not
|
||||
// spawn without a shell; the payload captures go through the installed
|
||||
// dist/cli.js there and the shim itself is smoke-tested once, separately.
|
||||
function installGlobal(prefix, spec) {
|
||||
const r = runShell(npmCmd, ['install', '-g', '--prefix', prefix, spec, '--no-audit', '--no-fund', '--loglevel', 'error'], { cwd: WORK })
|
||||
if (r.status !== 0) throw new Error(`npm install -g ${spec} exited ${r.status}\n${r.stdout}\n${r.stderr}`)
|
||||
const symlink = join(prefix, 'bin', 'codeburn')
|
||||
if (existsSync(symlink)) return { cmd: process.execPath, args: [symlink], shim: null }
|
||||
const winCmd = join(prefix, 'codeburn.cmd')
|
||||
const entry = join(prefix, 'node_modules', 'codeburn', 'dist', 'cli.js')
|
||||
if (existsSync(entry)) return { cmd: process.execPath, args: [entry], shim: existsSync(winCmd) ? winCmd : null }
|
||||
throw new Error(`no codeburn bin under ${prefix}`)
|
||||
}
|
||||
|
||||
// The npm shim, exercised once. Needs a shell on Windows, so nothing with a
|
||||
// space in it is passed through here.
|
||||
function checkShim(bin, cacheDir) {
|
||||
if (!bin.shim) { ok("CLI invoked through npm's bin symlink"); return }
|
||||
const r = runShell(bin.shim, ['--version'], { env: cliEnv(cacheDir), cwd: WORK })
|
||||
check(r.status === 0 && r.stdout.trim().length > 0, `npm .cmd shim runs: ${r.stdout.trim() || r.stderr?.slice(0, 200)}`)
|
||||
}
|
||||
|
||||
// Captured payloads. `export` is the token/call source, `menubar-json` the
|
||||
// unrounded per-provider cost; both are stable given a fixed corpus. --period all
|
||||
// so the whole three-month corpus is in scope on both sides.
|
||||
function capture(bin, cacheDir, outDir, extra = {}) {
|
||||
mkdirSync(outDir, { recursive: true })
|
||||
cli(bin, ['export', '--format', 'json', '--from', '2000-01-01', '--to', '2999-12-31', '-o', join(outDir, 'export.json')], cacheDir, extra)
|
||||
const menubar = cli(bin, ['status', '--format', 'menubar-json', '--period', 'all', '--no-optimize', '--no-timeline'], cacheDir, extra)
|
||||
writeFileSync(join(outDir, 'menubar.json'), menubar)
|
||||
const status = cli(bin, ['status', '--format', 'json', '--period', 'all'], cacheDir, extra)
|
||||
writeFileSync(join(outDir, 'status.json'), status)
|
||||
return { menubar: JSON.parse(menubar), status: JSON.parse(status) }
|
||||
}
|
||||
|
||||
// The one field that moves between two runs of the same payload.
|
||||
const stripGenerated = obj => JSON.parse(JSON.stringify(obj, (k, v) => (k.startsWith('generated') ? undefined : v)))
|
||||
|
||||
// Shards carry real stat data and are published under a random filename, so
|
||||
// "identical" means identical after normalizing both away.
|
||||
function shardSnapshot(cacheDir) {
|
||||
const dir = join(cacheDir, NEW_SESSION_CACHE_DIR)
|
||||
if (!existsSync(dir)) return null
|
||||
const out = {}
|
||||
for (const name of readdirSync(dir).sort()) {
|
||||
if (name === 'envelope.json') continue
|
||||
const body = JSON.parse(readFileSync(join(dir, name), 'utf8'))
|
||||
for (const entry of Object.values(body)) {
|
||||
if (entry && typeof entry === 'object' && entry.fingerprint) {
|
||||
delete entry.fingerprint.dev
|
||||
delete entry.fingerprint.ino
|
||||
delete entry.fingerprint.mtimeMs
|
||||
}
|
||||
}
|
||||
// Key the bucket off the shard's provider.month prefix, dropping the nonce.
|
||||
out[name.replace(/\.[0-9a-f]{16}\.json$/, '')] = sortDeep(body)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
function sortDeep(v) {
|
||||
if (Array.isArray(v)) return v.map(sortDeep)
|
||||
if (v && typeof v === 'object') return Object.fromEntries(Object.keys(v).sort().map(k => [k, sortDeep(v[k])]))
|
||||
return v
|
||||
}
|
||||
|
||||
function shardMtimes(cacheDir) {
|
||||
const dir = join(cacheDir, NEW_SESSION_CACHE_DIR)
|
||||
return Object.fromEntries(readdirSync(dir).sort().map(n => [n, statSync(join(dir, n)).mtimeMs]))
|
||||
}
|
||||
|
||||
// ── 1. corpus ────────────────────────────────────────────────────────────────
|
||||
|
||||
step(`work dir: ${WORK}`)
|
||||
try { rmSync(WORK, { recursive: true, force: true }) } catch (err) { console.log(` note could not clear the work dir (${err.code}); reusing it`) }
|
||||
mkdirSync(HOME, { recursive: true })
|
||||
mkdirSync(PAYLOADS, { recursive: true })
|
||||
|
||||
const gen = run(process.execPath, [join(HERE, 'gen-corpus.mjs'), HOME])
|
||||
if (gen.status !== 0) { console.log(gen.stderr); process.exit(1) }
|
||||
ok(`corpus generated: ${gen.stdout.trim()}`)
|
||||
|
||||
const upgradeCache = join(CACHES, 'upgrade')
|
||||
mkdirSync(upgradeCache, { recursive: true })
|
||||
|
||||
// ── 2. baseline: the last published CLI ──────────────────────────────────────
|
||||
|
||||
step(`baseline: codeburn@${OLD_VERSION}`)
|
||||
const oldBin = installGlobal(join(WORK, 'old'), `codeburn@${OLD_VERSION}`)
|
||||
const oldVersion = cli(oldBin, ['--version'], upgradeCache).trim()
|
||||
check(oldVersion === OLD_VERSION, `installed baseline reports ${oldVersion}`)
|
||||
|
||||
const baseline = capture(oldBin, upgradeCache, join(PAYLOADS, 'baseline'))
|
||||
check(baseline.menubar.current.calls > 0, `baseline counted ${baseline.menubar.current.calls} calls across ${baseline.menubar.current.sessions} sessions`)
|
||||
check(existsSync(join(upgradeCache, OLD_SESSION_CACHE)), `${OLD_VERSION} wrote ${OLD_SESSION_CACHE}`)
|
||||
check(existsSync(join(upgradeCache, OLD_DAILY_CACHE)), `${OLD_VERSION} wrote ${OLD_DAILY_CACHE}`)
|
||||
|
||||
// ── 3. upgrade: this build, same cache dir, through the npm bin shim ─────────
|
||||
|
||||
step('upgrade: this build against the same cache dir')
|
||||
const pack = runShell(npmCmd, ['pack', '--ignore-scripts', '--pack-destination', WORK, '--loglevel', 'error'], { cwd: REPO })
|
||||
if (pack.status !== 0) { console.log(pack.stdout, pack.stderr); process.exit(1) }
|
||||
const tarball = join(WORK, pack.stdout.trim().split('\n').pop().trim())
|
||||
const newBin = installGlobal(join(WORK, 'new'), tarball)
|
||||
ok(`this build installed as ${newBin.args[0] ?? newBin.cmd}`)
|
||||
checkShim(newBin, upgradeCache)
|
||||
|
||||
const upgraded = capture(newBin, upgradeCache, join(PAYLOADS, 'upgraded'))
|
||||
|
||||
check(!existsSync(join(upgradeCache, OLD_SESSION_CACHE)), `${OLD_SESSION_CACHE} removed after the re-layout`)
|
||||
check(existsSync(join(upgradeCache, NEW_SESSION_CACHE_DIR)), `${NEW_SESSION_CACHE_DIR}/ present`)
|
||||
check(existsSync(join(upgradeCache, NEW_SESSION_CACHE_DIR, 'envelope.json')), `${NEW_SESSION_CACHE_DIR}/envelope.json present`)
|
||||
const envelope = JSON.parse(readFileSync(join(upgradeCache, NEW_SESSION_CACHE_DIR, 'envelope.json'), 'utf8'))
|
||||
check(envelope.version === 9 && Object.keys(envelope.providers ?? {}).length > 0,
|
||||
`envelope at version ${envelope.version} with ${Object.keys(envelope.providers ?? {}).length} providers`)
|
||||
check(readdirSync(join(upgradeCache, NEW_SESSION_CACHE_DIR)).some(n => n !== 'envelope.json'), 'shards published alongside the envelope')
|
||||
check(existsSync(join(upgradeCache, NEW_DAILY_CACHE)), `${NEW_DAILY_CACHE} re-derived`)
|
||||
check(existsSync(join(upgradeCache, OLD_DAILY_CACHE)), `${OLD_DAILY_CACHE} kept as the carry-forward baseline`)
|
||||
const oldDays = JSON.parse(readFileSync(join(upgradeCache, OLD_DAILY_CACHE), 'utf8')).days.length
|
||||
const newDays = JSON.parse(readFileSync(join(upgradeCache, NEW_DAILY_CACHE), 'utf8')).days.length
|
||||
check(newDays >= oldDays, `daily history did not shrink: ${oldDays} -> ${newDays} days`)
|
||||
|
||||
// ── 4. payload parity ────────────────────────────────────────────────────────
|
||||
|
||||
step('payload parity vs the baseline')
|
||||
const cmp = run(process.execPath, [join(HERE, 'compare.mjs'), join(PAYLOADS, 'baseline'), join(PAYLOADS, 'upgraded')], { stdio: 'inherit' })
|
||||
if (cmp.status !== 0) failures++
|
||||
|
||||
// ── 5. serve smoke ───────────────────────────────────────────────────────────
|
||||
|
||||
step('serve --stdio')
|
||||
const serveFrames = await serveSmoke()
|
||||
if (serveFrames) {
|
||||
for (const [name, args] of [['menubar-json', ['status', '--format', 'menubar-json', '--period', 'all', '--no-optimize', '--no-timeline']], ['models', ['models', '--format', 'json']]]) {
|
||||
const frame = serveFrames.get(name)
|
||||
if (!frame?.ok) { fail(`serve returned no ok frame for ${name}: ${JSON.stringify(frame)}`); continue }
|
||||
ok(`serve ok frame for ${name}`)
|
||||
const oneShot = cli(newBin, args, upgradeCache)
|
||||
const a = JSON.stringify(stripGenerated(JSON.parse(frame.output)))
|
||||
const b = JSON.stringify(stripGenerated(JSON.parse(oneShot)))
|
||||
check(a === b, `serve ${name} matches the one-shot payload (ignoring generated*)`)
|
||||
}
|
||||
}
|
||||
|
||||
async function serveSmoke() {
|
||||
const child = spawn(newBin.cmd, [...newBin.args, 'serve', '--stdio'], { env: cliEnv(upgradeCache), cwd: WORK, stdio: ['pipe', 'pipe', 'pipe'] })
|
||||
const frames = new Map()
|
||||
let buf = ''
|
||||
let ready = false
|
||||
const done = new Promise(resolve => {
|
||||
child.stdout.on('data', d => {
|
||||
buf += d
|
||||
let nl
|
||||
while ((nl = buf.indexOf('\n')) >= 0) {
|
||||
const line = buf.slice(0, nl); buf = buf.slice(nl + 1)
|
||||
if (!line.trim()) continue
|
||||
let msg
|
||||
try { msg = JSON.parse(line) } catch { continue }
|
||||
if (msg.ready) { ready = true; continue }
|
||||
if (msg.progress !== undefined) continue
|
||||
if (msg.id === 1) frames.set('menubar-json', msg)
|
||||
if (msg.id === 2) frames.set('models', msg)
|
||||
if (frames.size === 2) resolve()
|
||||
}
|
||||
})
|
||||
})
|
||||
child.stdin.write(JSON.stringify({ id: 1, args: ['status', '--format', 'menubar-json', '--period', 'all', '--no-optimize', '--no-timeline'] }) + '\n')
|
||||
child.stdin.write(JSON.stringify({ id: 2, args: ['models', '--format', 'json'] }) + '\n')
|
||||
const timeout = new Promise(r => setTimeout(() => r('timeout'), 240_000))
|
||||
if ((await Promise.race([done, timeout])) === 'timeout') { child.kill(); fail('serve did not answer both requests within 240s'); return null }
|
||||
check(ready, 'serve announced itself with a ready frame')
|
||||
|
||||
// Closing stdin is the documented shutdown: the child must exit on its own.
|
||||
const exited = new Promise(r => child.once('exit', code => r(code)))
|
||||
child.stdin.end()
|
||||
const exitCode = await Promise.race([exited, new Promise(r => setTimeout(() => r('hung'), 30_000))])
|
||||
if (exitCode === 'hung') { child.kill(); fail('serve did not exit when stdin closed') }
|
||||
else ok(`serve exited on stdin close (code ${exitCode})`)
|
||||
return frames
|
||||
}
|
||||
|
||||
// ── 6. worker determinism ────────────────────────────────────────────────────
|
||||
|
||||
step('parse-worker determinism (CODEBURN_PARSE_WORKERS 0 vs 3)')
|
||||
const serialCache = join(CACHES, 'workers-0')
|
||||
const parallelCache = join(CACHES, 'workers-3')
|
||||
for (const dir of [serialCache, parallelCache]) {
|
||||
mkdirSync(dir, { recursive: true })
|
||||
// Seed the shared price table so the two runs cannot be priced differently by
|
||||
// a cache expiring between them. Parsing is unaffected either way.
|
||||
const priced = join(upgradeCache, 'litellm-pricing.json')
|
||||
if (existsSync(priced)) copyFileSync(priced, join(dir, 'litellm-pricing.json'))
|
||||
}
|
||||
const serialOut = capture(newBin, serialCache, join(PAYLOADS, 'workers-0'), { CODEBURN_PARSE_WORKERS: '0', CODEBURN_VERBOSE: '1' })
|
||||
const parallelOut = capture(newBin, parallelCache, join(PAYLOADS, 'workers-3'), { CODEBURN_PARSE_WORKERS: '3', CODEBURN_VERBOSE: '1' })
|
||||
check(JSON.stringify(stripGenerated(serialOut.menubar)) === JSON.stringify(stripGenerated(parallelOut.menubar)),
|
||||
'menubar-json payload identical with and without workers')
|
||||
const readExport = dir => stripGenerated(JSON.parse(readFileSync(join(PAYLOADS, dir, 'export.json'), 'utf8')))
|
||||
check(JSON.stringify(readExport('workers-0')) === JSON.stringify(readExport('workers-3')),
|
||||
'export payload identical with and without workers')
|
||||
check(JSON.stringify(shardSnapshot(serialCache)) === JSON.stringify(shardSnapshot(parallelCache)),
|
||||
'shard bodies identical with and without workers (fingerprint stat data and shard nonces normalized)')
|
||||
|
||||
// A forced pool that never actually spawned would make the check above vacuous.
|
||||
const verbose = run(newBin.cmd, [...newBin.args, 'status', '--format', 'json', '--period', 'all'], {
|
||||
env: cliEnv(join(CACHES, 'workers-probe'), { CODEBURN_PARSE_WORKERS: '3', CODEBURN_VERBOSE: '1' }), cwd: WORK,
|
||||
})
|
||||
const decision = (verbose.stderr || '').split('\n').filter(l => l.includes('parse workers='))
|
||||
if (decision.length === 0) skip('no "parse workers=" line on stderr; cannot confirm the pool was forced')
|
||||
else check(decision.some(l => /parse workers=[1-9]/.test(l)), `worker pool engaged: ${decision.map(l => l.trim()).join(' | ')}`)
|
||||
|
||||
// ── 7. second run is warm ────────────────────────────────────────────────────
|
||||
|
||||
step('second run is warm')
|
||||
const beforeBodies = shardSnapshot(upgradeCache)
|
||||
const beforeMtimes = shardMtimes(upgradeCache)
|
||||
const warm = capture(newBin, upgradeCache, join(PAYLOADS, 'warm'))
|
||||
|
||||
// The direct no-re-parse signal: the worker gate prints how many whole-file
|
||||
// re-parses are pending. On an unchanged corpus that must be zero for the two
|
||||
// providers big enough to be gated.
|
||||
const warmVerbose = run(newBin.cmd, [...newBin.args, 'status', '--format', 'menubar-json', '--period', 'all', '--no-optimize', '--no-timeline'], {
|
||||
env: cliEnv(upgradeCache, { CODEBURN_VERBOSE: '1' }), cwd: WORK,
|
||||
})
|
||||
const pending = (warmVerbose.stderr || '').split('\n').filter(l => l.includes('parse workers='))
|
||||
if (pending.length === 0) skip('no "parse workers=" line on a warm run; cannot confirm nothing re-parsed')
|
||||
else check(pending.every(l => /0 pending files|no full parses pending/.test(l)),
|
||||
`nothing re-parsed on the warm run: ${pending.map(l => l.replace(/^codeburn: /, '').trim()).join(' | ')}`)
|
||||
|
||||
check(JSON.stringify(shardSnapshot(upgradeCache)) === JSON.stringify(beforeBodies), 'warm run left every shard body unchanged')
|
||||
check(JSON.stringify(stripGenerated(warm.menubar)) === JSON.stringify(stripGenerated(upgraded.menubar)),
|
||||
'warm run reports the same payload as the run that migrated the cache')
|
||||
|
||||
// Republication without a content change is wasted I/O, not a correctness
|
||||
// problem, so it is reported rather than failed. It is real, and it has a
|
||||
// follow-up issue: a date-RANGED query (`status --format json`, the
|
||||
// statusline/menubar fast path) republishes the month shards its range skipped,
|
||||
// on every run, even when nothing changed — partially defeating #1007's
|
||||
// "a warm launch rewrites only the month that changed". The identical-bodies
|
||||
// check above is what proves the content survives it.
|
||||
const afterMtimes = shardMtimes(upgradeCache)
|
||||
const republished = Object.keys(afterMtimes).filter(n => n !== 'envelope.json' && beforeMtimes[n] !== afterMtimes[n])
|
||||
const retired = Object.keys(beforeMtimes).filter(n => n !== 'envelope.json' && !(n in afterMtimes))
|
||||
const churnNote = 'known defect, see #1032: a date-ranged run republishes the month shards it skipped'
|
||||
if (retired.length) console.log(` note ${retired.length} shard(s) republished under a new name with identical content (${churnNote}): ${retired.join(', ')}`)
|
||||
else if (republished.length) console.log(` note ${republished.length} shard(s) rewritten in place (${churnNote}): ${republished.join(', ')}`)
|
||||
else ok('no shard republished on an unchanged corpus')
|
||||
|
||||
// ── 8. partial source aging (release blocker, track C) ───────────────────────
|
||||
// Claude Code deletes its transcripts after ~30 days, so between one run and the
|
||||
// next a day can go from fully sourced to PARTIALLY sourced. The daily cache's
|
||||
// never-lose contract says a schema bump re-derives what it can and carries
|
||||
// forward what it cannot — but on a partially-sourced day the re-derivation
|
||||
// produces a smaller slice from the surviving files and that slice REPLACES the
|
||||
// baseline one instead of being unioned with it, so the aged-out portion is lost.
|
||||
// A day that aged out completely is carried forward correctly, which is what
|
||||
// makes the partial case a hole rather than a missing feature.
|
||||
//
|
||||
// Runs last, and on its own cache dir, so mutating the corpus cannot disturb the
|
||||
// parity comparison above.
|
||||
|
||||
step('never-lose across partial source aging')
|
||||
const agingCache = join(CACHES, 'aging')
|
||||
mkdirSync(agingCache, { recursive: true })
|
||||
|
||||
// A fresh 0.9.20 cache, taken while every transcript still exists.
|
||||
capture(oldBin, agingCache, join(PAYLOADS, 'aging-baseline'))
|
||||
const baseDaily = JSON.parse(readFileSync(join(agingCache, OLD_DAILY_CACHE), 'utf8'))
|
||||
const sliceOf = (cache, date) => cache.days.find(d => d.date === date)?.providers?.claude
|
||||
|
||||
// Group transcripts by the day their turns land on. Sidechain files are left out:
|
||||
// deleting a parent's subagent transcript entangles this with spawn-link
|
||||
// carry-forward, which is a different contract.
|
||||
const projectsDir = join(HOME, '.claude', 'projects')
|
||||
const byDay = new Map()
|
||||
for (const rel of readdirSync(projectsDir, { recursive: true })) {
|
||||
const relPath = String(rel)
|
||||
if (!relPath.endsWith('.jsonl') || relPath.includes('subagents')) continue
|
||||
const full = join(projectsDir, relPath)
|
||||
const first = readFileSync(full, 'utf8').split('\n', 1)[0]
|
||||
const date = JSON.parse(first).timestamp?.slice(0, 10)
|
||||
if (!date || !sliceOf(baseDaily, date)) continue
|
||||
if (!byDay.has(date)) byDay.set(date, [])
|
||||
byDay.get(date).push(full)
|
||||
}
|
||||
|
||||
// Densest days first, oldest among equals — the ones a retention window reaches
|
||||
// first, and the ones where losing the aged-out portion shows up largest.
|
||||
const candidates = [...byDay.entries()].filter(([, files]) => files.length >= 2)
|
||||
.sort((a, b) => (b[1].length - a[1].length) || a[0].localeCompare(b[0]))
|
||||
|
||||
let aged = []
|
||||
if (candidates.length < 3) fail(`need 3 multi-transcript claude days to age out, found ${candidates.length}`)
|
||||
else {
|
||||
for (const [date, files] of candidates.slice(0, 2)) {
|
||||
// Keep exactly one file, so the day is still sourced — just not fully.
|
||||
for (const f of files.slice(1)) rmSync(f)
|
||||
aged.push({ date, kind: 'partially sourceless', kept: 1, removed: files.length - 1 })
|
||||
}
|
||||
const [goneDate, goneFiles] = candidates[2]
|
||||
for (const f of goneFiles) rmSync(f)
|
||||
aged.push({ date: goneDate, kind: 'fully sourceless', kept: 0, removed: goneFiles.length })
|
||||
for (const a of aged) ok(`${a.date}: ${a.kind} (removed ${a.removed} of ${a.removed + a.kept} transcripts)`)
|
||||
|
||||
capture(newBin, agingCache, join(PAYLOADS, 'aging-upgraded'))
|
||||
const upDaily = JSON.parse(readFileSync(join(agingCache, NEW_DAILY_CACHE), 'utf8'))
|
||||
const usd = n => `$${n.toFixed(6)}`
|
||||
|
||||
for (const a of aged) {
|
||||
const b = sliceOf(baseDaily, a.date)
|
||||
const u = sliceOf(upDaily, a.date)
|
||||
if (!u) { fail(`${a.date} (${a.kind}): the claude slice is gone entirely; baseline had ${usd(b.cost)} over ${b.calls} calls`); continue }
|
||||
// A fully sourceless day has nothing to re-derive, so it must come back
|
||||
// EXACTLY. A partially sourceless one may legitimately grow (a re-parse
|
||||
// under new accounting), but must never shrink.
|
||||
const exact = a.kind === 'fully sourceless'
|
||||
const costOk = exact ? Math.abs(u.cost - b.cost) < 1e-9 : u.cost >= b.cost - 1e-9
|
||||
const callsOk = exact ? u.calls === b.calls : u.calls >= b.calls
|
||||
const loss = costOk && callsOk ? '' :
|
||||
` — LOST ${usd(b.cost - u.cost)} (${(100 * (b.cost - u.cost) / b.cost).toFixed(1)}%) and ${b.calls - u.calls} calls`
|
||||
check(costOk && callsOk,
|
||||
`${a.date} (${a.kind}): cost ${usd(b.cost)} -> ${usd(u.cost)}, calls ${b.calls} -> ${u.calls}${loss}`)
|
||||
}
|
||||
}
|
||||
|
||||
// ── done ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
console.log(`\n${failures ? `FAILED: ${failures} check(s)` : 'PASSED'}${skipped ? ` (${skipped} skipped)` : ''}`)
|
||||
if (failures && process.env['UPGRADE_PATH_KEEP'] !== '0') console.log(`payloads left in: ${PAYLOADS}`)
|
||||
else if (process.env['UPGRADE_PATH_KEEP'] !== '1') { try { rmSync(WORK, { recursive: true, force: true }) } catch { /* windows file locks */ } }
|
||||
process.exit(failures ? 1 : 0)
|
||||
Loading…
Add table
Add a link
Reference in a new issue