mirror of
https://github.com/AgentSeal/codeburn.git
synced 2026-08-21 14:34:32 +00:00
ci: verify the 0.9.20 upgrade path on every platform
Everyone upgrading from the last published CLI crosses the session-cache v7 -> v9 re-layout (#1005/#1007) and the daily-cache v17 -> v19 re-derivation (#1015) on their first run. That path was covered by unit tests on one platform against caches the tests wrote themselves. `npm run verify:upgrade` (and the matching matrix job over {ubuntu, windows, macos} x node {22.13.0, 22}) instead installs the real codeburn@0.9.20 into an isolated global prefix, points it at a generated seven-provider corpus in a HOME whose path contains a space, and lets it write a genuine session-cache.v7.json + daily-cache.v17.json. This build then runs against that same cache dir, installed the same way, so dist/parse-worker.js has to resolve from an entry point outside the checkout. It asserts the v7 file is retired, the v9 envelope and shards publish, the daily history does not shrink, and per-provider calls/tokens/cost match the baseline exactly for claude, codex, gemini, kiro and cursor. grok is reported rather than asserted (its accounting changed in #1015) and dsh is required to be new. It also smoke-tests serve --stdio against the one-shot payloads, pins CODEBURN_PARSE_WORKERS to 0 and 3 and requires identical shards and payloads either way, and checks that a second run re-parses nothing.
This commit is contained in:
iamtoruk
parent
564c618829
commit
74d718fe03
5 changed files with 904 additions and 0 deletions
53
.github/workflows/upgrade-path.yml
vendored
Normal file
53
.github/workflows/upgrade-path.yml
vendored
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
name: Upgrade path
|
||||
|
||||
# Every existing user upgrading from the last published CLI (0.9.20) crosses the
|
||||
# session-cache v7 -> v9 re-layout and the daily-cache v17 -> v19 re-derivation on
|
||||
# their first run. Unit tests cover the migration in isolation on one platform; this
|
||||
# job proves it against a cache that the REAL 0.9.20 binary wrote, on all three
|
||||
# platforms, at both the package floor and the newest 22.x.
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'src/**'
|
||||
- 'scripts/upgrade-path/**'
|
||||
- '.github/workflows/upgrade-path.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
upgrade-path:
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-latest]
|
||||
# Package floor, and the newest 22.x. The floor matters here: node:zlib
|
||||
# gained zstd in 22.15, so dsh degrades below it (the corpus writes the
|
||||
# uncompressed dsh variant so both legs still count the same numbers).
|
||||
node-version: [22.13.0, 22]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ matrix.node-version }}
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
# dist/cli.js + dist/parse-worker.js. The dash bundle is not exercised by
|
||||
# any command this job runs, so the full `npm run build` is not paid for.
|
||||
- run: npm run build:cli
|
||||
|
||||
- name: Upgrade path from codeburn@0.9.20
|
||||
run: npm run verify:upgrade
|
||||
env:
|
||||
# Under runner.temp so the artifact step below can find it. The space
|
||||
# is deliberate: a real Windows HOME almost always has one.
|
||||
UPGRADE_PATH_WORK: ${{ runner.temp }}/codeburn upgrade path
|
||||
|
||||
- name: Upload payloads on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: upgrade-path-${{ matrix.os }}-node${{ matrix.node-version }}
|
||||
path: ${{ runner.temp }}/codeburn upgrade path/payloads
|
||||
if-no-files-found: ignore
|
||||
|
|
@ -21,6 +21,7 @@
|
|||
"test": "vitest run tests --exclude \"tests/cache-refresh-lock*\"",
|
||||
"test:locks": "vitest run tests/cache-refresh-lock.test.ts tests/cache-refresh-lock-corrupt-body.test.ts tests/cache-refresh-lock-process.test.ts --poolOptions.forks.singleFork=true",
|
||||
"test:watch": "vitest tests --exclude \"tests/cache-refresh-lock*\"",
|
||||
"verify:upgrade": "node scripts/upgrade-path/run.mjs",
|
||||
"prepublishOnly": "npm run build"
|
||||
},
|
||||
"keywords": [
|
||||
|
|
|
|||
130
scripts/upgrade-path/compare.mjs
Normal file
130
scripts/upgrade-path/compare.mjs
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
// Per-provider payload parity between the published CLI and this build.
|
||||
//
|
||||
// node scripts/upgrade-path/compare.mjs <baselineDir> <upgradedDir>
|
||||
//
|
||||
// Each dir holds the payloads run.mjs captured: `export.json` (per-call records,
|
||||
// the token/call source) and `menubar.json` (the unrounded per-provider cost).
|
||||
// Prints one row per provider and exits non-zero on a diff that is not expected.
|
||||
//
|
||||
// Expectations, and why:
|
||||
// claude, codex, gemini, kiro, cursor parse identically either side of the
|
||||
// upgrade. Calls and every token field must match EXACTLY; cost is allowed
|
||||
// COST_TOLERANCE of drift because the two binaries carry different bundled
|
||||
// LiteLLM price snapshots and only agree when the shared pricing cache in
|
||||
// CODEBURN_CACHE_DIR is warm (which it is, unless the runner is offline).
|
||||
// grok changed by design in #1015: usage now comes from the CLI's own
|
||||
// turn_completed records instead of a context-curve estimate. The change
|
||||
// is REPORTED, never asserted — not even directionally. On real corpora
|
||||
// the changelog documents totals rising, but that is a property of real
|
||||
// Grok sessions, and the direction here would only reflect how the
|
||||
// generator happened to size its synthetic context curve against its
|
||||
// synthetic usage records. Both sides must still count the same SESSIONS,
|
||||
// which is the part the corpus can honestly establish.
|
||||
// dsh did not exist in the published CLI. Reported; required to be absent
|
||||
// in the baseline and present after the upgrade.
|
||||
const EXACT = ['claude', 'codex', 'gemini', 'kiro', 'cursor']
|
||||
const CHANGED_BY_DESIGN = ['grok']
|
||||
const NEW_IN_THIS_RELEASE = ['dsh']
|
||||
|
||||
const COST_TOLERANCE = 0.005 // 0.5% relative
|
||||
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
|
||||
const [baseDir, upDir] = process.argv.slice(2)
|
||||
if (!baseDir || !upDir) {
|
||||
console.error('usage: compare.mjs <baselineDir> <upgradedDir>')
|
||||
process.exit(2)
|
||||
}
|
||||
|
||||
const TOKEN_FIELDS = ['inputTokens', 'outputTokens', 'reasoningTokens', 'cacheWriteTokens', 'cacheReadTokens']
|
||||
|
||||
function load(dir) {
|
||||
const exported = JSON.parse(readFileSync(join(dir, 'export.json'), 'utf8'))
|
||||
const menubar = JSON.parse(readFileSync(join(dir, 'menubar.json'), 'utf8'))
|
||||
const byProvider = {}
|
||||
for (const r of exported.records ?? []) {
|
||||
const p = r.provider || 'unknown'
|
||||
const acc = (byProvider[p] ??= { calls: 0, cost: 0, ...Object.fromEntries(TOKEN_FIELDS.map(f => [f, 0])) })
|
||||
acc.calls++
|
||||
acc.cost += r.cost ?? 0
|
||||
for (const f of TOKEN_FIELDS) acc[f] += r[f] ?? 0
|
||||
}
|
||||
// Prefer the unrounded cost, keyed by the provider's internal id.
|
||||
// `providerDetails` is the only place that pairing exists — the sibling
|
||||
// `providers` map is keyed by lowercased display name. The per-record sum
|
||||
// above stands in when a binary predates providerDetails; it is rounded per
|
||||
// record, so it is the coarser of the two.
|
||||
for (const d of menubar.current?.providerDetails ?? []) {
|
||||
if (byProvider[d.id]) byProvider[d.id].cost = d.cost
|
||||
}
|
||||
return byProvider
|
||||
}
|
||||
|
||||
const base = load(baseDir)
|
||||
const up = load(upDir)
|
||||
const providers = [...new Set([...Object.keys(base), ...Object.keys(up)])].sort()
|
||||
|
||||
const failures = []
|
||||
const notes = []
|
||||
const rows = []
|
||||
|
||||
const relDiff = (a, b) => (a === 0 && b === 0 ? 0 : Math.abs(b - a) / Math.max(Math.abs(a), Math.abs(b)))
|
||||
const fmt = n => (Number.isInteger(n) ? String(n) : n.toFixed(6))
|
||||
|
||||
for (const name of providers) {
|
||||
const b = base[name]
|
||||
const u = up[name]
|
||||
let verdict
|
||||
|
||||
if (NEW_IN_THIS_RELEASE.includes(name)) {
|
||||
if (b) failures.push(`${name}: expected to be absent from the 0.9.20 baseline, but it reported ${b.calls} calls`)
|
||||
else if (!u || u.calls === 0) failures.push(`${name}: new in this release but the upgraded run reported nothing`)
|
||||
verdict = 'new (expected)'
|
||||
} else if (!b || !u) {
|
||||
failures.push(`${name}: present in ${b ? 'baseline' : 'upgraded'} only`)
|
||||
verdict = 'MISSING'
|
||||
} else if (CHANGED_BY_DESIGN.includes(name)) {
|
||||
const bt = TOKEN_FIELDS.reduce((s, f) => s + b[f], 0)
|
||||
const ut = TOKEN_FIELDS.reduce((s, f) => s + u[f], 0)
|
||||
if (b.calls !== u.calls) failures.push(`${name}: usage accounting changed in #1015, but the session/call COUNT should not have: ${b.calls} != ${u.calls}`)
|
||||
verdict = 'changed by design'
|
||||
notes.push(`${name}: tokens ${bt} -> ${ut}, cost ${fmt(b.cost)} -> ${fmt(u.cost)} (#1015, expected; magnitude here is a property of the fixture, not evidence)`)
|
||||
} else {
|
||||
const diffs = []
|
||||
if (b.calls !== u.calls) diffs.push(`calls ${b.calls} != ${u.calls}`)
|
||||
for (const f of TOKEN_FIELDS) if (b[f] !== u[f]) diffs.push(`${f} ${b[f]} != ${u[f]}`)
|
||||
const costDrift = relDiff(b.cost, u.cost)
|
||||
if (costDrift > COST_TOLERANCE) diffs.push(`cost ${fmt(b.cost)} != ${fmt(u.cost)} (${(costDrift * 100).toFixed(3)}% > ${(COST_TOLERANCE * 100).toFixed(1)}%)`)
|
||||
if (!EXACT.includes(name)) {
|
||||
notes.push(`${name}: no expectation declared in compare.mjs; ${diffs.length ? diffs.join(', ') : 'identical'}`)
|
||||
verdict = diffs.length ? 'differs (unclassified)' : 'identical'
|
||||
} else if (diffs.length) {
|
||||
failures.push(`${name}: ${diffs.join(', ')}`)
|
||||
verdict = 'DIFFERS'
|
||||
} else {
|
||||
verdict = costDrift === 0 ? 'identical' : `identical (cost ${(costDrift * 100).toFixed(3)}% drift)`
|
||||
}
|
||||
}
|
||||
|
||||
rows.push({
|
||||
provider: name,
|
||||
calls: `${b?.calls ?? '-'} -> ${u?.calls ?? '-'}`,
|
||||
tokens: `${b ? TOKEN_FIELDS.reduce((s, f) => s + b[f], 0) : '-'} -> ${u ? TOKEN_FIELDS.reduce((s, f) => s + u[f], 0) : '-'}`,
|
||||
cost: `${b ? fmt(b.cost) : '-'} -> ${u ? fmt(u.cost) : '-'}`,
|
||||
verdict,
|
||||
})
|
||||
}
|
||||
|
||||
const cols = ['provider', 'calls', 'tokens', 'cost', 'verdict']
|
||||
const width = Object.fromEntries(cols.map(c => [c, Math.max(c.length, ...rows.map(r => r[c].length))]))
|
||||
const line = r => cols.map(c => String(r[c]).padEnd(width[c])).join(' ')
|
||||
console.log('')
|
||||
console.log(line(Object.fromEntries(cols.map(c => [c, c.toUpperCase()]))))
|
||||
console.log(cols.map(c => '-'.repeat(width[c])).join(' '))
|
||||
for (const r of rows) console.log(line(r))
|
||||
console.log('')
|
||||
for (const n of notes) console.log(`note: ${n}`)
|
||||
for (const f of failures) console.log(`FAIL: ${f}`)
|
||||
console.log(failures.length ? `\nparity: ${failures.length} unexpected difference(s)` : '\nparity: ok')
|
||||
process.exit(failures.length ? 1 : 0)
|
||||
377
scripts/upgrade-path/gen-corpus.mjs
Normal file
377
scripts/upgrade-path/gen-corpus.mjs
Normal file
|
|
@ -0,0 +1,377 @@
|
|||
// Deterministic multi-provider fixture corpus for the upgrade-path check.
|
||||
//
|
||||
// node scripts/upgrade-path/gen-corpus.mjs <homeDir>
|
||||
//
|
||||
// Lays sessions out at each provider's DEFAULT path under <homeDir>, so the run
|
||||
// only has to set HOME/USERPROFILE and no per-provider override var. Everything
|
||||
// is seeded off a fixed constant: two invocations against the same day produce
|
||||
// byte-identical files, which is what makes the worker-determinism and
|
||||
// 0.9.20-vs-main payload comparisons meaningful.
|
||||
//
|
||||
// Day anchoring is the one thing that moves: sessions are dated relative to
|
||||
// today so they land inside the daily cache's backfill window. That is fine —
|
||||
// every comparison this corpus feeds happens inside a single run.
|
||||
|
||||
import { mkdirSync, writeFileSync, rmSync, existsSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { createRequire } from 'node:module'
|
||||
|
||||
const require_ = createRequire(import.meta.url)
|
||||
|
||||
const HOME = process.argv[2]
|
||||
if (!HOME) {
|
||||
console.error('usage: gen-corpus.mjs <homeDir>')
|
||||
process.exit(2)
|
||||
}
|
||||
|
||||
// mulberry32 — same seed, same corpus.
|
||||
let seedState = 0x9e3779b9
|
||||
function rnd() {
|
||||
seedState = (seedState + 0x6d2b79f5) | 0
|
||||
let t = seedState
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1)
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61)
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296
|
||||
}
|
||||
const pick = arr => arr[Math.floor(rnd() * arr.length)]
|
||||
const between = (lo, hi) => lo + Math.floor(rnd() * (hi - lo))
|
||||
|
||||
// Day 0 = 92 days ago (UTC midnight); the corpus spans day 0..91.
|
||||
const DAY_MS = 86_400_000
|
||||
const SPAN_DAYS = 92
|
||||
const day0 = Math.floor(Date.now() / DAY_MS) * DAY_MS - (SPAN_DAYS - 1) * DAY_MS
|
||||
const at = (day, hour, min = 0, sec = 0) =>
|
||||
new Date(day0 + day * DAY_MS + hour * 3_600_000 + min * 60_000 + sec * 1000)
|
||||
const iso = d => d.toISOString()
|
||||
|
||||
const write = (path, body) => {
|
||||
mkdirSync(join(path, '..'), { recursive: true })
|
||||
writeFileSync(path, body)
|
||||
}
|
||||
const writeLines = (path, lines) => write(path, lines.join('\n') + '\n')
|
||||
|
||||
const PROJECTS = ['/work/api-gateway', '/work/billing', '/work/web app', '/work/infra']
|
||||
|
||||
// ── claude ───────────────────────────────────────────────────────────────────
|
||||
// 204 transcripts across the span: 200 plain, 2 parent/sidechain pairs. One
|
||||
// plain transcript carries a single line over 32 KB (the large-line scanner
|
||||
// path); the parent/sidechain pairs exercise the v7 spawn-link capture that the
|
||||
// migration has to carry forward.
|
||||
|
||||
const CLAUDE_MODELS = ['claude-sonnet-4-5', 'claude-opus-4-8', 'claude-haiku-4-5']
|
||||
|
||||
function claudeUser(sessionId, ts, cwd, text) {
|
||||
return JSON.stringify({ type: 'user', sessionId, timestamp: iso(ts), cwd, gitBranch: 'main', message: { role: 'user', content: text } })
|
||||
}
|
||||
|
||||
function claudeAssistant(sessionId, ts, cwd, msgId, model, usage, content) {
|
||||
return JSON.stringify({
|
||||
type: 'assistant', sessionId, timestamp: iso(ts), cwd, gitBranch: 'main',
|
||||
message: { id: msgId, type: 'message', role: 'assistant', model, content, usage },
|
||||
})
|
||||
}
|
||||
|
||||
function claudeSession(sessionId, day, cwd, turns, opts = {}) {
|
||||
const lines = []
|
||||
for (let t = 0; t < turns; t++) {
|
||||
const ts = at(day, 9 + (t % 8), (t * 7) % 60)
|
||||
lines.push(claudeUser(sessionId, ts, cwd, `task ${t} for ${sessionId}`))
|
||||
const content = [
|
||||
{ type: 'text', text: `step ${t}` },
|
||||
{ type: 'tool_use', id: `tu-${sessionId}-${t}`, name: t % 3 === 0 ? 'Edit' : 'Read', input: { file_path: `${cwd}/src/f${t}.ts` } },
|
||||
]
|
||||
// One line north of 32 KB, on the file the caller asked for it on.
|
||||
if (opts.hugeLineAtTurn === t) content.push({ type: 'text', text: 'y'.repeat(40 * 1024) })
|
||||
lines.push(claudeAssistant(sessionId, at(day, 9 + (t % 8), (t * 7) % 60, 30), cwd, `msg-${sessionId}-${t}`, pick(CLAUDE_MODELS), {
|
||||
input_tokens: between(400, 4000),
|
||||
output_tokens: between(40, 900),
|
||||
cache_read_input_tokens: between(0, 20000),
|
||||
cache_creation_input_tokens: between(0, 3000),
|
||||
}, content))
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
function genClaude() {
|
||||
const projectsDir = join(HOME, '.claude', 'projects')
|
||||
let files = 0
|
||||
for (let i = 0; i < 200; i++) {
|
||||
const cwd = PROJECTS[i % PROJECTS.length]
|
||||
const day = (i * 7) % SPAN_DAYS
|
||||
const sid = `c-${String(i).padStart(4, '0')}`
|
||||
const dirName = cwd.replace(/[/ ]/g, '-')
|
||||
writeLines(join(projectsDir, dirName, `${sid}.jsonl`), claudeSession(sid, day, cwd, between(4, 14), i === 137 ? { hugeLineAtTurn: 2 } : {}))
|
||||
files++
|
||||
}
|
||||
|
||||
// Two parent transcripts, each spawning one subagent whose transcript lives
|
||||
// under <parent-uuid>/subagents/agent-<id>.jsonl and is marked isSidechain.
|
||||
for (let p = 0; p < 2; p++) {
|
||||
const cwd = PROJECTS[p]
|
||||
const dirName = cwd.replace(/[/ ]/g, '-')
|
||||
const parent = `p-000${p}`
|
||||
const agent = `a-000${p}`
|
||||
const day = 40 + p * 10
|
||||
const parentLines = claudeSession(parent, day, cwd, 5)
|
||||
parentLines.push(JSON.stringify({
|
||||
type: 'assistant', sessionId: parent, timestamp: iso(at(day, 12)), cwd,
|
||||
message: { id: `m-spawn-${p}`, type: 'message', role: 'assistant', model: 'claude-sonnet-4-5', content: [{ type: 'tool_use', id: `toolu_spawn_${p}`, name: 'Agent', input: {} }], usage: { input_tokens: 120, output_tokens: 30 } },
|
||||
}))
|
||||
parentLines.push(JSON.stringify({
|
||||
type: 'user', sessionId: parent, timestamp: iso(at(day, 12, 1)), cwd,
|
||||
message: { role: 'user', content: [{ type: 'tool_result', tool_use_id: `toolu_spawn_${p}`, content: 'subagent done' }] },
|
||||
toolUseResult: { status: 'completed', agentId: agent, content: 'subagent done' },
|
||||
}))
|
||||
parentLines.push(JSON.stringify({ type: 'pr-link', sessionId: parent, timestamp: iso(at(day, 12, 2)), cwd, prUrl: `https://github.com/acme/repo/pull/${100 + p}` }))
|
||||
writeLines(join(projectsDir, dirName, `${parent}.jsonl`), parentLines)
|
||||
files++
|
||||
|
||||
const side = []
|
||||
for (let t = 0; t < 4; t++) {
|
||||
side.push(JSON.stringify({ type: 'user', isSidechain: true, sessionId: parent, agentId: agent, timestamp: iso(at(day, 12, 3 + t)), cwd, message: { role: 'user', content: `sub task ${t}` } }))
|
||||
side.push(JSON.stringify({
|
||||
type: 'assistant', isSidechain: true, sessionId: parent, agentId: agent, timestamp: iso(at(day, 12, 3 + t, 20)), cwd,
|
||||
message: { id: `sub-${p}-${t}`, type: 'message', role: 'assistant', model: 'claude-opus-4-8', content: [{ type: 'text', text: 'ok' }], usage: { input_tokens: between(800, 2000), output_tokens: between(100, 400), cache_read_input_tokens: between(0, 5000) } },
|
||||
}))
|
||||
}
|
||||
writeLines(join(projectsDir, dirName, parent, 'subagents', `agent-${agent}.jsonl`), side)
|
||||
write(join(projectsDir, dirName, parent, 'subagents', `agent-${agent}.meta.json`), JSON.stringify({ agentType: 'reviewer' }))
|
||||
files++
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
// ── codex ────────────────────────────────────────────────────────────────────
|
||||
// token_count carries a CUMULATIVE total_token_usage; the parser diffs
|
||||
// consecutive events, so the running totals below must only ever grow.
|
||||
|
||||
function genCodex() {
|
||||
const root = join(HOME, '.codex', 'sessions')
|
||||
let files = 0
|
||||
for (let i = 0; i < 24; i++) {
|
||||
const day = (i * 4) % SPAN_DAYS
|
||||
const d = new Date(day0 + day * DAY_MS)
|
||||
const cwd = PROJECTS[i % PROJECTS.length]
|
||||
const sid = `codex-${String(i).padStart(3, '0')}`
|
||||
const lines = [JSON.stringify({ type: 'session_meta', timestamp: iso(at(day, 10)), payload: { cwd, originator: 'codex-cli', session_id: sid, model: 'gpt-5.3-codex' } })]
|
||||
const total = { input_tokens: 0, cached_input_tokens: 0, output_tokens: 0, reasoning_output_tokens: 0, total_tokens: 0 }
|
||||
for (let t = 0; t < between(3, 9); t++) {
|
||||
const ts = iso(at(day, 10, t * 5))
|
||||
const last = { input_tokens: between(500, 6000), cached_input_tokens: between(0, 2000), output_tokens: between(50, 800), reasoning_output_tokens: between(0, 300), total_tokens: 0 }
|
||||
last.total_tokens = last.input_tokens + last.output_tokens
|
||||
for (const k of Object.keys(total)) total[k] += last[k]
|
||||
lines.push(JSON.stringify({ type: 'event_msg', timestamp: ts, payload: { type: 'task_started' } }))
|
||||
lines.push(JSON.stringify({ type: 'response_item', timestamp: ts, payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text: `task ${t}` }] } }))
|
||||
lines.push(JSON.stringify({ type: 'response_item', timestamp: ts, payload: { type: 'function_call', name: 'shell', call_id: `c${t}`, arguments: JSON.stringify({ command: 'ls' }) } }))
|
||||
lines.push(JSON.stringify({ type: 'response_item', timestamp: ts, payload: { type: 'function_call_output', call_id: `c${t}` } }))
|
||||
lines.push(JSON.stringify({ type: 'response_item', timestamp: ts, payload: { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'done' }] } }))
|
||||
lines.push(JSON.stringify({ type: 'event_msg', timestamp: ts, payload: { type: 'token_count', info: { last_token_usage: last, total_token_usage: { ...total } } } }))
|
||||
lines.push(JSON.stringify({ type: 'event_msg', timestamp: ts, payload: { type: 'task_complete', duration_ms: 4000 } }))
|
||||
}
|
||||
const dir = join(root, String(d.getUTCFullYear()), String(d.getUTCMonth() + 1).padStart(2, '0'), String(d.getUTCDate()).padStart(2, '0'))
|
||||
writeLines(join(dir, `rollout-${sid}.jsonl`), lines)
|
||||
files++
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
// ── gemini ───────────────────────────────────────────────────────────────────
|
||||
|
||||
function genGemini() {
|
||||
const messages = []
|
||||
for (let t = 0; t < 12; t++) {
|
||||
messages.push({ id: `u${t}`, timestamp: iso(at(60, 9, t * 3)), type: 'user', content: `inspect ${t}` })
|
||||
messages.push({
|
||||
id: `g${t}`, timestamp: iso(at(60, 9, t * 3, 20)), type: 'gemini', content: 'reading files',
|
||||
model: 'gemini-3.1-pro-preview',
|
||||
tokens: { input: between(200, 3000), cached: between(0, 1000), output: between(30, 400), thoughts: between(0, 200) },
|
||||
toolCalls: [{ id: `t${t}`, name: 'read_file', args: { path: 'src/index.ts' } }],
|
||||
})
|
||||
}
|
||||
write(join(HOME, '.gemini', 'tmp', 'api-gateway', 'chats', 'session-upgrade-1.json'),
|
||||
JSON.stringify({ sessionId: 'gemini-session-1', startTime: iso(at(60, 9)), messages }))
|
||||
return 1
|
||||
}
|
||||
|
||||
// ── kiro ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
function genKiro() {
|
||||
const dir = join(HOME, '.kiro', 'sessions', 'cli')
|
||||
const id = 'kiro-upgrade-1'
|
||||
const lines = []
|
||||
for (let t = 0; t < 6; t++) {
|
||||
lines.push(JSON.stringify({ kind: 'Prompt', data: { content: [{ kind: 'text', data: `add feature ${t}` }] } }))
|
||||
lines.push(JSON.stringify({ kind: 'AssistantMessage', data: { content: [{ kind: 'text', data: `Done — added feature ${t} and its tests.` }] } }))
|
||||
}
|
||||
writeLines(join(dir, `${id}.jsonl`), lines)
|
||||
write(join(dir, `${id}.json`), JSON.stringify({
|
||||
session_id: id, cwd: '/work/billing',
|
||||
created_at: iso(at(70, 10)), updated_at: iso(at(70, 11)),
|
||||
session_state: {
|
||||
rts_model_state: { model_info: { model_id: 'auto' } },
|
||||
conversation_metadata: { user_turn_metadatas: [{ end_timestamp: iso(at(70, 11)), metering_usage: [] }] },
|
||||
},
|
||||
}))
|
||||
return 2
|
||||
}
|
||||
|
||||
// ── dsh ──────────────────────────────────────────────────────────────────────
|
||||
// Written UNCOMPRESSED on purpose: node:zlib gained zstd in 22.15 and the
|
||||
// package floor is 22.13, so the .zstd variant would silently drop out of the
|
||||
// floor matrix leg and the two legs would not be comparable.
|
||||
|
||||
function genDsh() {
|
||||
const cwd = '/work/api-gateway'
|
||||
const encoded = `--${cwd.replace(/[/\\]/g, '-')}--`
|
||||
const dir = join(HOME, '.dsh', 'sessions', encoded, 'session-upgrade-0001')
|
||||
const lines = [
|
||||
JSON.stringify({ type: 'session', version: 0, id: 'session-upgrade-0001', createdAt: at(75, 10).getTime(), cwd, delegationDepth: 0, agentPreset: 'cordis' }),
|
||||
JSON.stringify({ type: 'request/header', seq: 1, time: at(75, 10).getTime(), data: { header: { config: { provider: 'deepseek-official', model: 'deepseek-v3.2', reasoningEffort: 'max', maxTokens: 256000 } } } }),
|
||||
]
|
||||
let seq = 2
|
||||
for (let turn = 1; turn <= 8; turn++) {
|
||||
const base = at(75, 10, turn * 5).getTime()
|
||||
lines.push(JSON.stringify({ type: 'turn/start', seq: seq++, time: base, data: { turn } }))
|
||||
lines.push(JSON.stringify({ type: 'user/message', seq: seq++, time: base + 100, data: { content: [{ type: 'text', text: `build ${turn}` }], source: { kind: 'user' }, role: 'user', id: `msg-${turn}` } }))
|
||||
lines.push(JSON.stringify({ type: 'tool/call', seq: seq++, time: base + 200, data: { turn, step: 1, callId: `call_${turn}`, name: 'bash', arguments: JSON.stringify({ command: 'git status' }) } }))
|
||||
lines.push(JSON.stringify({
|
||||
type: 'assistant/message', seq: seq++, time: base + 900,
|
||||
data: { turn, step: 1, message: { role: 'assistant', content: [{ type: 'text', text: 'done' }] }, usage: { inputTokens: between(2000, 20000), outputTokens: between(100, 900), cacheReadTokens: between(0, 5000), reasoningTokens: between(0, 600) } },
|
||||
}))
|
||||
}
|
||||
writeLines(join(dir, 'session.jsonl'), lines)
|
||||
return 1
|
||||
}
|
||||
|
||||
// ── grok ─────────────────────────────────────────────────────────────────────
|
||||
// Uses the authoritative `turn_completed.usage` records that #1015 switched to.
|
||||
|
||||
function genGrok() {
|
||||
const cwd = '/work/infra'
|
||||
const root = join(HOME, '.grok', 'sessions', encodeURIComponent(cwd))
|
||||
let files = 0
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const id = `019edf9c-0000-7000-8000-00000000000${i + 1}`
|
||||
const day = 80 + i
|
||||
const dir = join(root, id)
|
||||
write(join(dir, 'summary.json'), JSON.stringify({
|
||||
info: { id, cwd }, created_at: iso(at(day, 11)), updated_at: iso(at(day, 12)), last_active_at: iso(at(day, 12)),
|
||||
num_messages: 12, current_model_id: 'grok-build', session_summary: 'repo work', generated_title: 'repo work',
|
||||
}))
|
||||
write(join(dir, 'signals.json'), JSON.stringify({
|
||||
primaryModelId: 'grok-build', modelsUsed: ['grok-build'], toolsUsed: ['read_file', 'grep'],
|
||||
contextTokensUsed: 40000, contextWindowTokens: 512000,
|
||||
}))
|
||||
const updates = []
|
||||
let running = 0
|
||||
for (let t = 0; t < 5; t++) {
|
||||
// Streamed chunk carrying the running context counter. This is all the
|
||||
// published CLI can see, and what it estimates from; main ignores it in
|
||||
// favour of the turn_completed record below. Both are present in a real
|
||||
// session, so the corpus carries both and the two versions have something
|
||||
// to disagree about.
|
||||
running += between(3000, 12000)
|
||||
updates.push(JSON.stringify({
|
||||
timestamp: iso(at(day, 11, t * 5)), method: 'session/update',
|
||||
params: { sessionId: id, update: { sessionUpdate: 'agent_message_chunk', content: { type: 'text', text: `chunk ${t}` } }, _meta: { totalTokens: running, promptId: `p${t}`, updateType: 'AgentMessageChunk', modelId: 'grok-build' } },
|
||||
}))
|
||||
const usage = {
|
||||
inputTokens: between(1000, 9000), outputTokens: between(80, 700), totalTokens: 0,
|
||||
cachedReadTokens: between(0, 40000), cacheCreationTokens: between(0, 2000), reasoningTokens: between(0, 300),
|
||||
modelCalls: 1, apiDurationMs: 1000, costUsdTicks: 125117780000, numTurns: 1,
|
||||
}
|
||||
usage.totalTokens = usage.inputTokens + usage.outputTokens
|
||||
usage.modelUsage = { 'grok-4.6-build': { ...usage } }
|
||||
updates.push(JSON.stringify({
|
||||
timestamp: Math.floor(at(day, 11, t * 5).getTime() / 1000), method: '_x.ai/session/update',
|
||||
params: { sessionId: id, update: { sessionUpdate: 'turn_completed', prompt_id: `p${t}`, usage }, _meta: { eventId: `event-${t}`, agentTimestampMs: at(day, 11, t * 5).getTime() } },
|
||||
}))
|
||||
}
|
||||
writeLines(join(dir, 'updates.jsonl'), updates)
|
||||
files += 3
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
// ── cursor (WAL-mode SQLite) ─────────────────────────────────────────────────
|
||||
// Left with an un-checkpointed -wal sidecar and NO -shm: that is what a live
|
||||
// Cursor database looks like on disk, and it is the shape that made
|
||||
// read-only opens fail before #1017.
|
||||
|
||||
function genCursor() {
|
||||
let DatabaseSync
|
||||
try { ({ DatabaseSync } = require_('node:sqlite')) } catch { return 0 }
|
||||
|
||||
const userDir = process.platform === 'darwin'
|
||||
? join(HOME, 'Library', 'Application Support', 'Cursor', 'User')
|
||||
: process.platform === 'win32'
|
||||
? join(HOME, 'AppData', 'Roaming', 'Cursor', 'User')
|
||||
: join(HOME, '.config', 'Cursor', 'User')
|
||||
|
||||
const globalDir = join(userDir, 'globalStorage')
|
||||
mkdirSync(globalDir, { recursive: true })
|
||||
const dbPath = join(globalDir, 'state.vscdb')
|
||||
for (const suffix of ['', '-wal', '-shm']) rmSync(dbPath + suffix, { force: true })
|
||||
const db = new DatabaseSync(dbPath)
|
||||
db.exec('PRAGMA journal_mode=WAL')
|
||||
db.exec('CREATE TABLE cursorDiskKV (key TEXT PRIMARY KEY, value BLOB)')
|
||||
db.exec('CREATE TABLE ItemTable (key TEXT UNIQUE, value BLOB)')
|
||||
const ins = db.prepare('INSERT INTO cursorDiskKV (key, value) VALUES (?, ?)')
|
||||
|
||||
const composers = []
|
||||
for (let c = 0; c < 6; c++) {
|
||||
const composerId = `composer-${c}`
|
||||
composers.push({ composerId, name: `session-${c}`, unifiedMode: 'agent' })
|
||||
ins.run(`composerData:${composerId}`, JSON.stringify({
|
||||
promptTokenBreakdown: { totalUsedTokens: between(10000, 90000) },
|
||||
createdAt: at(85, 9 + c).getTime(),
|
||||
}))
|
||||
for (let b = 0; b < 8; b++) {
|
||||
const createdAt = iso(at(85, 9 + c, b * 4))
|
||||
ins.run(`bubbleId:${composerId}:u${b}`, JSON.stringify({ type: 1, conversationId: composerId, createdAt, text: `ask ${b}`, codeBlocks: '[]' }))
|
||||
ins.run(`bubbleId:${composerId}:a${b}`, JSON.stringify({
|
||||
type: 2, conversationId: composerId, createdAt, text: `reply ${b}`, codeBlocks: '[]',
|
||||
tokenCount: { inputTokens: between(300, 4000), outputTokens: between(40, 500) },
|
||||
modelInfo: { modelName: 'claude-4.6-sonnet' },
|
||||
requestId: `req-${c}-${b}`,
|
||||
}))
|
||||
}
|
||||
}
|
||||
// Checkpoint what is written so far, then stop auto-checkpointing and append
|
||||
// more: the tail rows live only in the -wal the copy below carries.
|
||||
db.exec('PRAGMA wal_checkpoint(TRUNCATE)')
|
||||
db.exec('PRAGMA wal_autocheckpoint=0')
|
||||
ins.run('composerData:composer-tail', JSON.stringify({ promptTokenBreakdown: { totalUsedTokens: 12345 }, createdAt: at(86, 9).getTime() }))
|
||||
ins.run('bubbleId:composer-tail:a0', JSON.stringify({
|
||||
type: 2, conversationId: 'composer-tail', createdAt: iso(at(86, 9)), text: 'tail reply', codeBlocks: '[]',
|
||||
tokenCount: { inputTokens: 2222, outputTokens: 333 }, modelInfo: { modelName: 'claude-4.6-sonnet' },
|
||||
}))
|
||||
composers.push({ composerId: 'composer-tail', name: 'session-tail', unifiedMode: 'agent' })
|
||||
db.close()
|
||||
|
||||
// Per-workspace DB naming the composers, plus the workspace.json that gives
|
||||
// the project its name.
|
||||
const wsDir = join(userDir, 'workspaceStorage', 'ws0000000000000000000000000000000')
|
||||
mkdirSync(wsDir, { recursive: true })
|
||||
for (const suffix of ['', '-wal', '-shm']) rmSync(join(wsDir, 'state.vscdb' + suffix), { force: true })
|
||||
const wsDb = new DatabaseSync(join(wsDir, 'state.vscdb'))
|
||||
wsDb.exec('CREATE TABLE ItemTable (key TEXT UNIQUE, value BLOB)')
|
||||
wsDb.prepare('INSERT INTO ItemTable (key, value) VALUES (?, ?)').run('composer.composerData', JSON.stringify({ allComposers: composers }))
|
||||
wsDb.close()
|
||||
write(join(wsDir, 'workspace.json'), JSON.stringify({ folder: 'file:///work/billing' }))
|
||||
|
||||
return existsSync(dbPath + '-wal') ? 3 : 2
|
||||
}
|
||||
|
||||
// ── run ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
const counts = {
|
||||
claude: genClaude(),
|
||||
codex: genCodex(),
|
||||
gemini: genGemini(),
|
||||
kiro: genKiro(),
|
||||
dsh: genDsh(),
|
||||
grok: genGrok(),
|
||||
cursor: genCursor(),
|
||||
}
|
||||
console.log(JSON.stringify(counts))
|
||||
343
scripts/upgrade-path/run.mjs
Normal file
343
scripts/upgrade-path/run.mjs
Normal file
|
|
@ -0,0 +1,343 @@
|
|||
// Upgrade-path verification: prove that a cache written by the last PUBLISHED
|
||||
// CLI survives this build's first run, on this platform, with this Node.
|
||||
//
|
||||
// npm run verify:upgrade
|
||||
//
|
||||
// What it does, in order:
|
||||
// 1. generates a deterministic multi-provider corpus into an isolated HOME
|
||||
// (whose path contains a space, because a real Windows HOME usually does)
|
||||
// 2. installs codeburn@0.9.20 into an isolated global prefix and runs it,
|
||||
// producing a genuine session-cache.v7 + daily-cache.v17
|
||||
// 3. installs THIS build the same way and runs it against the SAME cache dir,
|
||||
// through the npm bin shim rather than `node dist/cli.js`, so
|
||||
// dist/parse-worker.js has to resolve from a symlinked entry point
|
||||
// 4. asserts the migration landed and compares payloads per provider
|
||||
// 5. serve --stdio smoke, worker determinism, warm-run stability
|
||||
//
|
||||
// Env: UPGRADE_PATH_WORK (work dir), UPGRADE_PATH_OLD (published version to
|
||||
// upgrade from), UPGRADE_PATH_KEEP=1 to leave the work dir behind.
|
||||
|
||||
import { spawnSync, spawn } from 'node:child_process'
|
||||
import { mkdirSync, rmSync, existsSync, readdirSync, statSync, readFileSync, writeFileSync, copyFileSync } from 'node:fs'
|
||||
import { join, dirname } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { tmpdir } from 'node:os'
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url))
|
||||
const REPO = join(HERE, '..', '..')
|
||||
const OLD_VERSION = process.env['UPGRADE_PATH_OLD'] || '0.9.20'
|
||||
const WORK = process.env['UPGRADE_PATH_WORK'] || join(tmpdir(), 'codeburn upgrade path')
|
||||
|
||||
// The published binary's cache versions. If a future baseline writes something
|
||||
// else these two are the knobs to move, and the assertions below will say so.
|
||||
const OLD_SESSION_CACHE = 'session-cache.v7.json'
|
||||
const OLD_DAILY_CACHE = 'daily-cache.v17.json'
|
||||
const NEW_SESSION_CACHE_DIR = 'session-cache.v9'
|
||||
const NEW_DAILY_CACHE = 'daily-cache.v19.json'
|
||||
|
||||
const HOME = join(WORK, 'user home')
|
||||
const PAYLOADS = join(WORK, 'payloads')
|
||||
const CACHES = join(WORK, 'caches')
|
||||
const npmCmd = process.platform === 'win32' ? 'npm.cmd' : 'npm'
|
||||
|
||||
let failures = 0
|
||||
let skipped = 0
|
||||
const step = msg => console.log(`\n=== ${msg}`)
|
||||
const ok = msg => console.log(` ok ${msg}`)
|
||||
const fail = msg => { failures++; console.log(` FAIL ${msg}`) }
|
||||
const skip = msg => { skipped++; console.log(` skip ${msg}`) }
|
||||
const check = (cond, msg) => (cond ? ok(msg) : fail(msg))
|
||||
|
||||
function run(cmd, args, opts = {}) {
|
||||
const r = spawnSync(cmd, args, { encoding: 'utf8', maxBuffer: 1 << 29, shell: false, ...opts })
|
||||
if (r.error) throw new Error(`${cmd} ${args.join(' ')}: ${r.error.message}`)
|
||||
return r
|
||||
}
|
||||
|
||||
// Node refuses to spawn a .cmd/.bat without a shell, and a shell spawn quotes
|
||||
// nothing for you — so on Windows anything holding a space (every path here
|
||||
// does, by design) has to be quoted by hand.
|
||||
const quoteForShell = s => (process.platform === 'win32' && /[\s&|^]/.test(s) ? `"${s}"` : s)
|
||||
function runShell(cmd, args, opts = {}) {
|
||||
if (process.platform !== 'win32') return run(cmd, args, opts)
|
||||
return run(quoteForShell(cmd), args.map(quoteForShell), { ...opts, shell: true })
|
||||
}
|
||||
|
||||
function cliEnv(cacheDir, extra = {}) {
|
||||
// Deliberately minimal: no provider override vars, so every provider resolves
|
||||
// its own default path under the isolated HOME. APPDATA/LOCALAPPDATA are
|
||||
// pinned under it too — several providers read them on Windows, and inheriting
|
||||
// the runner's would let real (or leftover) data into the comparison.
|
||||
const passthrough = {}
|
||||
for (const k of ['PATH', 'PATHEXT', 'SystemRoot', 'ComSpec', 'windir', 'TEMP', 'TMP', 'NUMBER_OF_PROCESSORS']) {
|
||||
if (process.env[k] !== undefined) passthrough[k] = process.env[k]
|
||||
}
|
||||
return {
|
||||
...passthrough,
|
||||
HOME, USERPROFILE: HOME, TZ: 'UTC', CODEBURN_CACHE_DIR: cacheDir,
|
||||
APPDATA: join(HOME, 'AppData', 'Roaming'), LOCALAPPDATA: join(HOME, 'AppData', 'Local'),
|
||||
...extra,
|
||||
}
|
||||
}
|
||||
|
||||
function cli(bin, args, cacheDir, extra = {}) {
|
||||
const r = run(bin.cmd, [...bin.args, ...args], { env: cliEnv(cacheDir, extra), cwd: WORK })
|
||||
if (r.status !== 0) throw new Error(`${args.join(' ')} exited ${r.status}\n${r.stderr?.slice(0, 4000)}`)
|
||||
return r.stdout
|
||||
}
|
||||
|
||||
// Install into an isolated global prefix, so the CLI runs from a location that
|
||||
// has nothing to do with this checkout — which is what makes dist/parse-worker.js
|
||||
// resolution worth testing. On POSIX npm's bin is a symlink INTO the package and
|
||||
// we drive that directly. On Windows it is a .cmd shim, which Node will not
|
||||
// spawn without a shell; the payload captures go through the installed
|
||||
// dist/cli.js there and the shim itself is smoke-tested once, separately.
|
||||
function installGlobal(prefix, spec) {
|
||||
const r = runShell(npmCmd, ['install', '-g', '--prefix', prefix, spec, '--no-audit', '--no-fund', '--loglevel', 'error'], { cwd: WORK })
|
||||
if (r.status !== 0) throw new Error(`npm install -g ${spec} exited ${r.status}\n${r.stdout}\n${r.stderr}`)
|
||||
const symlink = join(prefix, 'bin', 'codeburn')
|
||||
if (existsSync(symlink)) return { cmd: process.execPath, args: [symlink], shim: null }
|
||||
const winCmd = join(prefix, 'codeburn.cmd')
|
||||
const entry = join(prefix, 'node_modules', 'codeburn', 'dist', 'cli.js')
|
||||
if (existsSync(entry)) return { cmd: process.execPath, args: [entry], shim: existsSync(winCmd) ? winCmd : null }
|
||||
throw new Error(`no codeburn bin under ${prefix}`)
|
||||
}
|
||||
|
||||
// The npm shim, exercised once. Needs a shell on Windows, so nothing with a
|
||||
// space in it is passed through here.
|
||||
function checkShim(bin, cacheDir) {
|
||||
if (!bin.shim) { ok("CLI invoked through npm's bin symlink"); return }
|
||||
const r = runShell(bin.shim, ['--version'], { env: cliEnv(cacheDir), cwd: WORK })
|
||||
check(r.status === 0 && r.stdout.trim().length > 0, `npm .cmd shim runs: ${r.stdout.trim() || r.stderr?.slice(0, 200)}`)
|
||||
}
|
||||
|
||||
// Captured payloads. `export` is the token/call source, `menubar-json` the
|
||||
// unrounded per-provider cost; both are stable given a fixed corpus. --period all
|
||||
// so the whole three-month corpus is in scope on both sides.
|
||||
function capture(bin, cacheDir, outDir, extra = {}) {
|
||||
mkdirSync(outDir, { recursive: true })
|
||||
cli(bin, ['export', '--format', 'json', '--from', '2000-01-01', '--to', '2999-12-31', '-o', join(outDir, 'export.json')], cacheDir, extra)
|
||||
const menubar = cli(bin, ['status', '--format', 'menubar-json', '--period', 'all', '--no-optimize', '--no-timeline'], cacheDir, extra)
|
||||
writeFileSync(join(outDir, 'menubar.json'), menubar)
|
||||
const status = cli(bin, ['status', '--format', 'json', '--period', 'all'], cacheDir, extra)
|
||||
writeFileSync(join(outDir, 'status.json'), status)
|
||||
return { menubar: JSON.parse(menubar), status: JSON.parse(status) }
|
||||
}
|
||||
|
||||
// The one field that moves between two runs of the same payload.
|
||||
const stripGenerated = obj => JSON.parse(JSON.stringify(obj, (k, v) => (k.startsWith('generated') ? undefined : v)))
|
||||
|
||||
// Shards carry real stat data and are published under a random filename, so
|
||||
// "identical" means identical after normalizing both away.
|
||||
function shardSnapshot(cacheDir) {
|
||||
const dir = join(cacheDir, NEW_SESSION_CACHE_DIR)
|
||||
if (!existsSync(dir)) return null
|
||||
const out = {}
|
||||
for (const name of readdirSync(dir).sort()) {
|
||||
if (name === 'envelope.json') continue
|
||||
const body = JSON.parse(readFileSync(join(dir, name), 'utf8'))
|
||||
for (const entry of Object.values(body)) {
|
||||
if (entry && typeof entry === 'object' && entry.fingerprint) {
|
||||
delete entry.fingerprint.dev
|
||||
delete entry.fingerprint.ino
|
||||
delete entry.fingerprint.mtimeMs
|
||||
}
|
||||
}
|
||||
// Key the bucket off the shard's provider.month prefix, dropping the nonce.
|
||||
out[name.replace(/\.[0-9a-f]{16}\.json$/, '')] = sortDeep(body)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
function sortDeep(v) {
|
||||
if (Array.isArray(v)) return v.map(sortDeep)
|
||||
if (v && typeof v === 'object') return Object.fromEntries(Object.keys(v).sort().map(k => [k, sortDeep(v[k])]))
|
||||
return v
|
||||
}
|
||||
|
||||
function shardMtimes(cacheDir) {
|
||||
const dir = join(cacheDir, NEW_SESSION_CACHE_DIR)
|
||||
return Object.fromEntries(readdirSync(dir).sort().map(n => [n, statSync(join(dir, n)).mtimeMs]))
|
||||
}
|
||||
|
||||
// ── 1. corpus ────────────────────────────────────────────────────────────────
|
||||
|
||||
step(`work dir: ${WORK}`)
|
||||
try { rmSync(WORK, { recursive: true, force: true }) } catch (err) { console.log(` note could not clear the work dir (${err.code}); reusing it`) }
|
||||
mkdirSync(HOME, { recursive: true })
|
||||
mkdirSync(PAYLOADS, { recursive: true })
|
||||
|
||||
const gen = run(process.execPath, [join(HERE, 'gen-corpus.mjs'), HOME])
|
||||
if (gen.status !== 0) { console.log(gen.stderr); process.exit(1) }
|
||||
ok(`corpus generated: ${gen.stdout.trim()}`)
|
||||
|
||||
const upgradeCache = join(CACHES, 'upgrade')
|
||||
mkdirSync(upgradeCache, { recursive: true })
|
||||
|
||||
// ── 2. baseline: the last published CLI ──────────────────────────────────────
|
||||
|
||||
step(`baseline: codeburn@${OLD_VERSION}`)
|
||||
const oldBin = installGlobal(join(WORK, 'old'), `codeburn@${OLD_VERSION}`)
|
||||
const oldVersion = cli(oldBin, ['--version'], upgradeCache).trim()
|
||||
check(oldVersion === OLD_VERSION, `installed baseline reports ${oldVersion}`)
|
||||
|
||||
const baseline = capture(oldBin, upgradeCache, join(PAYLOADS, 'baseline'))
|
||||
check(baseline.menubar.current.calls > 0, `baseline counted ${baseline.menubar.current.calls} calls across ${baseline.menubar.current.sessions} sessions`)
|
||||
check(existsSync(join(upgradeCache, OLD_SESSION_CACHE)), `${OLD_VERSION} wrote ${OLD_SESSION_CACHE}`)
|
||||
check(existsSync(join(upgradeCache, OLD_DAILY_CACHE)), `${OLD_VERSION} wrote ${OLD_DAILY_CACHE}`)
|
||||
|
||||
// ── 3. upgrade: this build, same cache dir, through the npm bin shim ─────────
|
||||
|
||||
step('upgrade: this build against the same cache dir')
|
||||
const pack = runShell(npmCmd, ['pack', '--ignore-scripts', '--pack-destination', WORK, '--loglevel', 'error'], { cwd: REPO })
|
||||
if (pack.status !== 0) { console.log(pack.stdout, pack.stderr); process.exit(1) }
|
||||
const tarball = join(WORK, pack.stdout.trim().split('\n').pop().trim())
|
||||
const newBin = installGlobal(join(WORK, 'new'), tarball)
|
||||
ok(`this build installed as ${newBin.args[0] ?? newBin.cmd}`)
|
||||
checkShim(newBin, upgradeCache)
|
||||
|
||||
const upgraded = capture(newBin, upgradeCache, join(PAYLOADS, 'upgraded'))
|
||||
|
||||
check(!existsSync(join(upgradeCache, OLD_SESSION_CACHE)), `${OLD_SESSION_CACHE} removed after the re-layout`)
|
||||
check(existsSync(join(upgradeCache, NEW_SESSION_CACHE_DIR)), `${NEW_SESSION_CACHE_DIR}/ present`)
|
||||
check(existsSync(join(upgradeCache, NEW_SESSION_CACHE_DIR, 'envelope.json')), `${NEW_SESSION_CACHE_DIR}/envelope.json present`)
|
||||
const envelope = JSON.parse(readFileSync(join(upgradeCache, NEW_SESSION_CACHE_DIR, 'envelope.json'), 'utf8'))
|
||||
check(envelope.version === 9 && Object.keys(envelope.providers ?? {}).length > 0,
|
||||
`envelope at version ${envelope.version} with ${Object.keys(envelope.providers ?? {}).length} providers`)
|
||||
check(readdirSync(join(upgradeCache, NEW_SESSION_CACHE_DIR)).some(n => n !== 'envelope.json'), 'shards published alongside the envelope')
|
||||
check(existsSync(join(upgradeCache, NEW_DAILY_CACHE)), `${NEW_DAILY_CACHE} re-derived`)
|
||||
check(existsSync(join(upgradeCache, OLD_DAILY_CACHE)), `${OLD_DAILY_CACHE} kept as the carry-forward baseline`)
|
||||
const oldDays = JSON.parse(readFileSync(join(upgradeCache, OLD_DAILY_CACHE), 'utf8')).days.length
|
||||
const newDays = JSON.parse(readFileSync(join(upgradeCache, NEW_DAILY_CACHE), 'utf8')).days.length
|
||||
check(newDays >= oldDays, `daily history did not shrink: ${oldDays} -> ${newDays} days`)
|
||||
|
||||
// ── 4. payload parity ────────────────────────────────────────────────────────
|
||||
|
||||
step('payload parity vs the baseline')
|
||||
const cmp = run(process.execPath, [join(HERE, 'compare.mjs'), join(PAYLOADS, 'baseline'), join(PAYLOADS, 'upgraded')], { stdio: 'inherit' })
|
||||
if (cmp.status !== 0) failures++
|
||||
|
||||
// ── 5. serve smoke ───────────────────────────────────────────────────────────
|
||||
|
||||
step('serve --stdio')
|
||||
const serveFrames = await serveSmoke()
|
||||
if (serveFrames) {
|
||||
for (const [name, args] of [['menubar-json', ['status', '--format', 'menubar-json', '--period', 'all', '--no-optimize', '--no-timeline']], ['models', ['models', '--format', 'json']]]) {
|
||||
const frame = serveFrames.get(name)
|
||||
if (!frame?.ok) { fail(`serve returned no ok frame for ${name}: ${JSON.stringify(frame)}`); continue }
|
||||
ok(`serve ok frame for ${name}`)
|
||||
const oneShot = cli(newBin, args, upgradeCache)
|
||||
const a = JSON.stringify(stripGenerated(JSON.parse(frame.output)))
|
||||
const b = JSON.stringify(stripGenerated(JSON.parse(oneShot)))
|
||||
check(a === b, `serve ${name} matches the one-shot payload (ignoring generated*)`)
|
||||
}
|
||||
}
|
||||
|
||||
async function serveSmoke() {
|
||||
const child = spawn(newBin.cmd, [...newBin.args, 'serve', '--stdio'], { env: cliEnv(upgradeCache), cwd: WORK, stdio: ['pipe', 'pipe', 'pipe'] })
|
||||
const frames = new Map()
|
||||
let buf = ''
|
||||
let ready = false
|
||||
const done = new Promise(resolve => {
|
||||
child.stdout.on('data', d => {
|
||||
buf += d
|
||||
let nl
|
||||
while ((nl = buf.indexOf('\n')) >= 0) {
|
||||
const line = buf.slice(0, nl); buf = buf.slice(nl + 1)
|
||||
if (!line.trim()) continue
|
||||
let msg
|
||||
try { msg = JSON.parse(line) } catch { continue }
|
||||
if (msg.ready) { ready = true; continue }
|
||||
if (msg.progress !== undefined) continue
|
||||
if (msg.id === 1) frames.set('menubar-json', msg)
|
||||
if (msg.id === 2) frames.set('models', msg)
|
||||
if (frames.size === 2) resolve()
|
||||
}
|
||||
})
|
||||
})
|
||||
child.stdin.write(JSON.stringify({ id: 1, args: ['status', '--format', 'menubar-json', '--period', 'all', '--no-optimize', '--no-timeline'] }) + '\n')
|
||||
child.stdin.write(JSON.stringify({ id: 2, args: ['models', '--format', 'json'] }) + '\n')
|
||||
const timeout = new Promise(r => setTimeout(() => r('timeout'), 240_000))
|
||||
if ((await Promise.race([done, timeout])) === 'timeout') { child.kill(); fail('serve did not answer both requests within 240s'); return null }
|
||||
check(ready, 'serve announced itself with a ready frame')
|
||||
|
||||
// Closing stdin is the documented shutdown: the child must exit on its own.
|
||||
const exited = new Promise(r => child.once('exit', code => r(code)))
|
||||
child.stdin.end()
|
||||
const exitCode = await Promise.race([exited, new Promise(r => setTimeout(() => r('hung'), 30_000))])
|
||||
if (exitCode === 'hung') { child.kill(); fail('serve did not exit when stdin closed') }
|
||||
else ok(`serve exited on stdin close (code ${exitCode})`)
|
||||
return frames
|
||||
}
|
||||
|
||||
// ── 6. worker determinism ────────────────────────────────────────────────────
|
||||
|
||||
step('parse-worker determinism (CODEBURN_PARSE_WORKERS 0 vs 3)')
|
||||
const serialCache = join(CACHES, 'workers-0')
|
||||
const parallelCache = join(CACHES, 'workers-3')
|
||||
for (const dir of [serialCache, parallelCache]) {
|
||||
mkdirSync(dir, { recursive: true })
|
||||
// Seed the shared price table so the two runs cannot be priced differently by
|
||||
// a cache expiring between them. Parsing is unaffected either way.
|
||||
const priced = join(upgradeCache, 'litellm-pricing.json')
|
||||
if (existsSync(priced)) copyFileSync(priced, join(dir, 'litellm-pricing.json'))
|
||||
}
|
||||
const serialOut = capture(newBin, serialCache, join(PAYLOADS, 'workers-0'), { CODEBURN_PARSE_WORKERS: '0', CODEBURN_VERBOSE: '1' })
|
||||
const parallelOut = capture(newBin, parallelCache, join(PAYLOADS, 'workers-3'), { CODEBURN_PARSE_WORKERS: '3', CODEBURN_VERBOSE: '1' })
|
||||
check(JSON.stringify(stripGenerated(serialOut.menubar)) === JSON.stringify(stripGenerated(parallelOut.menubar)),
|
||||
'menubar-json payload identical with and without workers')
|
||||
const readExport = dir => stripGenerated(JSON.parse(readFileSync(join(PAYLOADS, dir, 'export.json'), 'utf8')))
|
||||
check(JSON.stringify(readExport('workers-0')) === JSON.stringify(readExport('workers-3')),
|
||||
'export payload identical with and without workers')
|
||||
check(JSON.stringify(shardSnapshot(serialCache)) === JSON.stringify(shardSnapshot(parallelCache)),
|
||||
'shard bodies identical with and without workers (fingerprint stat data and shard nonces normalized)')
|
||||
|
||||
// A forced pool that never actually spawned would make the check above vacuous.
|
||||
const verbose = run(newBin.cmd, [...newBin.args, 'status', '--format', 'json', '--period', 'all'], {
|
||||
env: cliEnv(join(CACHES, 'workers-probe'), { CODEBURN_PARSE_WORKERS: '3', CODEBURN_VERBOSE: '1' }), cwd: WORK,
|
||||
})
|
||||
const decision = (verbose.stderr || '').split('\n').filter(l => l.includes('parse workers='))
|
||||
if (decision.length === 0) skip('no "parse workers=" line on stderr; cannot confirm the pool was forced')
|
||||
else check(decision.some(l => /parse workers=[1-9]/.test(l)), `worker pool engaged: ${decision.map(l => l.trim()).join(' | ')}`)
|
||||
|
||||
// ── 7. second run is warm ────────────────────────────────────────────────────
|
||||
|
||||
step('second run is warm')
|
||||
const beforeBodies = shardSnapshot(upgradeCache)
|
||||
const beforeMtimes = shardMtimes(upgradeCache)
|
||||
const warm = capture(newBin, upgradeCache, join(PAYLOADS, 'warm'))
|
||||
|
||||
// The direct no-re-parse signal: the worker gate prints how many whole-file
|
||||
// re-parses are pending. On an unchanged corpus that must be zero for the two
|
||||
// providers big enough to be gated.
|
||||
const warmVerbose = run(newBin.cmd, [...newBin.args, 'status', '--format', 'menubar-json', '--period', 'all', '--no-optimize', '--no-timeline'], {
|
||||
env: cliEnv(upgradeCache, { CODEBURN_VERBOSE: '1' }), cwd: WORK,
|
||||
})
|
||||
const pending = (warmVerbose.stderr || '').split('\n').filter(l => l.includes('parse workers='))
|
||||
if (pending.length === 0) skip('no "parse workers=" line on a warm run; cannot confirm nothing re-parsed')
|
||||
else check(pending.every(l => /0 pending files|no full parses pending/.test(l)),
|
||||
`nothing re-parsed on the warm run: ${pending.map(l => l.replace(/^codeburn: /, '').trim()).join(' | ')}`)
|
||||
|
||||
check(JSON.stringify(shardSnapshot(upgradeCache)) === JSON.stringify(beforeBodies), 'warm run left every shard body unchanged')
|
||||
check(JSON.stringify(stripGenerated(warm.menubar)) === JSON.stringify(stripGenerated(upgraded.menubar)),
|
||||
'warm run reports the same payload as the run that migrated the cache')
|
||||
|
||||
// Republication without a content change is wasted I/O, not a correctness
|
||||
// problem, so it is reported rather than failed. It is real: a date-RANGED
|
||||
// query (`status --format json`, the statusline/menubar fast path) currently
|
||||
// republishes the month shards its range skipped, on every run, even when
|
||||
// nothing changed — the identical-bodies check above is what proves the
|
||||
// content survives it.
|
||||
const afterMtimes = shardMtimes(upgradeCache)
|
||||
const republished = Object.keys(afterMtimes).filter(n => n !== 'envelope.json' && beforeMtimes[n] !== afterMtimes[n])
|
||||
const retired = Object.keys(beforeMtimes).filter(n => n !== 'envelope.json' && !(n in afterMtimes))
|
||||
if (retired.length) console.log(` note ${retired.length} shard(s) republished under a new name with identical content: ${retired.join(', ')}`)
|
||||
else if (republished.length) console.log(` note ${republished.length} shard(s) rewritten in place: ${republished.join(', ')}`)
|
||||
else ok('no shard republished on an unchanged corpus')
|
||||
|
||||
// ── done ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
console.log(`\n${failures ? `FAILED: ${failures} check(s)` : 'PASSED'}${skipped ? ` (${skipped} skipped)` : ''}`)
|
||||
if (failures && process.env['UPGRADE_PATH_KEEP'] !== '0') console.log(`payloads left in: ${PAYLOADS}`)
|
||||
else if (process.env['UPGRADE_PATH_KEEP'] !== '1') { try { rmSync(WORK, { recursive: true, force: true }) } catch { /* windows file locks */ } }
|
||||
process.exit(failures ? 1 : 0)
|
||||
Loading…
Add table
Add a link
Reference in a new issue