From 9de3ca5fc93d77b358403ffae15879380a241f72 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 17 Aug 2026 21:19:24 -0700 Subject: [PATCH] test(upgrade): add high-volume SQLite survivor (#125571) Amp-Thread-ID: https://ampcode.com/threads/T-01a00a6a-b64e-74a5-8b15-2d3b966a468d Co-authored-by: Amp --- docs/help/testing-updates-plugins.md | 23 +- .../e2e/lib/upgrade-survivor/assertions.mjs | 13 +- .../lib/upgrade-survivor/config-recipe.mts | 44 +- scripts/e2e/lib/upgrade-survivor/run.sh | 76 ++- .../lib/upgrade-survivor/sqlite-volume.mjs | 579 ++++++++++++++++++ scripts/e2e/upgrade-survivor-docker.sh | 10 + scripts/lib/docker-e2e-plan.mts | 12 +- test/scripts/docker-build-helper.test.ts | 78 ++- test/scripts/docker-e2e-plan.test.ts | 28 +- .../upgrade-survivor-assertions.test.ts | 1 + .../upgrade-survivor-config-recipe.test.ts | 6 + 11 files changed, 831 insertions(+), 39 deletions(-) create mode 100644 scripts/e2e/lib/upgrade-survivor/sqlite-volume.mjs diff --git a/docs/help/testing-updates-plugins.md b/docs/help/testing-updates-plugins.md index 440cc4f01687..9520bd28a714 100644 --- a/docs/help/testing-updates-plugins.md +++ b/docs/help/testing-updates-plugins.md @@ -136,14 +136,31 @@ pnpm test:docker:published-upgrade-survivor OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC=openclaw@latest \ OPENCLAW_UPGRADE_SURVIVOR_SCENARIO=bootstrap-persona \ pnpm test:docker:published-upgrade-survivor + +OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC=openclaw@2026.7.1-2 \ +OPENCLAW_UPGRADE_SURVIVOR_SCENARIO=sqlite-volume \ +pnpm test:docker:published-upgrade-survivor ``` Available scenarios: `base`, `acpx-openclaw-tools-bridge`, `feishu-channel`, `bootstrap-persona`, `channel-post-core-restore`, `plugin-deps-cleanup`, `configured-plugin-installs`, `stale-source-plugin-shadow`, `tilde-log-path`, -and `versioned-runtime-deps`. In aggregate runs, `OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS=reported-issues` -(alias `far-reaching`) expands to all scenarios, including the -configured-plugin install migration. +`meeting-transcripts-sqlite`, `versioned-runtime-deps`, `cron-scheduled-authority`, +and `sqlite-volume`. In aggregate runs, +`OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS=reported-issues` expands the release-soak +fixtures but excludes the expensive `sqlite-volume` scenario. Use +`OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS=far-reaching` to include it. + +The `sqlite-volume` scenario combines configured Matrix, Discord, and Telegram +plugin/channel state with 4,800 sessions, 23,890 transcript events, and 2,200 +cron crawl jobs by default. It verifies exact JSONL-to-SQLite and cron migration, +legacy archival, database integrity, a second idempotent Doctor run, and Gateway +startup. Scale it with `OPENCLAW_UPGRADE_SURVIVOR_VOLUME_SESSIONS`, +`OPENCLAW_UPGRADE_SURVIVOR_VOLUME_EVENTS_PER_SESSION`, and +`OPENCLAW_UPGRADE_SURVIVOR_VOLUME_CRON_JOBS`. The default Doctor budgets are 120 +seconds for migration and 60 seconds for the idempotent pass; override them with +`OPENCLAW_UPGRADE_SURVIVOR_VOLUME_MIGRATION_BUDGET_SECONDS` and +`OPENCLAW_UPGRADE_SURVIVOR_VOLUME_IDEMPOTENCE_BUDGET_SECONDS` on slower hosts. Full update migration is intentionally separate from Full Release CI. Use the manual `Update Migration` workflow when the release question is "can every diff --git a/scripts/e2e/lib/upgrade-survivor/assertions.mjs b/scripts/e2e/lib/upgrade-survivor/assertions.mjs index efb984795d55..b1b695a2c465 100644 --- a/scripts/e2e/lib/upgrade-survivor/assertions.mjs +++ b/scripts/e2e/lib/upgrade-survivor/assertions.mjs @@ -4,6 +4,7 @@ import fs from "node:fs"; import path from "node:path"; import { DatabaseSync } from "node:sqlite"; import { readPluginInstallIndex } from "../plugin-index-sqlite.mjs"; +import { assertUpgradeVolumeMigrated, seedUpgradeVolume } from "./sqlite-volume.mjs"; const command = process.argv[2]; const SCENARIOS = new Set([ @@ -20,6 +21,7 @@ const SCENARIOS = new Set([ "meeting-transcripts-sqlite", "versioned-runtime-deps", "cron-scheduled-authority", + "sqlite-volume", "auth-profile-v2026-7-2-beta-5", ]); @@ -427,7 +429,7 @@ function assertConfigSurvived() { const pluginAllow = config.plugins?.allow ?? []; assert(pluginAllow.includes("discord"), "discord plugin allow entry missing"); assert(pluginAllow.includes("telegram"), "telegram plugin allow entry missing"); - if (getScenario() === "configured-plugin-installs") { + if (acceptsIntent(coverage, "configured-plugin-installs")) { assert(pluginAllow.includes("matrix"), "matrix plugin allow entry missing"); } else { assert(pluginAllow.includes("whatsapp"), "whatsapp plugin allow entry missing"); @@ -490,7 +492,7 @@ function assertConfigSurvived() { if ( acceptsIntent(coverage, "whatsapp-channel") && - getScenario() !== "configured-plugin-installs" + !acceptsIntent(coverage, "configured-plugin-installs") ) { const whatsapp = config.channels?.whatsapp; assert(whatsapp?.enabled === true, "whatsapp enabled flag changed"); @@ -564,6 +566,9 @@ function assertStateSurvived() { if (scenario === "cron-scheduled-authority") { assertCronScheduledAuthorityMigrated(stateDir, stage); } + if (scenario === "sqlite-volume") { + assertUpgradeVolumeMigrated(stateDir, stage); + } if (scenario === "auth-profile-v2026-7-2-beta-5") { assertAuthProfileMigrationSurvived(stateDir, stage); } @@ -1212,6 +1217,10 @@ if (command === "list-scenarios") { process.stdout.write(`${JSON.stringify([...SCENARIOS])}\n`); } else if (command === "seed") { seedState(); +} else if (command === "seed-volume") { + assert(getScenario() === "sqlite-volume", "seed-volume requires the sqlite-volume scenario"); + const stateDir = requireEnv("OPENCLAW_STATE_DIR"); + seedUpgradeVolume(stateDir); } else if (command === "assert-config") { assertConfigSurvived(); } else if (command === "assert-state") { diff --git a/scripts/e2e/lib/upgrade-survivor/config-recipe.mts b/scripts/e2e/lib/upgrade-survivor/config-recipe.mts index 08a623a87858..a50f30a32712 100644 --- a/scripts/e2e/lib/upgrade-survivor/config-recipe.mts +++ b/scripts/e2e/lib/upgrade-survivor/config-recipe.mts @@ -154,6 +154,26 @@ const representativeConfigSteps: ConfigStep[] = [ ), ]; +const configuredPluginInstallSteps = [ + configSetJsonFile( + "plugins-configured-installs", + "configured-plugin-installs", + "plugins", + "plugins-configured-installs.json", + ), + { + id: "channels-whatsapp-unset", + intent: "configured-plugin-installs", + argv: ["config", "unset", "channels.whatsapp"], + }, + configSetJsonFile( + "channels-matrix", + "configured-plugin-installs", + "channels.matrix", + "channels-matrix.json", + ), +]; + const scenarioConfigSteps = new Map([ [ "acpx-openclaw-tools-bridge", @@ -188,28 +208,8 @@ const scenarioConfigSteps = new Map([ }, ], ], - [ - "configured-plugin-installs", - [ - configSetJsonFile( - "plugins-configured-installs", - "configured-plugin-installs", - "plugins", - "plugins-configured-installs.json", - ), - { - id: "channels-whatsapp-unset", - intent: "configured-plugin-installs", - argv: ["config", "unset", "channels.whatsapp"], - }, - configSetJsonFile( - "channels-matrix", - "configured-plugin-installs", - "channels.matrix", - "channels-matrix.json", - ), - ], - ], + ["configured-plugin-installs", configuredPluginInstallSteps], + ["sqlite-volume", configuredPluginInstallSteps], [ "codex-allowlist-survival", [ diff --git a/scripts/e2e/lib/upgrade-survivor/run.sh b/scripts/e2e/lib/upgrade-survivor/run.sh index 501c889e92e5..e5b9ecfe900e 100644 --- a/scripts/e2e/lib/upgrade-survivor/run.sh +++ b/scripts/e2e/lib/upgrade-survivor/run.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash set -Eeuo pipefail +# Signal traps inherit the foreground command's redirections. Keep harness stdout separate so the +# final summary location cannot corrupt a command artifact when the run is interrupted. +exec 3>&1 source scripts/lib/openclaw-e2e-instance.sh @@ -38,7 +41,7 @@ export TELEGRAM_BOT_TOKEN="123456:upgrade-survivor-telegram-token" if [ "$SCENARIO" = "feishu-channel" ]; then export FEISHU_APP_SECRET="upgrade-survivor-feishu-secret" fi -if [ "$SCENARIO" = "configured-plugin-installs" ]; then +if [ "$SCENARIO" = "configured-plugin-installs" ] || [ "$SCENARIO" = "sqlite-volume" ]; then export MATRIX_ACCESS_TOKEN="upgrade-survivor-matrix-token" export BRAVE_API_KEY="BSA_upgrade_survivor_brave_key" fi @@ -85,6 +88,9 @@ status_seconds="" healthz_seconds="" readyz_seconds="" update_restart_seconds="" +migration_seconds="" +idempotence_seconds="" +run_completed="0" BASELINE_INSTALL_LOG="$ARTIFACT_ROOT/baseline-install.log" UPDATE_JSON="$ARTIFACT_ROOT/update.json" @@ -197,6 +203,8 @@ write_summary() { SUMMARY_UPDATE_RESTART_MODE="$UPDATE_RESTART_MODE" \ SUMMARY_START_SECONDS="$start_seconds" \ SUMMARY_UPDATE_RESTART_SECONDS="$update_restart_seconds" \ + SUMMARY_MIGRATION_SECONDS="$migration_seconds" \ + SUMMARY_IDEMPOTENCE_SECONDS="$idempotence_seconds" \ SUMMARY_HEALTHZ_SECONDS="$healthz_seconds" \ SUMMARY_READYZ_SECONDS="$readyz_seconds" \ SUMMARY_STATUS_SECONDS="$status_seconds" \ @@ -234,6 +242,8 @@ const summary = { timings: { startupSeconds: numberOrNull(process.env.SUMMARY_START_SECONDS), updateRestartSeconds: numberOrNull(process.env.SUMMARY_UPDATE_RESTART_SECONDS), + migrationSeconds: numberOrNull(process.env.SUMMARY_MIGRATION_SECONDS), + idempotenceSeconds: numberOrNull(process.env.SUMMARY_IDEMPOTENCE_SECONDS), healthzSeconds: numberOrNull(process.env.SUMMARY_HEALTHZ_SECONDS), readyzSeconds: numberOrNull(process.env.SUMMARY_READYZ_SECONDS), statusSeconds: numberOrNull(process.env.SUMMARY_STATUS_SECONDS), @@ -281,24 +291,40 @@ on_error() { return "$status" } +on_signal() { + local signal="$1" + local status="$2" + trap - HUP INT TERM + FAILURE_PHASE="${CURRENT_PHASE:-unknown}" + FAILURE_MESSAGE="phase ${FAILURE_PHASE} interrupted by ${signal}" + exit "$status" +} + on_exit() { local status="$1" + trap - ERR EXIT HUP INT TERM set +e cleanup - if [ "$status" -eq 0 ]; then + if [ "$status" -eq 0 ] && [ "$run_completed" = "1" ]; then write_summary passed "" else + if [ "$status" -eq 0 ]; then + status=1 + FAILURE_MESSAGE="upgrade survivor exited before all phases completed" + fi [ -n "$FAILURE_PHASE" ] || FAILURE_PHASE="${CURRENT_PHASE:-unknown}" [ -n "$FAILURE_MESSAGE" ] || FAILURE_MESSAGE="upgrade survivor failed with status $status" write_summary failed "$FAILURE_MESSAGE" fi - echo "Upgrade survivor summary: $SUMMARY_JSON" - cat "$SUMMARY_JSON" 2>/dev/null || true + echo "Upgrade survivor summary: $SUMMARY_JSON" >&3 exit "$status" } trap 'on_error $?' ERR trap 'on_exit $?' EXIT +trap 'on_signal SIGHUP 129' HUP +trap 'on_signal SIGINT 130' INT +trap 'on_signal SIGTERM 143' TERM phase() { local name="$1" @@ -357,7 +383,7 @@ plugin_deps_cleanup_plugin_dirs() { } configured_plugin_installs_enabled() { - [ "$SCENARIO" = "configured-plugin-installs" ] + [ "$SCENARIO" = "configured-plugin-installs" ] || [ "$SCENARIO" = "sqlite-volume" ] } source_only_plugin_shadow_enabled() { @@ -1449,11 +1475,40 @@ assert_root_managed_vps_cli_usable() { } run_doctor() { + local started_at budget + started_at="$(date +%s)" if ! openclaw_e2e_maybe_timeout "$COMMAND_TIMEOUT" openclaw doctor --fix --non-interactive >"$DOCTOR_LOG" 2>&1; then echo "openclaw doctor failed" >&2 openclaw_e2e_print_log "$DOCTOR_LOG" >&2 return 1 fi + if [ "$SCENARIO" = "sqlite-volume" ]; then + migration_seconds=$(($(date +%s) - started_at)) + budget="$(openclaw_e2e_read_positive_int_env OPENCLAW_UPGRADE_SURVIVOR_VOLUME_MIGRATION_BUDGET_SECONDS 120)" + echo "SQLite volume migration doctor completed in ${migration_seconds}s (budget ${budget}s)." + if [ "$migration_seconds" -gt "$budget" ]; then + echo "SQLite volume migration exceeded budget: ${migration_seconds}s > ${budget}s" >&2 + return 1 + fi + fi +} + +assert_volume_idempotence() { + local started_at budget + started_at="$(date +%s)" + if ! openclaw_e2e_maybe_timeout "$COMMAND_TIMEOUT" openclaw doctor --fix --non-interactive >>"$DOCTOR_LOG" 2>&1; then + echo "openclaw idempotence doctor failed" >&2 + openclaw_e2e_print_log "$DOCTOR_LOG" >&2 + return 1 + fi + idempotence_seconds=$(($(date +%s) - started_at)) + budget="$(openclaw_e2e_read_positive_int_env OPENCLAW_UPGRADE_SURVIVOR_VOLUME_IDEMPOTENCE_BUDGET_SECONDS 60)" + echo "SQLite volume idempotence doctor completed in ${idempotence_seconds}s (budget ${budget}s)." + if [ "$idempotence_seconds" -gt "$budget" ]; then + echo "SQLite volume idempotence exceeded budget: ${idempotence_seconds}s > ${budget}s" >&2 + return 1 + fi + node scripts/e2e/lib/upgrade-survivor/assertions.mjs assert-state } validate_post_doctor_config() { @@ -1601,6 +1656,9 @@ phase install-baseline-plugin-dependencies install_baseline_plugin_dependencies phase seed-legacy-plugin-dependency-debris seed_legacy_plugin_dependency_debris phase assert-legacy-plugin-dependency-debris assert_legacy_plugin_dependency_debris_present phase seed-source-only-plugin-shadow seed_source_only_plugin_shadow +if [ "$SCENARIO" = "sqlite-volume" ]; then + phase seed-volume-state node scripts/e2e/lib/upgrade-survivor/assertions.mjs seed-volume +fi phase assert-baseline assert_baseline_state phase seed-legacy-runtime-deps-symlink seed_legacy_runtime_deps_symlink phase resolve-candidate resolve_candidate_version @@ -1611,7 +1669,7 @@ phase update-candidate update_candidate if [ -n "${OPENCLAW_CLAWHUB_URL:-}" ]; then clawhub_security_mode="required" prepublish_package="@openclaw/whatsapp" - if [ "$SCENARIO" = "configured-plugin-installs" ]; then + if configured_plugin_installs_enabled; then prepublish_package="@openclaw/matrix" fi # 2026.6.35 predates the release-security endpoint. The trusted fixture still @@ -1630,6 +1688,9 @@ phase assert-legacy-plugin-dependency-debris-cleaned assert_legacy_plugin_depend phase assert-legacy-runtime-deps-symlink-repaired assert_legacy_runtime_deps_symlink_repaired phase validate-post-doctor-config validate_post_doctor_config phase assert-survival assert_survival +if [ "$SCENARIO" = "sqlite-volume" ]; then + phase assert-volume-idempotence assert_volume_idempotence +fi phase gateway-start ensure_gateway_started phase gateway-probes check_gateway_probes phase gateway-status check_gateway_status @@ -1637,4 +1698,5 @@ if [ "$LIVE_OPENAI" = "1" ]; then phase live-openai run_live_openai fi -echo "Upgrade survivor Docker E2E passed baseline=${baseline_spec} scenario=${SCENARIO} candidate=${candidate_version} updateRestartMode=${UPDATE_RESTART_MODE} startup=${start_seconds}s updateRestart=${update_restart_seconds:-manual}s healthz=${healthz_seconds}s readyz=${readyz_seconds}s status=${status_seconds}s." +run_completed="1" +echo "Upgrade survivor Docker E2E passed baseline=${baseline_spec} scenario=${SCENARIO} candidate=${candidate_version} updateRestartMode=${UPDATE_RESTART_MODE} migration=${migration_seconds:-n/a}s idempotence=${idempotence_seconds:-n/a}s startup=${start_seconds}s updateRestart=${update_restart_seconds:-manual}s healthz=${healthz_seconds}s readyz=${readyz_seconds}s status=${status_seconds}s." diff --git a/scripts/e2e/lib/upgrade-survivor/sqlite-volume.mjs b/scripts/e2e/lib/upgrade-survivor/sqlite-volume.mjs new file mode 100644 index 000000000000..1c4f48ef32f3 --- /dev/null +++ b/scripts/e2e/lib/upgrade-survivor/sqlite-volume.mjs @@ -0,0 +1,579 @@ +import fs from "node:fs"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; + +const VOLUME_AGENT_IDS = ["main", "ops"]; +const VOLUME_CRON_CREATED_AT_MS = Date.parse("2026-07-01T10:00:00.000Z"); +const PREEXISTING_SESSION_FIXTURES = [ + { + agentId: "main", + sessionKey: "agent:main:main", + sessionId: "upgrade-main-session", + }, + { + agentId: "main", + sessionKey: "agent:main:+15551234567", + sessionId: "upgrade-direct-session", + }, + { + agentId: "main", + sessionKey: "agent:main:slack:channel:cupgrade", + sessionId: "upgrade-group-session", + }, +]; + +function assert(condition, message) { + if (!condition) { + throw new Error(message); + } +} + +function assertJsonEqual(actual, expected, message) { + assert(JSON.stringify(actual) === JSON.stringify(expected), message); +} + +function readJson(file) { + return JSON.parse(fs.readFileSync(file, "utf8")); +} + +function write(file, contents) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, contents); +} + +function writeJson(file, value) { + write(file, `${JSON.stringify(value, null, 2)}\n`); +} + +function readPositiveIntegerEnv(name, fallback) { + const raw = process.env[name]?.trim(); + if (!raw) { + return fallback; + } + assert(/^[1-9][0-9]*$/u.test(raw), `${name} must be a positive integer`); + const value = Number(raw); + assert(Number.isSafeInteger(value), `${name} must be a safe positive integer`); + return value; +} + +function getVolumeSpec() { + return { + sessions: readPositiveIntegerEnv("OPENCLAW_UPGRADE_SURVIVOR_VOLUME_SESSIONS", 4800), + eventsPerSession: readPositiveIntegerEnv( + "OPENCLAW_UPGRADE_SURVIVOR_VOLUME_EVENTS_PER_SESSION", + 5, + ), + cronJobs: readPositiveIntegerEnv("OPENCLAW_UPGRADE_SURVIVOR_VOLUME_CRON_JOBS", 2200), + }; +} + +function getVolumeSessionFixture(index) { + const agentId = VOLUME_AGENT_IDS[index % VOLUME_AGENT_IDS.length]; + const paddedIndex = String(index).padStart(6, "0"); + const sessionId = + index === 0 + ? "volume-main-unicode-000000" + : index === 1 + ? "volume-ops-combining-000001" + : index === 2 + ? `volume-main-${"x".repeat(116)}` + : `volume-${agentId}-${paddedIndex}`; + const target = + index === 1 + ? `naïve-user-${paddedIndex}` + : index % 3 === 0 + ? `channel-${paddedIndex}:thread:${index % 97}` + : `user-${paddedIndex}`; + const sessionKey = + index % 3 === 0 + ? `agent:${agentId}:slack:channel:${target}` + : index % 3 === 1 + ? `agent:${agentId}:discord:personal:direct:${target}` + : `agent:${agentId}:telegram:group:-1000000000000:topic:${target}`; + return { + agentId, + label: index % 17 === 0 ? `Volume user ${index} — 東京` : `Volume user ${index}`, + metadataOnly: index === 4 || index % 401 === 400, + missingTranscript: index === 5 || index % 503 === 502, + sessionId, + sessionKey, + }; +} + +function getVolumeSessionsDir(stateDir, agentId) { + return path.join(stateDir, "agents", agentId, "sessions"); +} + +function getVolumeTranscriptEvent(index, sessionId, sequence) { + if (sequence === 0) { + return { + type: "session", + version: 3, + id: sessionId, + timestamp: "2026-07-01T10:00:00.000Z", + cwd: "/tmp/openclaw-upgrade-survivor-workspace", + }; + } + const textSize = sequence % 3 === 1 ? 257 : sequence % 3 === 2 ? 1025 : 32; + return { + type: "message", + id: `${sessionId}-event-${sequence}`, + parentId: sequence === 1 ? null : `${sessionId}-event-${sequence - 1}`, + timestamp: new Date(VOLUME_CRON_CREATED_AT_MS + sequence * 1000).toISOString(), + message: { + role: sequence % 2 === 0 ? "assistant" : "user", + content: [{ type: "text", text: `${index}:${sequence}:λ${"x".repeat(textSize)}` }], + }, + }; +} + +function getVolumeCronJob(index) { + const paddedIndex = String(index).padStart(6, "0"); + return { + id: `volume-cron-${paddedIndex}`, + name: index === 0 ? "Archive crawl — 東京" : `Archive crawl ${paddedIndex}`, + enabled: index % 5 !== 0, + createdAtMs: VOLUME_CRON_CREATED_AT_MS + index, + updatedAtMs: VOLUME_CRON_CREATED_AT_MS + index, + schedule: { + kind: "every", + everyMs: 86_400_000, + anchorMs: VOLUME_CRON_CREATED_AT_MS + index, + }, + sessionTarget: "isolated", + wakeMode: "now", + delivery: { mode: "none" }, + payload: { + kind: "agentTurn", + message: `crawl archive partition ${paddedIndex} ${"z".repeat((index % 5) * 128)}`.trimEnd(), + }, + state: { + nextRunAtMs: VOLUME_CRON_CREATED_AT_MS + 365 * 86_400_000 + index, + ...(index % 11 === 0 ? { lastStatus: "error", lastError: "stale crawl lease" } : {}), + crawlCursor: { partition: paddedIndex, offset: index * 1000 }, + }, + }; +} + +function getVolumeSessionFixtures(spec) { + return Array.from({ length: spec.sessions }, (_, index) => ({ + index, + ...getVolumeSessionFixture(index), + })); +} + +function assertVolumeSessionStores(stores, fixtures, context) { + for (const agentId of VOLUME_AGENT_IDS) { + const expectedCount = [...fixtures, ...PREEXISTING_SESSION_FIXTURES].filter( + (fixture) => fixture.agentId === agentId, + ).length; + assert( + Object.keys(stores.get(agentId) ?? {}).length === expectedCount, + `${agentId} ${context} session-store count changed`, + ); + } + for (const fixture of fixtures) { + const entry = stores.get(fixture.agentId)?.[fixture.sessionKey]; + assert( + entry?.sessionId === fixture.sessionId, + `${context} session row changed: ${fixture.index}`, + ); + assert(entry?.label === fixture.label, `${context} session label changed: ${fixture.index}`); + assert( + fixture.metadataOnly === !Object.hasOwn(entry, "sessionFile"), + `${context} session transcript ownership changed: ${fixture.index}`, + ); + } + for (const fixture of PREEXISTING_SESSION_FIXTURES) { + assert( + stores.get(fixture.agentId)?.[fixture.sessionKey]?.sessionId === fixture.sessionId, + `${context} preexisting session changed: ${fixture.sessionKey}`, + ); + } +} + +function assertVolumeCronJobs(jobs, spec, context) { + assert(jobs.length === spec.cronJobs, `${context} cron fixture count changed`); + for (let index = 0; index < spec.cronJobs; index += 1) { + const expected = getVolumeCronJob(index); + const actual = jobs[index]; + assert(actual?.id === expected.id, `${context} cron identity changed: ${index}`); + assert( + actual?.payload?.message === expected.payload.message, + `${context} cron changed: ${index}`, + ); + } +} + +function seedUpgradeVolumeSessions(stateDir) { + const stores = new Map( + VOLUME_AGENT_IDS.map((agentId) => { + const sessionsDir = getVolumeSessionsDir(stateDir, agentId); + const storePath = path.join(sessionsDir, "sessions.json"); + return [ + agentId, + { + sessionsDir, + store: fs.existsSync(storePath) ? readJson(storePath) : {}, + }, + ]; + }), + ); + const spec = getVolumeSpec(); + const baseUpdatedAt = Date.now() - 12 * 60 * 60 * 1000; + for (let index = 0; index < spec.sessions; index += 1) { + const { agentId, label, metadataOnly, missingTranscript, sessionId, sessionKey } = + getVolumeSessionFixture(index); + const target = stores.get(agentId); + assert(target, `unknown volume fixture agent: ${agentId}`); + const { sessionsDir, store } = target; + store[sessionKey] = { + sessionId, + ...(metadataOnly ? {} : { sessionFile: path.join(sessionsDir, `${sessionId}.jsonl`) }), + provider: "openai", + model: "gpt-5.5", + updatedAt: baseUpdatedAt + index, + label, + }; + if (metadataOnly || missingTranscript) { + continue; + } + const events = Array.from({ length: spec.eventsPerSession }, (_, sequence) => + JSON.stringify(getVolumeTranscriptEvent(index, sessionId, sequence)), + ); + write(path.join(sessionsDir, `${sessionId}.jsonl`), `${events.join("\n")}\n`); + } + for (let index = 0; index < 24; index += 1) { + const agentId = VOLUME_AGENT_IDS[index % VOLUME_AGENT_IDS.length]; + write( + path.join( + getVolumeSessionsDir(stateDir, agentId), + `deleted-orphan-${String(index).padStart(2, "0")}.jsonl`, + ), + `${JSON.stringify({ type: "message", id: `deleted-orphan-${index}` })}\n`, + ); + } + for (const { sessionsDir, store } of stores.values()) { + writeJson(path.join(sessionsDir, "sessions.json"), store); + } +} + +function seedUpgradeVolumeCronJobs(stateDir) { + const spec = getVolumeSpec(); + const jobs = Array.from({ length: spec.cronJobs }, (_, index) => getVolumeCronJob(index)); + writeJson(path.join(stateDir, "cron", "jobs.json"), { version: 1, jobs }); +} + +export function seedUpgradeVolume(stateDir) { + seedUpgradeVolumeSessions(stateDir); + seedUpgradeVolumeCronJobs(stateDir); +} + +function assertHealthySqlite(databasePath, assertContents) { + const db = new DatabaseSync(databasePath, { readOnly: true }); + let contents; + try { + assert( + db.prepare("PRAGMA journal_mode").get()?.journal_mode === "wal", + `${databasePath} is not WAL`, + ); + assert( + db.prepare("PRAGMA integrity_check").get()?.integrity_check === "ok", + `${databasePath} failed integrity_check`, + ); + assert( + db.prepare("PRAGMA foreign_key_check").all().length === 0, + `${databasePath} has FK errors`, + ); + contents = assertContents(db); + } finally { + db.close(); + } + const reopened = new DatabaseSync(databasePath, { readOnly: true }); + try { + assert( + reopened.prepare("PRAGMA integrity_check").get()?.integrity_check === "ok", + `${databasePath} failed reopen`, + ); + } finally { + reopened.close(); + } + return contents; +} + +export function assertUpgradeVolumeMigrated(stateDir, stage) { + const spec = getVolumeSpec(); + const fixtures = getVolumeSessionFixtures(spec); + const legacyCronPath = path.join(stateDir, "cron", "jobs.json"); + if (stage === "baseline") { + const stores = new Map( + VOLUME_AGENT_IDS.map((agentId) => [ + agentId, + readJson(path.join(getVolumeSessionsDir(stateDir, agentId), "sessions.json")), + ]), + ); + assertVolumeSessionStores(stores, fixtures, "volume baseline"); + for (const fixture of fixtures) { + if (fixture.missingTranscript) { + assert( + !fs.existsSync( + path.join( + getVolumeSessionsDir(stateDir, fixture.agentId), + `${fixture.sessionId}.jsonl`, + ), + ), + `volume missing transcript fixture unexpectedly exists: ${fixture.index}`, + ); + } + } + assertVolumeCronJobs(readJson(legacyCronPath).jobs ?? [], spec, "volume baseline"); + return; + } + + for (const agentId of VOLUME_AGENT_IDS) { + assert( + !fs.existsSync(path.join(getVolumeSessionsDir(stateDir, agentId), "sessions.json")), + `${agentId} volume legacy session store remained active`, + ); + } + assert(!fs.existsSync(legacyCronPath), "volume legacy cron store remained active"); + let migratedSessions = 0; + let migratedEvents = 0; + for (const agentId of VOLUME_AGENT_IDS) { + const agentFixtures = fixtures.filter((fixture) => fixture.agentId === agentId); + const expectedEvents = + agentFixtures.filter((fixture) => !fixture.metadataOnly && !fixture.missingTranscript) + .length * spec.eventsPerSession; + const databasePath = path.join(stateDir, "agents", agentId, "agent", "openclaw-agent.sqlite"); + const counts = assertHealthySqlite(databasePath, (db) => { + const sessionRows = db + .prepare( + "SELECT session_key, current_session_id, entry_json FROM session_nodes WHERE current_session_id LIKE 'volume-%'", + ) + .all(); + const windowRows = db + .prepare( + "SELECT session_id, session_key FROM session_windows WHERE session_id LIKE 'volume-%'", + ) + .all(); + const eventRows = db + .prepare( + "SELECT session_id, seq, event_json FROM transcript_events WHERE session_id LIKE 'volume-%'", + ) + .all(); + const sessionsByKey = new Map(sessionRows.map((row) => [row.session_key, row])); + const windowsById = new Map(windowRows.map((row) => [row.session_id, row])); + const missingSessions = agentFixtures + .filter((fixture) => !sessionsByKey.has(fixture.sessionKey)) + .map((fixture) => fixture.index); + assert( + sessionRows.length === agentFixtures.length, + `${agentId} volume session count changed: ${sessionRows.length}; missing indexes: ${missingSessions.join(", ")}`, + ); + assert( + windowRows.length === agentFixtures.length, + `${agentId} volume session window count changed: ${windowRows.length}`, + ); + assert( + eventRows.length === expectedEvents, + `${agentId} volume event count changed: ${eventRows.length}`, + ); + const eventsByIdAndSequence = new Map( + eventRows.map((row) => [`${row.session_id}\0${row.seq}`, row]), + ); + for (const fixture of agentFixtures) { + const row = sessionsByKey.get(fixture.sessionKey); + assert( + row?.current_session_id === fixture.sessionId, + `volume session changed: ${fixture.index}`, + ); + const entry = JSON.parse(row?.entry_json ?? "null"); + assert(entry?.sessionId === fixture.sessionId, `volume entry changed: ${fixture.index}`); + assert(entry?.label === fixture.label, `volume label changed: ${fixture.index}`); + assert( + entry?.provider === "openai" || entry?.delivery?.origin?.provider === "openai", + `volume provider changed: ${fixture.index}`, + ); + assert(entry?.model === "gpt-5.5", `volume model changed: ${fixture.index}`); + assert( + !Object.hasOwn(entry, "sessionFile"), + `volume session retained retired sessionFile metadata: ${fixture.index}`, + ); + assert( + windowsById.get(fixture.sessionId)?.session_key === fixture.sessionKey, + `volume session window changed: ${fixture.index}`, + ); + if (fixture.metadataOnly || fixture.missingTranscript) { + continue; + } + for (let sequence = 0; sequence < spec.eventsPerSession; sequence += 1) { + const event = eventsByIdAndSequence.get(`${fixture.sessionId}\0${sequence}`); + const expected = getVolumeTranscriptEvent(fixture.index, fixture.sessionId, sequence); + assertJsonEqual( + JSON.parse(event?.event_json ?? "null"), + expected, + `volume transcript event changed: ${fixture.index}:${sequence}`, + ); + } + } + return { sessions: sessionRows.length, events: eventRows.length }; + }); + migratedSessions += counts.sessions; + migratedEvents += counts.events; + } + assert(migratedSessions === spec.sessions, `volume session count changed: ${migratedSessions}`); + + const stateDatabasePath = path.join(stateDir, "state", "openclaw.sqlite"); + assertHealthySqlite(stateDatabasePath, (db) => { + const rows = db + .prepare( + `SELECT job_id, job_json, state_json, enabled, schedule_kind, every_ms, anchor_ms, + payload_kind, payload_message, delivery_mode, next_run_at_ms, running_at_ms, + last_run_status, last_error, updated_at, runtime_updated_at_ms + FROM cron_jobs + WHERE job_id LIKE 'volume-cron-%'`, + ) + .all(); + assert(rows.length === spec.cronJobs, `volume cron job count changed: ${rows.length}`); + const jobsById = new Map(rows.map((row) => [row.job_id, row])); + for (let index = 0; index < spec.cronJobs; index += 1) { + const expected = getVolumeCronJob(index); + const row = jobsById.get(expected.id); + const actual = JSON.parse(row?.job_json ?? "null"); + for (const field of [ + "id", + "name", + "enabled", + "createdAtMs", + "schedule", + "sessionTarget", + "wakeMode", + "delivery", + "payload", + ]) { + assertJsonEqual(actual?.[field], expected[field], `volume cron ${field} changed: ${index}`); + } + assert(row?.updated_at === expected.updatedAtMs, `volume cron timestamp changed: ${index}`); + assert( + row?.runtime_updated_at_ms === expected.updatedAtMs, + `volume cron runtime timestamp changed: ${index}`, + ); + const actualState = JSON.parse(row?.state_json ?? "null"); + assertJsonEqual( + actualState?.crawlCursor, + expected.state.crawlCursor, + `volume cron residual state changed: ${index}`, + ); + assert( + row?.enabled === (expected.enabled ? 1 : 0), + `volume cron enabled column changed: ${index}`, + ); + assert(row?.schedule_kind === "every", `volume cron schedule column changed: ${index}`); + assert(row?.every_ms === expected.schedule.everyMs, `volume cron interval changed: ${index}`); + assert(row?.anchor_ms === expected.schedule.anchorMs, `volume cron anchor changed: ${index}`); + assert(row?.payload_kind === "agentTurn", `volume cron payload kind changed: ${index}`); + assert( + row?.payload_message === expected.payload.message, + `volume cron payload changed: ${index}`, + ); + assert(row?.delivery_mode === "none", `volume cron delivery mode changed: ${index}`); + assert( + row?.next_run_at_ms === (expected.enabled ? expected.state.nextRunAtMs : null), + `volume cron next-run state changed: ${index}`, + ); + assert(row?.running_at_ms === null, `volume cron running state changed: ${index}`); + assert( + row?.last_run_status === (expected.state.lastStatus ?? null), + `volume cron status state changed: ${index}`, + ); + assert( + row?.last_error === (expected.state.lastError ?? null), + `volume cron error state changed: ${index}`, + ); + } + }); + + const archivedStores = new Map(); + const archivedTranscripts = new Map(); + for (const agentId of VOLUME_AGENT_IDS) { + const archiveDir = path.join(stateDir, "agents", agentId, "session-sqlite-import-archive"); + assert(fs.existsSync(archiveDir), `${agentId} volume session migration archive missing`); + const entries = fs.readdirSync(archiveDir); + const storeEntries = entries.filter((entry) => + /\.sessions\.json\.imported-\d+(?:\.\d+)?$/u.test(entry), + ); + assert(storeEntries.length === 1, `${agentId} volume legacy session-store archive changed`); + archivedStores.set(agentId, readJson(path.join(archiveDir, storeEntries[0]))); + + const transcriptsByName = new Map(); + for (const entry of entries) { + const match = /\.([^.]+\.jsonl)\.imported-\d+(?:\.\d+)?$/u.exec(entry); + if (match?.[1]) { + transcriptsByName.set(match[1], entry); + } + } + const expectedTranscriptCount = + fixtures.filter( + (fixture) => + fixture.agentId === agentId && !fixture.metadataOnly && !fixture.missingTranscript, + ).length + + PREEXISTING_SESSION_FIXTURES.filter((fixture) => fixture.agentId === agentId).length + + 12; + assert( + transcriptsByName.size === expectedTranscriptCount, + `${agentId} volume transcript archive count changed: ${transcriptsByName.size}`, + ); + archivedTranscripts.set(agentId, { archiveDir, transcriptsByName }); + } + assertVolumeSessionStores(archivedStores, fixtures, "archived volume"); + for (const fixture of fixtures) { + if (fixture.metadataOnly || fixture.missingTranscript) { + continue; + } + assert( + archivedTranscripts.get(fixture.agentId)?.transcriptsByName.has(`${fixture.sessionId}.jsonl`), + `referenced volume transcript was not archived: ${fixture.index}`, + ); + } + for (const fixture of PREEXISTING_SESSION_FIXTURES) { + assert( + archivedTranscripts.get(fixture.agentId)?.transcriptsByName.has(`${fixture.sessionId}.jsonl`), + `preexisting transcript was not archived: ${fixture.sessionId}`, + ); + } + for (let index = 0; index < 24; index += 1) { + const orphan = `deleted-orphan-${String(index).padStart(2, "0")}.jsonl`; + const agentId = VOLUME_AGENT_IDS[index % VOLUME_AGENT_IDS.length]; + assert( + archivedTranscripts.get(agentId)?.transcriptsByName.has(orphan), + `unreferenced volume transcript was not archived: ${orphan}`, + ); + } + for (const index of [0, 1, 2]) { + const fixture = getVolumeSessionFixture(index); + const archived = archivedTranscripts.get(fixture.agentId); + const entry = archived?.transcriptsByName.get(`${fixture.sessionId}.jsonl`); + assert(archived && entry, `archived volume transcript sample missing: ${index}`); + const events = fs + .readFileSync(path.join(archived.archiveDir, entry), "utf8") + .trimEnd() + .split("\n") + .map((line) => JSON.parse(line)); + const expected = Array.from({ length: spec.eventsPerSession }, (_, sequence) => + getVolumeTranscriptEvent(index, fixture.sessionId, sequence), + ); + assertJsonEqual(events, expected, `archived volume transcript changed: ${index}`); + } + + const cronArchiveEntries = fs + .readdirSync(path.dirname(legacyCronPath)) + .filter((entry) => /^jobs\.json\.migrated(?:\.\d+)?$/u.test(entry)); + assert(cronArchiveEntries.length === 1, "volume legacy cron archive count changed"); + const archivedCronJobs = readJson( + path.join(path.dirname(legacyCronPath), cronArchiveEntries[0]), + ).jobs; + assertVolumeCronJobs(archivedCronJobs ?? [], spec, "archived volume"); + process.stdout.write( + `sqlite-volume sessions=${migratedSessions} events=${migratedEvents} cronJobs=${spec.cronJobs}\n`, + ); +} diff --git a/scripts/e2e/upgrade-survivor-docker.sh b/scripts/e2e/upgrade-survivor-docker.sh index ec33a5f04bed..96d8d511220f 100755 --- a/scripts/e2e/upgrade-survivor-docker.sh +++ b/scripts/e2e/upgrade-survivor-docker.sh @@ -58,6 +58,11 @@ case "$LIVE_OPENAI" in ;; esac +if [ "$SCENARIO" = "sqlite-volume" ] && [ "${OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE:-0}" != "1" ]; then + echo "sqlite-volume requires OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE=1" >&2 + exit 1 +fi + resolve_lane_artifact_suffix() { if [ -n "${OPENCLAW_DOCKER_ALL_LANE_NAME:-}" ]; then printf "%s" "$OPENCLAW_DOCKER_ALL_LANE_NAME" @@ -231,6 +236,11 @@ if [ "${OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE:-0}" = "1" ]; then -e OPENCLAW_UPGRADE_SURVIVOR_SCENARIO="$SCENARIO" \ -e OPENCLAW_UPGRADE_SURVIVOR_UPDATE_RESTART_MODE="$UPDATE_RESTART_MODE" \ -e OPENCLAW_UPGRADE_SURVIVOR_COMMAND_TIMEOUT="$COMMAND_TIMEOUT" \ + -e OPENCLAW_UPGRADE_SURVIVOR_VOLUME_SESSIONS="${OPENCLAW_UPGRADE_SURVIVOR_VOLUME_SESSIONS:-}" \ + -e OPENCLAW_UPGRADE_SURVIVOR_VOLUME_EVENTS_PER_SESSION="${OPENCLAW_UPGRADE_SURVIVOR_VOLUME_EVENTS_PER_SESSION:-}" \ + -e OPENCLAW_UPGRADE_SURVIVOR_VOLUME_CRON_JOBS="${OPENCLAW_UPGRADE_SURVIVOR_VOLUME_CRON_JOBS:-}" \ + -e OPENCLAW_UPGRADE_SURVIVOR_VOLUME_MIGRATION_BUDGET_SECONDS="${OPENCLAW_UPGRADE_SURVIVOR_VOLUME_MIGRATION_BUDGET_SECONDS:-120}" \ + -e OPENCLAW_UPGRADE_SURVIVOR_VOLUME_IDEMPOTENCE_BUDGET_SECONDS="${OPENCLAW_UPGRADE_SURVIVOR_VOLUME_IDEMPOTENCE_BUDGET_SECONDS:-60}" \ -e OPENCLAW_UPGRADE_SURVIVOR_LEGACY_RUNTIME_DEPS_SYMLINK="${OPENCLAW_UPGRADE_SURVIVOR_LEGACY_RUNTIME_DEPS_SYMLINK:-}" \ -e OPENCLAW_UPGRADE_SURVIVOR_ROOT_MANAGED_VPS="$ROOT_MANAGED_VPS" \ -e OPENCLAW_UPGRADE_SURVIVOR_TSX_IMPORT=/tmp/openclaw-release-harness/node_modules/tsx/dist/loader.mjs \ diff --git a/scripts/lib/docker-e2e-plan.mts b/scripts/lib/docker-e2e-plan.mts index 73ffa0282104..75afe2b469bc 100644 --- a/scripts/lib/docker-e2e-plan.mts +++ b/scripts/lib/docker-e2e-plan.mts @@ -117,10 +117,14 @@ const UPGRADE_SURVIVOR_SCENARIOS = [ "meeting-transcripts-sqlite", "versioned-runtime-deps", "cron-scheduled-authority", + "sqlite-volume", ]; const UPGRADE_SURVIVOR_SCENARIO_ALIASES = new Map([ - ["reported-issues", UPGRADE_SURVIVOR_SCENARIOS], + [ + "reported-issues", + UPGRADE_SURVIVOR_SCENARIOS.filter((scenario) => scenario !== "sqlite-volume"), + ], ["far-reaching", UPGRADE_SURVIVOR_SCENARIOS], ]); @@ -131,6 +135,10 @@ const UPGRADE_SURVIVOR_RUNTIME_COMPANION_PACKAGES = ["@openclaw/codex"]; // Pre-protocol catalogs are content-addressed. Unknown legacy blocks fail // closed instead of requiring a dependency or reimplementing a JavaScript parser. const LEGACY_UPGRADE_SURVIVOR_SCENARIO_CATALOGS = new Map([ + [ + "0c5d3ce3533c035033890923aae7e210f4fdb24e7b8af32371930cdf12a00fd5", + "base acpx-openclaw-tools-bridge feishu-channel bootstrap-persona channel-post-core-restore codex-allowlist-survival plugin-deps-cleanup configured-plugin-installs stale-source-plugin-shadow tilde-log-path meeting-transcripts-sqlite versioned-runtime-deps cron-scheduled-authority sqlite-volume auth-profile-v2026-7-2-beta-5", + ], [ "837ab1c89821d52519f385e0f3d2067e0b923f730e3a4791e67f578bf5d29f8e", "base acpx-openclaw-tools-bridge feishu-channel bootstrap-persona channel-post-core-restore codex-allowlist-survival plugin-deps-cleanup configured-plugin-installs stale-source-plugin-shadow tilde-log-path meeting-transcripts-sqlite versioned-runtime-deps cron-scheduled-authority auth-profile-v2026-7-2-beta-5", @@ -312,7 +320,7 @@ function normalizeUpgradeSurvivorScenario(raw: string | undefined): string | und throw new Error( `invalid published upgrade survivor scenario: ${JSON.stringify( value, - )}. Expected one of: ${UPGRADE_SURVIVOR_SCENARIOS.join(", ")}, reported-issues.`, + )}. Expected one of: ${UPGRADE_SURVIVOR_SCENARIOS.join(", ")}, reported-issues, or far-reaching.`, ); } return value; diff --git a/test/scripts/docker-build-helper.test.ts b/test/scripts/docker-build-helper.test.ts index 912451dee066..242b0a0a0883 100644 --- a/test/scripts/docker-build-helper.test.ts +++ b/test/scripts/docker-build-helper.test.ts @@ -2460,7 +2460,7 @@ docker_e2e_docker_run_cmd run demo ); expect(publishedRunner).toContain('if [ "$candidate_version" = "2026.6.35" ]; then'); expect(publishedRunner).toContain('prepublish_package="@openclaw/whatsapp"'); - expect(publishedRunner).toContain('if [ "$SCENARIO" = "configured-plugin-installs" ]; then'); + expect(publishedRunner).toContain("if configured_plugin_installs_enabled; then"); expect(publishedRunner).toContain('prepublish_package="@openclaw/matrix"'); expect(publishedRunner).toContain( 'assert-prepublish-requests "$OPENCLAW_CLAWHUB_URL" "$prepublish_package" "$candidate_version"', @@ -2493,7 +2493,7 @@ docker_e2e_docker_run_cmd run demo expect(publishedRunner).toContain('if [ "$SCENARIO" = "feishu-channel" ]; then'); expect(publishedRunner).toContain( [ - 'if [ "$SCENARIO" = "configured-plugin-installs" ]; then', + 'if [ "$SCENARIO" = "configured-plugin-installs" ] || [ "$SCENARIO" = "sqlite-volume" ]; then', ' export MATRIX_ACCESS_TOKEN="upgrade-survivor-matrix-token"', ' export BRAVE_API_KEY="BSA_upgrade_survivor_brave_key"', "fi", @@ -2789,6 +2789,80 @@ docker_e2e_docker_run_cmd run demo } }); + it("records an interrupted upgrade survivor phase as failed", async () => { + const workDir = tempDirs.make("openclaw-upgrade-survivor-signal-"); + const binDir = join(workDir, "bin"); + const markerPath = join(workDir, "npm-started"); + const summaryPath = join(workDir, "artifacts", "summary.json"); + writeExecutables(binDir, { + npm: `#!/bin/sh +touch "$FAKE_NPM_MARKER" +exec sleep 300 +`, + timeout: `#!/bin/sh +while [ "\${1#--}" != "$1" ]; do shift; done +shift +exec "$@" +`, + }); + + const child = spawn("bash", [UPGRADE_SURVIVOR_RUN_SCRIPT], { + detached: true, + env: { + ...process.env, + FAKE_NPM_MARKER: markerPath, + OPENCLAW_TEST_STATE_FUNCTION_B64: Buffer.from( + "openclaw_test_state_create() { :; }", + ).toString("base64"), + OPENCLAW_UPGRADE_SURVIVOR_BASELINE: "openclaw@2026.7.1-2", + OPENCLAW_UPGRADE_SURVIVOR_CANDIDATE_SPEC: join(workDir, "unused.tgz"), + OPENCLAW_UPGRADE_SURVIVOR_RUNTIME_ROOT: join(workDir, "runtime"), + OPENCLAW_UPGRADE_SURVIVOR_STATE_HOME_ROOT: join(workDir, "state-home"), + OPENCLAW_UPGRADE_SURVIVOR_SUMMARY_JSON: summaryPath, + PATH: `${binDir}:${process.env.PATH ?? ""}`, + }, + stdio: "ignore", + }); + const childPid = child.pid; + if (!childPid) { + throw new Error("upgrade survivor process did not start"); + } + const exitPromise = new Promise<{ + code: number | null; + signal: NodeJS.Signals | null; + }>((resolve) => child.once("exit", (code, signal) => resolve({ code, signal }))); + + try { + for (let attempt = 0; attempt < 500 && !existsSync(markerPath); attempt += 1) { + await delay(10); + } + expect(existsSync(markerPath)).toBe(true); + process.kill(-childPid, "SIGTERM"); + const exit = await exitPromise; + + expect(exit).toEqual({ code: 143, signal: null }); + const summary = JSON.parse(readFileSync(summaryPath, "utf8")); + expect(summary).toMatchObject({ + failure: { + message: "phase install-baseline interrupted by SIGTERM", + phase: "install-baseline", + }, + status: "failed", + }); + expect(summary.phases.at(-1)).toMatchObject({ + phase: "install-baseline", + status: "started", + }); + expect( + readFileSync(join(workDir, "artifacts", "baseline-install.log"), "utf8"), + ).not.toContain("Upgrade survivor summary:"); + } finally { + if (child.exitCode === null && child.signalCode === null) { + process.kill(-childPid, "SIGKILL"); + } + } + }); + it("keeps multi-node update Docker artifacts isolated by default", () => { const multiNode = readFileSync(MULTI_NODE_UPDATE_DOCKER_E2E_PATH, "utf8"); expect(multiNode).toContain( diff --git a/test/scripts/docker-e2e-plan.test.ts b/test/scripts/docker-e2e-plan.test.ts index c4c1be52c577..8c0c1a1f5957 100644 --- a/test/scripts/docker-e2e-plan.test.ts +++ b/test/scripts/docker-e2e-plan.test.ts @@ -956,7 +956,7 @@ describe("scripts/lib/docker-e2e-plan", () => { const plan = planFor({ selectedLaneNames: ["published-upgrade-survivor"], upgradeSurvivorBaselines: "2026.4.29 2026.4.23", - upgradeSurvivorScenarios: "base feishu-channel tilde-log-path", + upgradeSurvivorScenarios: "base feishu-channel tilde-log-path sqlite-volume", }); expect(plan.lanes.map(summarizeLane)).toEqual([ @@ -975,6 +975,11 @@ describe("scripts/lib/docker-e2e-plan", () => { "openclaw@2026.4.29", "tilde-log-path", ), + publishedUpgradeSurvivorLane( + "published-upgrade-survivor-2026.4.29-sqlite-volume", + "openclaw@2026.4.29", + "sqlite-volume", + ), publishedUpgradeSurvivorLane( "published-upgrade-survivor-2026.4.23", "openclaw@2026.4.23", @@ -990,6 +995,11 @@ describe("scripts/lib/docker-e2e-plan", () => { "openclaw@2026.4.23", "tilde-log-path", ), + publishedUpgradeSurvivorLane( + "published-upgrade-survivor-2026.4.23-sqlite-volume", + "openclaw@2026.4.23", + "sqlite-volume", + ), ]); }); @@ -1016,6 +1026,22 @@ describe("scripts/lib/docker-e2e-plan", () => { ]); }); + it("keeps SQLite volume stress out of release soak and in far-reaching runs", () => { + const scenariosFor = (upgradeSurvivorScenarios: string) => + planFor({ + selectedLaneNames: ["published-upgrade-survivor"], + upgradeSurvivorBaselines: "2026.7.1-2", + upgradeSurvivorScenarios, + }).lanes.map((lane) => lane.name); + + expect(scenariosFor("reported-issues")).not.toContain( + "published-upgrade-survivor-2026.7.1-2-sqlite-volume", + ); + expect(scenariosFor("far-reaching")).toContain( + "published-upgrade-survivor-2026.7.1-2-sqlite-volume", + ); + }); + it("omits trusted-current scenarios unsupported by a frozen target harness", () => { const targetRoot = tempDirs.make("openclaw-frozen-upgrade-harness-"); writeFrozenScenarioContract(targetRoot, [ diff --git a/test/scripts/upgrade-survivor-assertions.test.ts b/test/scripts/upgrade-survivor-assertions.test.ts index b4d2b2a0e4c9..4dfa6e7e76f5 100644 --- a/test/scripts/upgrade-survivor-assertions.test.ts +++ b/test/scripts/upgrade-survivor-assertions.test.ts @@ -367,6 +367,7 @@ describe("upgrade survivor assertions", () => { expect(scenarios).toContain("base"); expect(scenarios).toContain("acpx-openclaw-tools-bridge"); + expect(scenarios).toContain("sqlite-volume"); expect(new Set(scenarios).size).toBe(scenarios.length); }); diff --git a/test/scripts/upgrade-survivor-config-recipe.test.ts b/test/scripts/upgrade-survivor-config-recipe.test.ts index 7a0a1ae03f9d..ddb6532a26d9 100644 --- a/test/scripts/upgrade-survivor-config-recipe.test.ts +++ b/test/scripts/upgrade-survivor-config-recipe.test.ts @@ -153,6 +153,12 @@ describe("upgrade survivor config recipe command resolution", () => { expect(steps.at(-1)?.id).toBe("validate"); }); + it("composes configured plugin installs into the SQLite volume scenario", () => { + expect(resolveScenarioConfigSteps("sqlite-volume")).toEqual( + resolveScenarioConfigSteps("configured-plugin-installs"), + ); + }); + it("removes unsupported scenario config for older baselines", () => { const steps = resolveUpgradeSurvivorConfigStepsForBaseline("feishu-channel", "2026.3.13"); expect(steps.find((step) => step.id === "channels-discord")).toBeDefined();