From 73ace5dcefe03358f2e05727d2fc36e0eae0ba32 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 15 Sep 2026 19:48:58 -0700 Subject: [PATCH] fix(state): keep legacy catalog repair in Doctor (#148834) * perf(state): preserve prepared queries during healthy reads Inspect the legacy Workshop index without toggling SQLite catalog flags on healthy handles. Retain the existing bounded repair admission when malformed schema blocks ordinary reads. A native-authorizer regression covers prepared-query reuse. * fix(state): keep legacy catalog repair in Doctor * refactor(state): use native repair connection options * test(doctor): verify maintenance-only catalog admission * fix(doctor): admit update ledger reads before service stop Use Doctor's bounded read admission for pre-stop ledger inspection and its live rechecks on fresh private snapshots. Reset SQLite's schema cache when ending admission so unknown malformed indexes remain refused. Keep legacy catalog repair in Doctor and preserve updater ownership, future-version refusal, and the existing continuation writer fence. * fix(e2e): register Workshop recovery assertions Allow the existing Workshop Doctor recovery scenario through shared successful-update validation so it can reach the fresh candidate repair stage. Extend the real assertion CLI regression while preserving failed-step checks. * fix(e2e): recognize the Workshop frozen catalog * fix(doctor): repair legacy catalogs before migration reads Keep the prepared Doctor repair inside the migration transaction, ahead of publication-ledger reads. This lets standalone Doctor repair handle the v16 Workshop catalog while regular Gateway reads continue to refuse it. --- .../integrity-and-recovery.md | 15 + .../e2e/lib/upgrade-survivor/assertions.mjs | 1 + .../e2e/lib/upgrade-survivor/diagnostics.mjs | 17 +- scripts/e2e/lib/upgrade-survivor/run.sh | 61 ++- .../workshop-doctor-recovery.mjs | 422 ++++++++++++++++++ scripts/e2e/upgrade-survivor-docker.sh | 42 +- scripts/lib/docker-e2e-plan.mts | 11 +- scripts/lib/upgrade-survivor-policy.mjs | 11 +- src/cli/run-main.exit.test.ts | 15 +- src/cli/run-main.ts | 3 +- ...or-maintenance.finish-revalidation.test.ts | 166 ++++++- src/commands/doctor-maintenance.ts | 5 +- src/commands/doctor-update-schema-guard.ts | 12 +- ...tor-health.dangling-workshop-index.test.ts | 163 +++++++ .../doctor-health.migration-refusal.test.ts | 17 +- src/flows/doctor-health.ts | 17 +- src/infra/update-run-reader.ts | 7 +- src/state/openclaw-agent-db-lease.ts | 9 +- ...-preflight.dangling-workshop-index.test.ts | 10 +- src/state/openclaw-database-preflight.ts | 27 +- src/state/openclaw-state-db-contract.ts | 2 + ...enclaw-state-db-dangling-workshop-index.ts | 107 ----- src/state/openclaw-state-db-doctor-schema.ts | 93 ++++ src/state/openclaw-state-db-maintenance.ts | 5 +- src/state/openclaw-state-db-open.ts | 8 - .../openclaw-state-db-prepared-reads.test.ts | 33 ++ src/state/openclaw-state-db-readonly.test.ts | 10 +- src/state/openclaw-state-db-readonly.ts | 82 ++-- src/state/openclaw-state-db-repair.ts | 250 +++++++++++ ...claw-state-db-schema-migration-required.ts | 21 + src/state/openclaw-state-db-schema-version.ts | 27 +- src/state/openclaw-state-db.test.ts | 112 +++-- src/state/openclaw-state-db.ts | 220 ++------- src/state/openclaw-state-ownership.ts | 59 ++- test/scripts/docker-e2e-plan.test.ts | 23 + .../upgrade-survivor-assertions.test.ts | 47 +- .../upgrade-survivor-migration-order.test.ts | 278 +++++++++++- .../upgrade-survivor-plugin-registry.test.ts | 18 + .../vitest.database-worker-core-paths.mjs | 1 + 39 files changed, 1931 insertions(+), 496 deletions(-) create mode 100644 scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs create mode 100644 src/flows/doctor-health.dangling-workshop-index.test.ts delete mode 100644 src/state/openclaw-state-db-dangling-workshop-index.ts create mode 100644 src/state/openclaw-state-db-doctor-schema.ts create mode 100644 src/state/openclaw-state-db-prepared-reads.test.ts create mode 100644 src/state/openclaw-state-db-repair.ts diff --git a/docs/reference/database-schemas/integrity-and-recovery.md b/docs/reference/database-schemas/integrity-and-recovery.md index bda09f9c0d23..4618b0cf5f83 100644 --- a/docs/reference/database-schemas/integrity-and-recovery.md +++ b/docs/reference/database-schemas/integrity-and-recovery.md @@ -126,6 +126,21 @@ The heartbeat proves ownership, not migration progress. A live but stuck mainten `SQLite read-only worker` failures append `code` and numeric SQLite `errcode` diagnostics when the underlying error supplies valid values, including through a bounded cause chain. Report the full code suffix when investigating a failure. Snapshot and integrity-child timeout errors include the applied budget and source file size; snapshot timeouts report an unknown size if the source stat failed. Integrity-child timeouts also retain `lastObservedPhase`. A generic `disk I/O error` or `SQLITE_IOERR` alone does not prove the disk is full. +### A legacy Workshop index prevents shared-state reads + +The `legacy-workshop-review-index` error requires `openclaw doctor --fix`. +Ordinary Gateway reads and automatic migration do not enter the legacy catalog +repair path. Healthy reads retain their prepared SQLite queries. + +With OpenClaw 2026.9.4, run Doctor before retrying `openclaw update`: the installed +updater checks database integrity before it can launch the target version. + +Doctor checks database versions and active owners before repairing the exact +known index. It restores catalog readability before loading dependent config +and plugin state, then continues its normal migration and verification flow. +The readability repair preserves review rows and schema-version markers; +unrecognized damage and newer databases remain refused. + ### The shared-state WAL keeps growing The running Gateway records the result of its existing WAL maintenance pass, diff --git a/scripts/e2e/lib/upgrade-survivor/assertions.mjs b/scripts/e2e/lib/upgrade-survivor/assertions.mjs index 552b37cfa097..68380cdea92c 100644 --- a/scripts/e2e/lib/upgrade-survivor/assertions.mjs +++ b/scripts/e2e/lib/upgrade-survivor/assertions.mjs @@ -31,6 +31,7 @@ const SCENARIOS = new Set([ "msteams-polls", "abandoned-update", "legacy-operator-state", + "workshop-doctor-recovery", "mobile-pairing-reconnect", "acpx-openclaw-tools-bridge", "feishu-channel", diff --git a/scripts/e2e/lib/upgrade-survivor/diagnostics.mjs b/scripts/e2e/lib/upgrade-survivor/diagnostics.mjs index 452a0543120d..499d6a78007e 100644 --- a/scripts/e2e/lib/upgrade-survivor/diagnostics.mjs +++ b/scripts/e2e/lib/upgrade-survivor/diagnostics.mjs @@ -25,6 +25,11 @@ const logNames = [ "post-update-validate.err", "doctor.log", "baseline-doctor.log", + "workshop-doctor-recovery.json", + "workshop-published-refusal.json", + "workshop-baseline-doctor.json", + "workshop-recovered-upgrade.json", + "workshop-candidate-doctor.json", "gateway.log", "gateway.log.doctor", "baseline-service-install.err", @@ -912,10 +917,14 @@ function publishedSuccessSummary(artifactRoot, sanitize) { return { phase: sanitize(event.phase, "phase"), status: event.status, at: event.at }; }), logs: Object.fromEntries( - ["update.json", "repair.json", "recovery-update.json"].map((name) => [ - name, - sanitize(readOwned(artifactRoot, name, name), name), - ]), + [ + "update.json", + "repair.json", + "recovery-update.json", + ...(snapshot.scenario === "workshop-doctor-recovery" + ? ["workshop-doctor-recovery.json", "baseline-doctor.log", "doctor.log"] + : []), + ].map((name) => [name, sanitize(readOwned(artifactRoot, name, name), name)]), ), omissions, }; diff --git a/scripts/e2e/lib/upgrade-survivor/run.sh b/scripts/e2e/lib/upgrade-survivor/run.sh index 7737ee72ac09..51e2cc5cf6f5 100644 --- a/scripts/e2e/lib/upgrade-survivor/run.sh +++ b/scripts/e2e/lib/upgrade-survivor/run.sh @@ -133,6 +133,7 @@ update_restart_source="" update_repair_required="0" initial_update_observation_root="" last_update_observation_root="" +workshop_doctor_observation_root="" idempotence_seconds="" run_completed="0" update_outcome="" @@ -247,6 +248,13 @@ validate_update_restart_mode() { return 1 ;; esac + if [ "$SCENARIO" = "workshop-doctor-recovery" ] && { + [ "$LIVE_OPENAI" != "0" ] || [ "$ROOT_MANAGED_VPS" != "0" ] || + [ "$UPDATE_RESTART_MODE" != "manual" ] || [ "$CANDIDATE_KIND" != "tarball" ]; + }; then + echo "workshop-doctor-recovery requires a candidate tarball, manual restart, and no live provider or managed VPS" >&2 + return 1 + fi } json_event() { @@ -344,6 +352,9 @@ const summary = { updateRecovery: process.env.SUMMARY_UPDATE_REPAIR_REQUIRED === "1" ? "capability-consent" : null, updateRestartSource: process.env.SUMMARY_UPDATE_RESTART_SOURCE || null, firstHopPostCore, + workshopDoctorRecovery: process.env.SUMMARY_SCENARIO === "workshop-doctor-recovery" + ? readJsonOrNull(path.join(path.dirname(process.env.SUMMARY_JSON), "workshop-doctor-recovery.json")) + : undefined, restartFixture: readJsonOrNull(process.env.SUMMARY_RESTART_FIXTURE), restartRuntimeFixture: readJsonOrNull(process.env.SUMMARY_RESTART_RUNTIME_FIXTURE), restartInference: process.env.SUMMARY_RESTART_INFERENCE || null, @@ -1408,9 +1419,14 @@ update_candidate() { if [ "$ROOT_MANAGED_VPS" != "1" ]; then update_env+=(OPENCLAW_ALLOW_ROOT=1) fi + local update_node_options="${NODE_OPTIONS:+$NODE_OPTIONS }--import=$PWD/scripts/e2e/lib/upgrade-survivor/diagnostics.mjs" + if [ "$SCENARIO" = "workshop-doctor-recovery" ]; then + update_node_options+=" --import=$PWD/scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs" + update_env+=("OPENCLAW_UPGRADE_SURVIVOR_WORKSHOP_STATE_DIR=$OPENCLAW_STATE_DIR") + fi update_env+=( "OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_ROOT=$observation_root" - "NODE_OPTIONS=${NODE_OPTIONS:+$NODE_OPTIONS }--import=$PWD/scripts/e2e/lib/upgrade-survivor/diagnostics.mjs" + "NODE_OPTIONS=$update_node_options" ) local update_status=0 if [ "$SCENARIO" = "recovery-cleanup" ]; then @@ -1466,6 +1482,25 @@ update_candidate() { fi } +assert_workshop_published_refusal() { + local refusal_exit=0 + update_candidate || refusal_exit=$? + node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs refusal \ + "$initial_update_observation_root" "$(package_root)" "$refusal_exit" || return "$?" + update_outcome="refused-before-candidate" +} + +run_workshop_doctor() { + local stage="$1" log="$2" + workshop_doctor_observation_root="$(mktemp -d "$ARTIFACT_ROOT/workshop-$stage-doctor.XXXXXX")" + local doctor_node_options="${NODE_OPTIONS:+$NODE_OPTIONS }--import=$PWD/scripts/e2e/lib/upgrade-survivor/diagnostics.mjs --import=$PWD/scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs" + openclaw_e2e_maybe_timeout "$COMMAND_TIMEOUT" env -u OPENCLAW_UPDATE_IN_PROGRESS \ + "OPENCLAW_UPGRADE_SURVIVOR_WORKSHOP_STATE_DIR=$OPENCLAW_STATE_DIR" \ + "OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_ROOT=$workshop_doctor_observation_root" \ + "NODE_OPTIONS=$doctor_node_options" \ + openclaw doctor --fix --non-interactive >"$log" 2>&1 +} + replace_historical_mobile_pairing_candidate() { local update_spec update_spec="$(candidate_update_spec)" @@ -1907,6 +1942,30 @@ phase validate-update-restart-mode validate_update_restart_mode phase reset-run-state reset_run_state phase install-baseline install_baseline phase initialize-state initialize_state +if [ "$SCENARIO" = "workshop-doctor-recovery" ]; then + if [ "$baseline_spec" != "openclaw@2026.9.4" ]; then + echo "workshop-doctor-recovery requires the exact published openclaw@2026.9.4 baseline" >&2 + exit 2 + fi + phase configure-workshop-baseline node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs configure + phase prepare-workshop-baseline openclaw_e2e_maybe_timeout "$COMMAND_TIMEOUT" openclaw doctor --fix --non-interactive + phase resolve-workshop-candidate resolve_candidate_version + phase capture-workshop-baseline node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs baseline "$(package_root)" + phase capture-workshop-candidate node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs candidate "$CANDIDATE_SPEC" "$candidate_version" + phase seed-workshop-baseline-index node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs seed baseline + phase assert-workshop-published-refusal assert_workshop_published_refusal + phase repair-workshop-baseline run_workshop_doctor baseline "$ARTIFACT_ROOT/baseline-doctor.log" + phase assert-workshop-baseline-repair node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs doctor "$workshop_doctor_observation_root" baseline + phase update-workshop-recovered-state update_candidate 1 + phase assert-workshop-recovered-upgrade node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs upgrade "$last_update_observation_root" "$(package_root)" + phase seed-workshop-candidate-index node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs seed candidate + phase repair-workshop-candidate run_workshop_doctor candidate "$DOCTOR_LOG" + phase assert-workshop-candidate-repair node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs doctor "$workshop_doctor_observation_root" candidate + phase assert-workshop-recovery node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs complete + run_completed="1" + echo "Workshop Doctor recovery passed: published updater refused unchanged malformed state; explicit baseline Doctor, recovered upgrade, and explicit candidate Doctor succeeded." + exit 0 +fi if [ "$SCENARIO" = "custom-plugin-siblings" ]; then phase seed-sibling-plugin node scripts/e2e/lib/upgrade-survivor/custom-plugin-siblings.mjs seed phase validate-baseline-config validate_baseline_config diff --git a/scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs b/scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs new file mode 100644 index 000000000000..0df20d2407ff --- /dev/null +++ b/scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs @@ -0,0 +1,422 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { pathToFileURL } from "node:url"; +import { isMainThread } from "node:worker_threads"; + +const INDEX = "idx_skill_workshop_collection_reviews_workspace_time"; +const INDEX_SQL = `CREATE INDEX ${INDEX} ON skill_workshop_collection_reviews(workspace_dir, create_time DESC, review_id DESC)`; +const REVIEW = { + review_id: "survivor-workshop-review", + owner_agent_id: "main", + backup_id: "survivor-workshop-backup", + create_time: 1, + kept_names_json: '["retained-skill"]', + written_names_json: "[]", + dropped_json: "[]", +}; + +const readJson = (filename) => JSON.parse(fs.readFileSync(filename, "utf8")); +const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex"); + +function writeJson(filename, value) { + fs.mkdirSync(path.dirname(filename), { recursive: true }); + fs.writeFileSync(filename, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 }); +} + +function databasePath(stateDir) { + const filename = path.join(stateDir, "state", "openclaw.sqlite"); + assert(fs.statSync(filename).isFile(), "Published baseline did not create shared SQLite state"); + return filename; +} + +function buildIdentity(manifest, buildInfo) { + assert.equal(manifest.name, "openclaw"); + assert.equal(typeof manifest.version, "string"); + JSON.parse(buildInfo.toString("utf8")); + return { version: manifest.version, buildInfoSha256: sha256(buildInfo) }; +} + +function installedIdentity(packageRoot) { + return buildIdentity( + readJson(path.join(packageRoot, "package.json")), + fs.readFileSync(path.join(packageRoot, "dist", "build-info.json")), + ); +} + +export function captureWorkshopBaseline(packageRoot, artifactRoot) { + const identity = installedIdentity(packageRoot); + assert.equal(identity.version, "2026.9.4"); + writeJson(path.join(artifactRoot, "workshop-baseline.json"), identity); + return identity; +} + +export function captureWorkshopCandidate(tarball, artifactRoot, candidateVersion) { + const readPackedFile = (relative) => + execFileSync("tar", ["-xOf", tarball.replace(/^file:/u, ""), `package/${relative}`], { + maxBuffer: 1024 * 1024, + }); + const identity = buildIdentity( + JSON.parse(readPackedFile("package.json").toString("utf8")), + readPackedFile("dist/build-info.json"), + ); + assert.equal(identity.version, candidateVersion, "Candidate artifact version changed"); + const baseline = readJson(path.join(artifactRoot, "workshop-baseline.json")); + assert.notEqual( + identity.buildInfoSha256, + baseline.buildInfoSha256, + "Candidate must be a distinct build", + ); + writeJson(path.join(artifactRoot, "workshop-candidate.json"), identity); + return identity; +} + +function hasMalformedWorkshopIndex(filename) { + const database = new DatabaseSync(filename, { readOnly: true }); + try { + database.prepare("SELECT review_id FROM skill_workshop_collection_reviews LIMIT 1").get(); + return false; + } catch (error) { + if ( + error instanceof Error && + error.message.includes("malformed database schema") && + error.message.includes(INDEX) + ) { + return true; + } + throw error; + } finally { + database.close(); + } +} + +function inspectMalformedState(filename) { + assert(hasMalformedWorkshopIndex(filename), "Legacy Workshop fixture is not malformed"); + const database = new DatabaseSync(filename, { readOnly: true }); + try { + database.enableDefensive?.(false); + database.exec("PRAGMA writable_schema = ON;"); + const catalog = database + .prepare("SELECT type, name, tbl_name, rootpage, sql FROM sqlite_schema ORDER BY type, name") + .all(); + const index = catalog.find((entry) => entry.name === INDEX); + assert.equal(index?.sql, INDEX_SQL); + assert( + typeof index.rootpage === "number" && index.rootpage > 0, + "Malformed index must retain its physical b-tree", + ); + // Compare logical state, including the ledger, without mistaking WAL housekeeping for writes. + const digest = createHash("sha256").update(JSON.stringify(catalog)); + for (const pragma of ["user_version", "schema_version", "application_id"]) { + digest.update(JSON.stringify(database.prepare(`PRAGMA ${pragma}`).get())); + } + for (const table of catalog.filter((entry) => entry.type === "table")) { + const rows = database + .prepare(`SELECT * FROM "${table.name.replaceAll('"', '""')}"`) + .all() + .map((row) => JSON.stringify(row)) + .toSorted(); + digest.update(JSON.stringify([table.name, rows])); + } + const review = database + .prepare("SELECT * FROM skill_workshop_collection_reviews WHERE review_id = ?") + .get(REVIEW.review_id); + assert.deepEqual({ ...review }, REVIEW, "Retained Workshop review changed"); + return { + index: INDEX, + sql: index.sql, + rootpage: index.rootpage, + stateSha256: digest.digest("hex"), + }; + } finally { + database.close(); + } +} + +export function seedWorkshopIndex(stateDir, artifactRoot, stage) { + assert(["baseline", "candidate"].includes(stage)); + const filename = databasePath(stateDir); + const database = new DatabaseSync(filename); + try { + if (stage === "baseline") { + database + .prepare( + `INSERT INTO skill_workshop_collection_reviews ( + review_id, owner_agent_id, backup_id, create_time, kept_names_json, written_names_json, dropped_json + ) VALUES (?, ?, ?, ?, ?, ?, ?)`, + ) + .run(...Object.values(REVIEW)); + } else { + assert.deepEqual( + { + ...database + .prepare("SELECT * FROM skill_workshop_collection_reviews WHERE review_id = ?") + .get(REVIEW.review_id), + }, + REVIEW, + "Candidate reseeding must preserve the upgraded review", + ); + } + database.exec( + `CREATE INDEX ${INDEX} ON skill_workshop_collection_reviews(review_id, create_time DESC);`, + ); + database.enableDefensive?.(false); + database.exec("PRAGMA writable_schema = ON;"); + database + .prepare("UPDATE sqlite_schema SET sql = ? WHERE type = 'index' AND name = ?") + .run(INDEX_SQL, INDEX); + const { schema_version } = database.prepare("PRAGMA schema_version").get(); + database.exec(`PRAGMA writable_schema = OFF; PRAGMA schema_version = ${schema_version + 1};`); + } finally { + database.close(); + } + const seeded = inspectMalformedState(filename); + writeJson(path.join(artifactRoot, `workshop-${stage}-seeded.json`), seeded); + return seeded; +} + +function observeProcess() { + const role = process.argv[2]; + const stateDir = process.env.OPENCLAW_STATE_DIR; + const observations = process.env.OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_ROOT; + if ( + !isMainThread || + !["update", "doctor"].includes(role) || + !observations || + !stateDir || + stateDir !== process.env.OPENCLAW_UPGRADE_SURVIVOR_WORKSHOP_STATE_DIR + ) { + return; + } + let identity = null; + try { + let directory = path.dirname(fs.realpathSync(process.argv[1])); + // Installed CLI entrypoints are at the package root or inside dist. + for (let depth = 0; depth < 3; depth++, directory = path.dirname(directory)) { + const manifest = path.join(directory, "package.json"); + if (fs.existsSync(manifest) && readJson(manifest).name === "openclaw") { + identity = installedIdentity(directory); + break; + } + } + } catch { + // Missing identity rejects the evidence without changing the observed CLI. + } + const evidence = { + role, + pid: process.pid, + parentPid: process.ppid, + identity, + updateInProgress: process.env.OPENCLAW_UPDATE_IN_PROGRESS === "1", + malformedAtStart: hasMalformedWorkshopIndex(databasePath(stateDir)), + }; + const filename = path.join(observations, `workshop-process-${process.pid}.json`); + writeJson(filename, evidence); + process.once("exit", (exitCode) => writeJson(filename, { ...evidence, exitCode })); +} + +function processWitness(observations, identity, expected) { + assert.match(identity.buildInfoSha256, /^[a-f0-9]{64}$/u, "Missing build identity"); + const receipts = fs + .readdirSync(path.join(observations, "diagnostics")) + .filter((name) => /^process-\d+-exited\.json$/u.test(name)) + .map((name) => readJson(path.join(observations, "diagnostics", name))); + const witnesses = fs + .readdirSync(observations) + .filter((name) => /^workshop-process-\d+\.json$/u.test(name)) + .map((name) => readJson(path.join(observations, name))); + const witness = witnesses.find( + (entry) => + entry.identity?.version === identity.version && + entry.identity?.buildInfoSha256 === identity.buildInfoSha256 && + Object.entries(expected).every(([key, value]) => entry[key] === value) && + receipts.some( + (receipt) => + receipt.role === entry.role && + receipt.packageVersion === identity.version && + receipt.pid === entry.pid && + receipt.parentPid === entry.parentPid && + receipt.exitCode === entry.exitCode, + ), + ); + assert(witness, `Missing matching ${expected.role} build, source-state, and exit evidence`); + return witness; +} + +function assertRepairedState(stateDir) { + const database = new DatabaseSync(databasePath(stateDir), { readOnly: true }); + try { + assert.equal( + database.prepare("SELECT name FROM sqlite_schema WHERE name = ?").get(INDEX), + undefined, + "Doctor left the malformed Workshop index behind", + ); + assert.deepEqual( + { + ...database + .prepare("SELECT * FROM skill_workshop_collection_reviews WHERE review_id = ?") + .get(REVIEW.review_id), + }, + REVIEW, + "Doctor changed the retained Workshop review", + ); + assert.deepEqual( + database + .prepare("PRAGMA integrity_check") + .all() + .map((row) => row.integrity_check), + ["ok"], + ); + } finally { + database.close(); + } +} + +export function assertWorkshopUpdateRefusal( + stateDir, + artifactRoot, + observations, + packageRoot, + exitCode, +) { + assert.equal(exitCode, 1, "Published updater must refuse before installing the candidate"); + const baseline = readJson(path.join(artifactRoot, "workshop-baseline.json")); + assert.deepEqual( + installedIdentity(packageRoot), + baseline, + "Refused updater changed installed build", + ); + const seeded = readJson(path.join(artifactRoot, "workshop-baseline-seeded.json")); + assert.deepEqual( + inspectMalformedState(databasePath(stateDir)), + seeded, + "Refused updater changed state", + ); + const result = readJson(path.join(artifactRoot, "update.json")); + assert.equal(result.ok, false); + assert.equal(result.error?.type, "cli_error"); + assert(result.error.message.includes("SQLite integrity_check failed")); + assert(result.error.message.includes(`Page ${seeded.rootpage}: never used`)); + const updater = processWitness(observations, baseline, { + role: "update", + exitCode: 1, + malformedAtStart: true, + }); + const doctorStarted = fs + .readdirSync(path.join(observations, "diagnostics")) + .filter((name) => /^process-\d+-started\.json$/u.test(name)) + .some((name) => + ["doctor", "post-core"].includes(readJson(path.join(observations, "diagnostics", name)).role), + ); + assert.equal(doctorStarted, false, "Published refusal must precede the candidate handoff"); + const refusal = { + status: "refused-before-candidate", + automaticRepair: false, + stateSha256: seeded.stateSha256, + updater, + }; + writeJson(path.join(artifactRoot, "workshop-published-refusal.json"), refusal); + return refusal; +} + +export function assertWorkshopDoctorRepair(stateDir, artifactRoot, observations, stage) { + assert(["baseline", "candidate"].includes(stage)); + const identity = readJson(path.join(artifactRoot, `workshop-${stage}.json`)); + assertRepairedState(stateDir); + const doctor = processWitness(observations, identity, { + role: "doctor", + exitCode: 0, + malformedAtStart: true, + updateInProgress: false, + }); + const repair = { status: "explicit-doctor-repaired", doctor }; + writeJson(path.join(artifactRoot, `workshop-${stage}-doctor.json`), repair); + return repair; +} + +export function assertWorkshopRecoveredUpgrade(stateDir, artifactRoot, observations, packageRoot) { + const baseline = readJson(path.join(artifactRoot, "workshop-baseline.json")); + const candidate = readJson(path.join(artifactRoot, "workshop-candidate.json")); + assert.deepEqual( + installedIdentity(packageRoot), + candidate, + "Updater did not install the exact candidate", + ); + assertRepairedState(stateDir); + const updater = processWitness(observations, baseline, { + role: "update", + exitCode: 0, + malformedAtStart: false, + }); + const doctor = processWitness(observations, candidate, { + role: "doctor", + exitCode: 0, + malformedAtStart: false, + updateInProgress: true, + }); + const upgraded = { status: "upgraded-after-explicit-repair", updater, doctor }; + writeJson(path.join(artifactRoot, "workshop-recovered-upgrade.json"), upgraded); + return upgraded; +} + +export function completeWorkshopRecovery(stateDir, artifactRoot) { + assertRepairedState(stateDir); + const firstAttempt = readJson(path.join(artifactRoot, "workshop-published-refusal.json")); + const baselineDoctor = readJson(path.join(artifactRoot, "workshop-baseline-doctor.json")); + const upgrade = readJson(path.join(artifactRoot, "workshop-recovered-upgrade.json")); + const candidateDoctor = readJson(path.join(artifactRoot, "workshop-candidate-doctor.json")); + assert.equal(firstAttempt.status, "refused-before-candidate"); + assert.equal(firstAttempt.automaticRepair, false); + assert.equal(baselineDoctor.status, "explicit-doctor-repaired"); + assert.equal(upgrade.status, "upgraded-after-explicit-repair"); + assert.equal(candidateDoctor.status, "explicit-doctor-repaired"); + const result = { firstAttempt, baselineDoctor, upgrade, candidateDoctor }; + writeJson(path.join(artifactRoot, "workshop-doctor-recovery.json"), result); + return result; +} + +const direct = + process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url; +if (direct) { + const [mode, first, second, third] = process.argv.slice(2); + const stateDir = process.env.OPENCLAW_STATE_DIR; + const artifacts = process.env.OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_ROOT; + assert(stateDir && artifacts, "Missing isolated survivor paths"); + if (mode === "configure") { + assert(process.env.OPENCLAW_CONFIG_PATH, "Missing isolated config path"); + writeJson(process.env.OPENCLAW_CONFIG_PATH, { + gateway: { + mode: "local", + bind: "loopback", + auth: { mode: "token", token: "upgrade-survivor-token" }, + controlUi: { enabled: false }, + }, + agents: { + ownership: "explicit", + entries: { main: { workspace: path.join(stateDir, "workspace") } }, + }, + plugins: { enabled: false }, + }); + } else if (mode === "baseline") { + captureWorkshopBaseline(first, artifacts); + } else if (mode === "candidate") { + captureWorkshopCandidate(first, artifacts, second); + } else if (mode === "seed") { + seedWorkshopIndex(stateDir, artifacts, first); + } else if (mode === "refusal") { + assertWorkshopUpdateRefusal(stateDir, artifacts, first, second, Number(third)); + } else if (mode === "doctor") { + assertWorkshopDoctorRepair(stateDir, artifacts, first, second); + } else if (mode === "upgrade") { + assertWorkshopRecoveredUpgrade(stateDir, artifacts, first, second); + } else if (mode === "complete") { + completeWorkshopRecovery(stateDir, artifacts); + } else { + throw new Error(`Unknown Workshop recovery fixture mode: ${mode}`); + } +} else { + observeProcess(); +} diff --git a/scripts/e2e/upgrade-survivor-docker.sh b/scripts/e2e/upgrade-survivor-docker.sh index 463b736add8f..fef670b3ec9b 100755 --- a/scripts/e2e/upgrade-survivor-docker.sh +++ b/scripts/e2e/upgrade-survivor-docker.sh @@ -218,6 +218,14 @@ if [ "$SCENARIO" = "abandoned-update" ] && { exit 1 fi +if [ "$SCENARIO" = "workshop-doctor-recovery" ] && { + [ "${OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE:-0}" != "1" ] || + [ "$UPDATE_RESTART_MODE" != "manual" ] || [ "$ROOT_MANAGED_VPS" != "0" ] || [ "$LIVE_OPENAI" != "0" ]; +}; then + echo "workshop-doctor-recovery requires the published baseline, manual restart, and no live provider" >&2 + exit 1 +fi + resolve_lane_artifact_suffix() { if [ -n "${OPENCLAW_DOCKER_ALL_LANE_NAME:-}" ]; then printf "%s" "$OPENCLAW_DOCKER_ALL_LANE_NAME" @@ -327,17 +335,31 @@ if [ "${OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE:-0}" = "1" ]; then CANDIDATE_SPEC="$(normalize_npm_candidate "$CANDIDATE_RAW")" fi - if [ "$CANDIDATE_IS_CURRENT" = "1" ] && [ "$SCENARIO" != "custom-plugin-siblings" ] && [ -z "${OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR:-}" ]; then - AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT="$( - mktemp -d "${TMPDIR:-/tmp}/openclaw-upgrade-survivor-plugin-registry.XXXXXX" + if [ "$CANDIDATE_IS_CURRENT" = "1" ] && [ -z "${OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR:-}" ]; then + registry_required="$( + OPENCLAW_DOCKER_ALL_LANES=published-upgrade-survivor \ + OPENCLAW_DOCKER_ALL_TIMINGS=0 \ + OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS="$BASELINE_SPEC" \ + OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS="$SCENARIO" \ + node "$HARNESS_ROOT_DIR/scripts/test-docker-all.mjs" --plan-json | node -e ' + const plan = JSON.parse(require("node:fs").readFileSync(0, "utf8")); + const required = plan.needs?.prepublishPluginRegistry; + if (typeof required !== "boolean") throw new Error("Docker planner omitted plugin registry requirements"); + process.stdout.write(required ? "1" : "0"); + ' )" - OPENCLAW_DOCKER_ALL_LANES=published-upgrade-survivor \ - OPENCLAW_DOCKER_ALL_LOG_DIR="$AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT" \ - OPENCLAW_DOCKER_ALL_TIMINGS=0 \ - OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS="$BASELINE_SPEC" \ - OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS="$SCENARIO" \ - node "$HARNESS_ROOT_DIR/scripts/test-docker-all.mjs" --prepare-plugin-registry - export OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR="$AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT/prepublish-plugin-registry" + if [ "$registry_required" = "1" ]; then + AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT="$( + mktemp -d "${TMPDIR:-/tmp}/openclaw-upgrade-survivor-plugin-registry.XXXXXX" + )" + OPENCLAW_DOCKER_ALL_LANES=published-upgrade-survivor \ + OPENCLAW_DOCKER_ALL_LOG_DIR="$AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT" \ + OPENCLAW_DOCKER_ALL_TIMINGS=0 \ + OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS="$BASELINE_SPEC" \ + OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS="$SCENARIO" \ + node "$HARNESS_ROOT_DIR/scripts/test-docker-all.mjs" --prepare-plugin-registry + export OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR="$AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT/prepublish-plugin-registry" + fi fi if [ -n "$PACKAGE_TGZ" ]; then diff --git a/scripts/lib/docker-e2e-plan.mts b/scripts/lib/docker-e2e-plan.mts index 829f325273c3..749146b5dd82 100644 --- a/scripts/lib/docker-e2e-plan.mts +++ b/scripts/lib/docker-e2e-plan.mts @@ -133,6 +133,10 @@ const UPGRADE_SURVIVOR_RUNTIME_COMPANION_PACKAGES = ["@openclaw/codex"]; // Pre-protocol catalogs are content-addressed. Unknown legacy blocks fail // closed instead of requiring a dependency or reimplementing a JavaScript parser. const LEGACY_UPGRADE_SURVIVOR_SCENARIO_CATALOGS = new Map([ + [ + "7d9d7520c2c34d51fff78e542a7f539b77080bfd04648438e95aec4af3fe362e", + "base msteams-polls abandoned-update legacy-operator-state workshop-doctor-recovery mobile-pairing-reconnect acpx-openclaw-tools-bridge feishu-channel bootstrap-persona channel-post-core-restore codex-allowlist-survival plugin-deps-cleanup configured-plugin-installs missing-configured-plugin-migration custom-plugin-siblings stale-source-plugin-shadow prerelease-plugin-registry tilde-log-path meeting-transcripts-sqlite versioned-runtime-deps cron-scheduled-authority sqlite-volume recovery-cleanup auth-profile-v2026-7-2-beta-5 watchos-direct-node", + ], [ "5e8821538f3722fdf0dc3b3917ae639853f305e4c7a9b84febb8905601a9b5c7", "base msteams-polls abandoned-update legacy-operator-state mobile-pairing-reconnect acpx-openclaw-tools-bridge feishu-channel bootstrap-persona channel-post-core-restore codex-allowlist-survival plugin-deps-cleanup configured-plugin-installs missing-configured-plugin-migration custom-plugin-siblings stale-source-plugin-shadow prerelease-plugin-registry tilde-log-path meeting-transcripts-sqlite versioned-runtime-deps cron-scheduled-authority sqlite-volume recovery-cleanup auth-profile-v2026-7-2-beta-5 watchos-direct-node", @@ -682,7 +686,12 @@ export function requiredPrepublishPluginPackagesForLanes(poolLanes: DockerE2eLan requiredPackages.add(packageName); } const scenario = upgradeSurvivorScenarioForLane(poolLane); - if (!scenario || scenario === "abandoned-update" || scenario === "custom-plugin-siblings") { + if ( + !scenario || + scenario === "abandoned-update" || + scenario === "custom-plugin-siblings" || + scenario === "workshop-doctor-recovery" + ) { continue; } if (scenario === "legacy-operator-state") { diff --git a/scripts/lib/upgrade-survivor-policy.mjs b/scripts/lib/upgrade-survivor-policy.mjs index fef94051c684..b56cb92e96b4 100644 --- a/scripts/lib/upgrade-survivor-policy.mjs +++ b/scripts/lib/upgrade-survivor-policy.mjs @@ -3,6 +3,7 @@ const UPGRADE_SURVIVOR_SCENARIOS = Object.freeze([ "msteams-polls", "abandoned-update", "legacy-operator-state", + "workshop-doctor-recovery", "mobile-pairing-reconnect", "acpx-openclaw-tools-bridge", "feishu-channel", @@ -39,7 +40,11 @@ const scenarioMinimumBaselines = new Map([ // These black-box scenarios are implemented entirely by the current trusted // release harness and treat the selected tree only as the package under test. -const TRUSTED_HARNESS_OWNED_SCENARIOS = new Set(["mobile-pairing-reconnect", "abandoned-update"]); +const TRUSTED_HARNESS_OWNED_SCENARIOS = new Set([ + "mobile-pairing-reconnect", + "abandoned-update", + "workshop-doctor-recovery", +]); export function isTrustedHarnessOwnedUpgradeSurvivorScenario(scenario) { return TRUSTED_HARNESS_OWNED_SCENARIOS.has(scenario); @@ -55,6 +60,7 @@ const aggregateScenarios = UPGRADE_SURVIVOR_SCENARIOS.filter( scenario !== "msteams-polls" && scenario !== "abandoned-update" && scenario !== "missing-configured-plugin-migration" && + scenario !== "workshop-doctor-recovery" && scenario !== "mobile-pairing-reconnect" && scenario !== "watchos-direct-node" && scenario !== "prerelease-plugin-registry" && @@ -153,6 +159,9 @@ export function supportsUpgradeSurvivorScenarioAtBaseline(scenario, baselineSpec if (scenario === "abandoned-update" || scenario === "missing-configured-plugin-migration") { return baselineSpec === "openclaw@2026.9.2"; } + if (scenario === "workshop-doctor-recovery") { + return baselineSpec === "openclaw@2026.9.4"; + } const minimumBaseline = scenarioMinimumBaselines.get(scenario); return ( !minimumBaseline || diff --git a/src/cli/run-main.exit.test.ts b/src/cli/run-main.exit.test.ts index d9434e83c8cb..f468901456ae 100644 --- a/src/cli/run-main.exit.test.ts +++ b/src/cli/run-main.exit.test.ts @@ -2449,15 +2449,22 @@ describe("runCli exit behavior", () => { expect(startProxyMock).toHaveBeenCalledWith(undefined); }); - it("reads source-only proxy config before doctor lint owns plugin-aware validation", async () => { + it.each([ + ["lint", ["--lint", "--json"]], + ["repair", ["--fix", "--non-interactive"]], + ["diagnosis", []], + ])("reads source-only proxy config before Doctor %s owns state access", async (_mode, args) => { tryRouteCliMock.mockResolvedValueOnce(true); - readSourceConfigBestEffortMock.mockResolvedValueOnce({ proxy: { selected: "doctor-lint" } }); + readSourceConfigBestEffortMock.mockResolvedValueOnce({ proxy: { selected: "doctor" } }); + loadConfigMock.mockImplementation(() => { + throw new Error("Shared state requires Doctor repair"); + }); - await runCli(["node", "openclaw", "doctor", "--lint", "--json"]); + await runCli(["node", "openclaw", "doctor", ...args]); expect(readSourceConfigBestEffortMock).toHaveBeenCalledOnce(); expect(loadConfigMock).not.toHaveBeenCalled(); - expect(startProxyMock).toHaveBeenCalledWith({ selected: "doctor-lint" }); + expect(startProxyMock).toHaveBeenCalledWith({ selected: "doctor" }); }); it.each([ diff --git a/src/cli/run-main.ts b/src/cli/run-main.ts index 66a2b67e8e8c..7e49179f9bb2 100644 --- a/src/cli/run-main.ts +++ b/src/cli/run-main.ts @@ -22,7 +22,6 @@ import type { PluginCliLoadSession } from "../plugins/cli-registry-loader.js"; import { createPluginCache, getPluginCache, withPluginCache } from "../plugins/plugin-cache.js"; import { resolveCliArgvInvocation } from "./argv-invocation.js"; import { - hasFlag, normalizeGeneratedHelpCommandArgv, normalizeRootHelpTargetArgv, normalizeRootLogLevelArgv, @@ -1209,7 +1208,7 @@ async function runCliWithPreparedOutputMode( const useSourceOnlyBestEffortConfig = !(await isCurrentRuntimeSupported()) || normalizedInvocation.primary === "update" || - (normalizedInvocation.primary === "doctor" && hasFlag(normalizedArgv, "--lint")); + normalizedInvocation.primary === "doctor"; const readBestEffortCliConfig = async (): Promise => { if (!bestEffortConfigPromise) { bestEffortConfigPromise = import("../config/io.js").then(async (configIo) => { diff --git a/src/commands/doctor-maintenance.finish-revalidation.test.ts b/src/commands/doctor-maintenance.finish-revalidation.test.ts index 7fddda2e05d9..84458657f04c 100644 --- a/src/commands/doctor-maintenance.finish-revalidation.test.ts +++ b/src/commands/doctor-maintenance.finish-revalidation.test.ts @@ -1,18 +1,26 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; import path from "node:path"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import type { GatewayService } from "../daemon/service.js"; import { createMockGatewayService, mockSystemAccountHome } from "../daemon/service.test-helpers.js"; +import { openNodeSqliteDatabase } from "../infra/node-sqlite.js"; import * as updateRunDriver from "../infra/update-run-driver.js"; import { readUpdateRunDriver } from "../infra/update-run-driver.js"; import { createUpdateRun, getUpdateRun, + listUpdateRuns, recordUpdateRunPhase, recordUpdateRunStep, finishUpdateRun, } from "../infra/update-run-ledger.js"; -import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { OPENCLAW_STATE_SCHEMA_VERSION } from "../state/openclaw-state-db-contract.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; import { withEnvAsync } from "../test-utils/env.js"; import { mockProcessPlatform } from "../test-utils/vitest-spies.js"; import { beginDoctorMaintenance } from "./doctor-maintenance.js"; @@ -102,10 +110,18 @@ type Continuation = | "lost-before-restart" | "dead-before-restart" | "terminal-dead-before-restart"; +type LegacyCatalog = + | "exact" + | "unknown" + | "future-version" + | "future-content" + | "conflict-on-recheck" + | "different-state"; async function runDoctorFinishForStoppedUnit( scenario: StoppedUnitState, continuation?: Continuation, + legacyCatalog?: LegacyCatalog, ): Promise<{ finishError: unknown; restartCalls: number; @@ -176,6 +192,79 @@ async function runDoctorFinishForStoppedUnit( ); } } + let assertCatalogUnchanged = () => {}; + let assertPreStopArtifactsUnchanged = () => {}; + let activateCompetingUpdate: (() => void) | undefined; + if (legacyCatalog) { + const lateRun = + legacyCatalog === "conflict-on-recheck" + ? createUpdateRun({ trigger: "cli", origin: { driver: readUpdateRunDriver() } }) + : undefined; + if (lateRun) { + finishUpdateRun(lateRun.runId, { status: "skipped" }); + } + const pathname = openOpenClawStateDatabase().path; + closeOpenClawStateDatabaseForTest(); + const db = openNodeSqliteDatabase(pathname); + try { + db.exec( + "CREATE INDEX idx_skill_workshop_collection_reviews_workspace_time ON skill_workshop_collection_reviews(review_id, create_time DESC);", + ); + if (legacyCatalog === "future-version") { + db.exec(`PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION + 1}`); + } + if (legacyCatalog === "future-content") { + db.prepare( + "INSERT INTO config_machine_state (state_key, value_json, updated_at_ms) VALUES ('state.schema.contentVersion', ?, 1)", + ).run(String(OPENCLAW_STATE_SCHEMA_VERSION + 1)); + } + db.enableDefensive?.(false); + db.exec("PRAGMA writable_schema = ON"); + db.prepare("UPDATE sqlite_schema SET sql = ? WHERE type = 'index' AND name = ?").run( + `CREATE INDEX idx_skill_workshop_collection_reviews_workspace_time ON skill_workshop_collection_reviews(${legacyCatalog === "unknown" ? "unexpected_column" : "workspace_dir"}, create_time DESC, review_id DESC)`, + "idx_skill_workshop_collection_reviews_workspace_time", + ); + const schema = db.prepare("PRAGMA schema_version").get() as { schema_version: number }; + db.exec( + `PRAGMA writable_schema = OFF; PRAGMA schema_version = ${schema.schema_version + 1}`, + ); + } finally { + db.close(); + } + const readArtifacts = () => + [pathname, `${pathname}-wal`, `${pathname}-shm`].map((file) => + fs.existsSync(file) + ? createHash("sha256").update(fs.readFileSync(file)).digest("hex") + : undefined, + ); + const beforeArtifacts = readArtifacts(); + const beforeCatalog = fs.readFileSync(pathname); + assertCatalogUnchanged = () => + expect(fs.readFileSync(pathname).equals(beforeCatalog)).toBe(true); + assertPreStopArtifactsUnchanged = () => expect(readArtifacts()).toEqual(beforeArtifacts); + expect(() => listUpdateRuns()).toThrow( + legacyCatalog === "future-version" + ? /uses newer schema version/ + : /legacy-workshop-review-index.*doctor --fix/, + ); + assertPreStopArtifactsUnchanged(); + if (lateRun) { + activateCompetingUpdate = () => { + const writer = openNodeSqliteDatabase(pathname); + try { + writer.enableDefensive?.(false); + writer.exec("PRAGMA writable_schema = ON"); + writer + .prepare( + "UPDATE update_runs SET status = 'running', phase = 'validating', finished_at_ms = NULL WHERE run_id = ?", + ) + .run(lateRun.runId); + } finally { + writer.close(); + } + }; + } + } mockProcessPlatform("linux"); let running = continuation !== "parked" && @@ -191,7 +280,9 @@ async function runDoctorFinishForStoppedUnit( "--port", "18789", ], - environment: { HOME: home }, + environment: { + HOME: legacyCatalog === "different-state" ? path.join(home, "other") : home, + }, }; const restart = vi.fn(async () => { if (scenario === "restart-failed") { @@ -206,8 +297,11 @@ async function runDoctorFinishForStoppedUnit( hasInstalledDefinition: async () => true, isLoaded: async () => scenario === "retained", readCommand: async (_env, opts) => { - if (continuation === "lost-before-stop" && ++commandReads === 2 && runId) { - createUpdateRun({ trigger: "cli", origin: { driver: readUpdateRunDriver() } }); + if (++commandReads === 2) { + activateCompetingUpdate?.(); + if (continuation === "lost-before-stop" && runId) { + createUpdateRun({ trigger: "cli", origin: { driver: readUpdateRunDriver() } }); + } } if ( stopObserved && @@ -241,6 +335,7 @@ async function runDoctorFinishForStoppedUnit( : { status: "stopped" }; }, stop: vi.fn(async () => { + assertPreStopArtifactsUnchanged(); mocks.stops += 1; running = false; stopObserved = true; @@ -259,8 +354,18 @@ async function runDoctorFinishForStoppedUnit( error: () => {}, exit: () => {}, }, + }).finally(() => { + if (!activateCompetingUpdate) { + assertCatalogUnchanged(); + if (mocks.stops === 0) { + assertPreStopArtifactsUnchanged(); + } + } }); expect(maintenance).toBeDefined(); + if (legacyCatalog) { + expect(() => maintenance?.run(() => listUpdateRuns())).toThrow(); + } if (continuation === "lost-before-restart" && runId) { createUpdateRun({ trigger: "cli", origin: { driver: readUpdateRunDriver() } }); } @@ -282,20 +387,63 @@ async function runDoctorFinishForStoppedUnit( } catch (error) { finishError = error; } + assertCatalogUnchanged(); + const savedRun = runId && !legacyCatalog ? getUpdateRun(runId) : undefined; return { finishError, restartCalls: restart.mock.calls.length, logs, - takeoverSteps: runId - ? (getUpdateRun(runId)?.steps.filter((step) => step.step === "finalize:repair-takeover") - .length ?? 0) - : 0, - runStatus: runId ? getUpdateRun(runId)?.status : undefined, + takeoverSteps: + savedRun?.steps.filter((step) => step.step === "finalize:repair-takeover").length ?? 0, + runStatus: savedRun?.status, }; }, ); } +it("admits exact legacy catalog reads for an owned running service without repairing it", async () => { + const result = await runDoctorFinishForStoppedUnit("retained", undefined, "exact"); + expect(result.finishError).toBeUndefined(); + expect(mocks.stops).toBe(1); + expect(result.restartCalls).toBe(1); +}); + +it("preserves the existing malformed continuation writer refusal before stopping the service", async () => { + await expect(runDoctorFinishForStoppedUnit("retained", "own", "exact")).rejects.toThrow( + "schema migration required", + ); + expect(mocks.stops).toBe(0); +}); + +it.each([ + { catalog: "exact", continuation: "manual", message: "is still in progress" }, + { catalog: "conflict-on-recheck", continuation: undefined, message: "is still in progress" }, + { + catalog: "future-version", + continuation: undefined, + message: `uses newer schema version ${OPENCLAW_STATE_SCHEMA_VERSION + 1}`, + }, + { + catalog: "future-content", + continuation: undefined, + message: `uses newer schema version ${OPENCLAW_STATE_SCHEMA_VERSION + 1}`, + }, + { + catalog: "different-state", + continuation: undefined, + message: "non-default state dir or config path", + }, + { catalog: "unknown", continuation: undefined, message: "schema migration required" }, +] as const)( + "refuses $catalog/$continuation before stopping the service", + async ({ catalog, continuation, message }) => { + await expect(runDoctorFinishForStoppedUnit("retained", continuation, catalog)).rejects.toThrow( + message, + ); + expect(mocks.stops).toBe(0); + }, +); + it.each(["own", "parked", "normal-update-parked", "unrecorded-parked"] as const)( "continues owning-run Doctor maintenance with service %s", async (continuation) => { diff --git a/src/commands/doctor-maintenance.ts b/src/commands/doctor-maintenance.ts index 52351f71754c..b88c7e781e7c 100644 --- a/src/commands/doctor-maintenance.ts +++ b/src/commands/doctor-maintenance.ts @@ -18,6 +18,7 @@ import { createOpenClawDatabaseMaintenanceScope, type OpenClawDatabaseMaintenanceScope, } from "../state/openclaw-state-db-async-lifecycle.js"; +import { openDoctorStateSchemaReadAdmission } from "../state/openclaw-state-db-doctor-schema.js"; import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; import type { DoctorOptions } from "./doctor-prompter.js"; import { isDoctorUpdateRepairMode, resolveDoctorRepairMode } from "./doctor-repair-mode.js"; @@ -136,6 +137,7 @@ export async function beginDoctorMaintenance(params: { const runs = listUpdateRuns( { active: true, limit: 100, includeRunId: inheritedRunId }, { env }, + openDoctorStateSchemaReadAdmission, ); const admission = inspectUpdateRepairDriverAdmission(runs, inheritedRunId); if (admission.kind === "conflict") { @@ -209,7 +211,7 @@ export async function beginDoctorMaintenance(params: { const { assertNoOpenClawAgentDatabaseLeasesReadOnly, OpenClawAgentDatabaseLeaseActiveError } = await import("../state/openclaw-agent-db-lease.js"); try { - assertNoOpenClawAgentDatabaseLeasesReadOnly({ env }); + assertNoOpenClawAgentDatabaseLeasesReadOnly({ env }, openDoctorStateSchemaReadAdmission); } catch (error) { if (error instanceof OpenClawAgentDatabaseLeaseActiveError) { throw error; @@ -220,6 +222,7 @@ export async function beginDoctorMaintenance(params: { const schemas = await preflightOpenClawDatabaseSchemas({ env, scope: "state", + openStateSchemaReadAdmission: openDoctorStateSchemaReadAdmission, }); const unreadable = schemas.indeterminate.find((database) => database.kind === "state"); if (unreadable) { diff --git a/src/commands/doctor-update-schema-guard.ts b/src/commands/doctor-update-schema-guard.ts index 02ac20318287..b45f6efba2c8 100644 --- a/src/commands/doctor-update-schema-guard.ts +++ b/src/commands/doctor-update-schema-guard.ts @@ -8,6 +8,7 @@ import { preflightOpenClawDatabaseSchemas, type OpenClawDatabaseSchemaPreflight, } from "../state/openclaw-database-preflight.js"; +import { openDoctorStateSchemaReadAdmission } from "../state/openclaw-state-db-doctor-schema.js"; import { tableExists } from "../state/openclaw-state-db-schema-helpers.js"; import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; import { readStateSchemaPublicationBlocker } from "../state/openclaw-state-schema-publication.js"; @@ -28,7 +29,9 @@ async function readDrivingUpdater(): Promise< }); try { const database = openNodeSqliteDatabase(snapshot.location, { readOnly: true }); + let closeSchemaReadAdmission: (() => void) | undefined; try { + closeSchemaReadAdmission = openDoctorStateSchemaReadAdmission(database); const blocker = readStateSchemaPublicationBlocker(database); return blocker ? { @@ -37,8 +40,12 @@ async function readDrivingUpdater(): Promise< } : undefined; } finally { - clearNodeSqliteKyselyCacheForDatabase(database); - database.close(); + try { + closeSchemaReadAdmission?.(); + } finally { + clearNodeSqliteKyselyCacheForDatabase(database); + database.close(); + } } } finally { await snapshot.cleanupAsync(); @@ -58,6 +65,7 @@ export async function guardUpdateDoctorSchemaUpgrade(options: { options.schemas ?? (await preflightOpenClawDatabaseSchemas({ env: process.env, + openStateSchemaReadAdmission: openDoctorStateSchemaReadAdmission, })); if (!schemas.pendingMigrations?.length) { return; diff --git a/src/flows/doctor-health.dangling-workshop-index.test.ts b/src/flows/doctor-health.dangling-workshop-index.test.ts new file mode 100644 index 000000000000..887b671b8ecd --- /dev/null +++ b/src/flows/doctor-health.dangling-workshop-index.test.ts @@ -0,0 +1,163 @@ +import fs from "node:fs"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { doctorCommand } from "../commands/doctor.js"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { readSqliteNumberPragma } from "../infra/sqlite-pragma.test-support.js"; +import { OPENCLAW_AGENT_SCHEMA_VERSION } from "../state/openclaw-agent-db-contract.js"; +import { closeOpenClawStateDatabaseAsync } from "../state/openclaw-state-db-cache.js"; +import { OPENCLAW_STATE_SCHEMA_VERSION } from "../state/openclaw-state-db-contract.js"; +import { openOpenClawStateDatabase } from "../state/openclaw-state-db.js"; +import { claimOpenClawStateOwnership } from "../state/openclaw-state-ownership-operations.js"; +import { + withOpenClawTestState, + type OpenClawTestState, +} from "../test-utils/openclaw-test-state.js"; +import { mocks } from "./doctor-health.test-support.js"; + +// Keep the real Doctor config/migration chain; the shared harness isolates service and UI checks. +vi.doUnmock("../commands/doctor-config-flow.js"); +vi.doUnmock("../commands/doctor-prompter.js"); +vi.doUnmock("../commands/doctor/shared/plugin-runtime-symlinks.js"); + +beforeEach(() => { + mocks.packageRoot.mockReturnValue(undefined); + mocks.outro.mockClear(); + mocks.runContributions.mockReset(); +}); +afterEach(() => vi.restoreAllMocks()); + +async function seedState(state: OpenClawTestState) { + await state.writeConfig({ + agents: { ownership: "explicit", entries: { main: { workspace: state.workspaceDir } } }, + gateway: { mode: "local" }, + plugins: { enabled: false }, + }); + const database = openOpenClawStateDatabase({ env: state.env }); + database.db.exec(` + INSERT INTO skill_workshop_collection_reviews ( + review_id, owner_agent_id, backup_id, create_time, kept_names_json, written_names_json, dropped_json + ) VALUES ('review-preserved', 'main', 'backup-preserved', 1, '[]', '[]', '[]'); + `); + return database; +} + +function damageWorkshopIndex(databasePath: string): void { + const { DatabaseSync } = requireNodeSqlite(); + const database = new DatabaseSync(databasePath); + try { + database.exec( + "CREATE INDEX idx_skill_workshop_collection_reviews_workspace_time ON skill_workshop_collection_reviews(review_id, create_time DESC);", + ); + database.enableDefensive?.(false); + database.exec("PRAGMA writable_schema = ON;"); + database + .prepare( + `UPDATE sqlite_schema + SET sql = 'CREATE INDEX idx_skill_workshop_collection_reviews_workspace_time + ON skill_workshop_collection_reviews(workspace_dir, create_time DESC, review_id DESC)' + WHERE type = 'index' AND name = 'idx_skill_workshop_collection_reviews_workspace_time'`, + ) + .run(); + const schemaVersion = readSqliteNumberPragma(database, "schema_version"); + database.exec(`PRAGMA writable_schema = OFF; PRAGMA schema_version = ${schemaVersion + 1};`); + } finally { + database.close(); + } +} + +describe("Doctor malformed Workshop catalog recovery", () => { + it("restores readability before the real config and schema repair chain", async () => { + await withOpenClawTestState({ scenario: "minimal" }, async (state) => { + const database = await seedState(state); + database.db.exec("DROP INDEX idx_task_runs_status;"); + await closeOpenClawStateDatabaseAsync(); + damageWorkshopIndex(database.path); + const runtime = { log: vi.fn(), error: vi.fn(), exit: vi.fn() }; + + await doctorCommand(runtime, { repair: true, nonInteractive: true }); + + expect(runtime.exit).not.toHaveBeenCalled(); + expect(mocks.outro).toHaveBeenCalledWith("Doctor complete."); + expect(runtime.log).toHaveBeenCalledWith( + "Removed dangling legacy Skill Workshop review index", + ); + const { DatabaseSync } = requireNodeSqlite(); + const repaired = new DatabaseSync(database.path, { readOnly: true }); + try { + expect( + repaired + .prepare("SELECT review_id, backup_id FROM skill_workshop_collection_reviews") + .all(), + ).toEqual([{ review_id: "review-preserved", backup_id: "backup-preserved" }]); + expect( + repaired + .prepare( + "SELECT name FROM sqlite_schema WHERE name = 'idx_skill_workshop_collection_reviews_workspace_time'", + ) + .get(), + ).toBeUndefined(); + expect( + repaired.prepare("SELECT name FROM pragma_index_info('idx_task_runs_status')").all(), + ).toEqual([{ name: "status" }]); + expect(readSqliteNumberPragma(repaired, "user_version")).toBe( + OPENCLAW_STATE_SCHEMA_VERSION, + ); + expect(repaired.prepare("PRAGMA integrity_check").all()).toEqual([ + { integrity_check: "ok" }, + ]); + } finally { + repaired.close(); + } + }); + }); + + it.each(["shared-schema", "custom-agent-schema", "external-owner"] as const)( + "refuses %s before changing the malformed source", + async (reason) => { + await withOpenClawTestState({ scenario: "minimal" }, async (state) => { + const database = await seedState(state); + if (reason === "shared-schema") { + database.db.exec(`PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION + 1};`); + } else if (reason === "custom-agent-schema") { + const customPath = state.path("custom", "sessions.sqlite"); + fs.mkdirSync(path.dirname(customPath)); + const { DatabaseSync } = requireNodeSqlite(); + const custom = new DatabaseSync(customPath); + custom.exec(` + PRAGMA user_version = ${OPENCLAW_AGENT_SCHEMA_VERSION + 1}; + CREATE TABLE schema_meta (meta_key TEXT PRIMARY KEY, agent_id TEXT); + INSERT INTO schema_meta VALUES ('primary', 'main'); + `); + custom.close(); + await state.writeConfig({ + agents: { ownership: "explicit", entries: { main: { workspace: state.workspaceDir } } }, + session: { store: customPath }, + gateway: { mode: "local" }, + plugins: { enabled: false }, + }); + } else { + claimOpenClawStateOwnership("fixture-manager", { + env: { ...state.env, OPENCLAW_SUPERVISOR_MODE: "external" }, + }); + } + await closeOpenClawStateDatabaseAsync(); + damageWorkshopIndex(database.path); + const before = fs.readFileSync(database.path); + const configBefore = fs.readFileSync(state.configPath); + + await expect( + doctorCommand( + { log: vi.fn(), error: vi.fn(), exit: vi.fn() }, + { repair: true, nonInteractive: true }, + ), + ).rejects.toThrow(reason === "external-owner" ? /externally supervised/ : /newer/); + + expect(fs.readFileSync(database.path)).toEqual(before); + expect(fs.readFileSync(state.configPath)).toEqual(configBefore); + expect(mocks.runContributions).not.toHaveBeenCalled(); + expect(mocks.outro).not.toHaveBeenCalledWith("Doctor complete."); + }); + }, + ); +}); diff --git a/src/flows/doctor-health.migration-refusal.test.ts b/src/flows/doctor-health.migration-refusal.test.ts index 5408d1c4448d..b5eeaeefda57 100644 --- a/src/flows/doctor-health.migration-refusal.test.ts +++ b/src/flows/doctor-health.migration-refusal.test.ts @@ -203,7 +203,7 @@ describe("Doctor maintenance admission", () => { }); describe("Doctor agent lease admission", () => { - it("admits the exact dangling Workshop index without mutating state", async () => { + it("reserves dangling Workshop index admission for Doctor without mutating state", async () => { await withOpenClawTestState({ scenario: "minimal" }, async (state) => { const opened = openOpenClawStateDatabase({ env: state.env }); const pathname = opened.path; @@ -231,8 +231,21 @@ describe("Doctor agent lease admission", () => { } const before = fs.readFileSync(pathname); - expect(() => assertNoOpenClawAgentDatabaseLeasesReadOnly({ env: state.env })).not.toThrow(); + expect(() => assertNoOpenClawAgentDatabaseLeasesReadOnly({ env: state.env })).toThrow( + /malformed database schema/, + ); expect(fs.readFileSync(pathname)).toEqual(before); + const doctor = await doctorMaintenance.beginDoctorMaintenance({ + options: { repair: true, nonInteractive: true }, + root: null, + runtime: { log: vi.fn(), error: vi.fn(), exit: vi.fn() }, + }); + try { + expect(doctor).toBeDefined(); + expect(fs.readFileSync(pathname)).toEqual(before); + } finally { + await doctor?.release(); + } }); }); diff --git a/src/flows/doctor-health.ts b/src/flows/doctor-health.ts index 8e4bfdbb3798..dbb4d0e5ad9d 100644 --- a/src/flows/doctor-health.ts +++ b/src/flows/doctor-health.ts @@ -33,9 +33,10 @@ const loadConfigModule = createLazyRuntimeModule(() => import("../config/config. async function assertDoctorDatabaseSchemasCompatible(scope?: "state") { const databasePreflight = await import("../state/openclaw-database-preflight.js"); - const [{ createConfigIO }, targets] = await Promise.all([ + const [{ createConfigIO }, targets, { openDoctorStateSchemaReadAdmission }] = await Promise.all([ import("../config/io.js"), import("../config/sessions/targets.js"), + import("../state/openclaw-state-db-doctor-schema.js"), ]); const snapshot = await createConfigIO({ env: { ...process.env }, @@ -46,6 +47,7 @@ async function assertDoctorDatabaseSchemasCompatible(scope?: "state") { const databaseSchemas = await databasePreflight.preflightOpenClawDatabaseSchemas({ env: process.env, scope, + openStateSchemaReadAdmission: openDoctorStateSchemaReadAdmission, configuredAgentDatabaseTargets: (registeredDatabases) => targets.resolveConfiguredAgentDatabaseTargets(cfg, { env: process.env, registeredDatabases }), configuredAgentDatabaseCandidatePaths: targets.resolveConfiguredAgentDatabaseCandidatePaths( @@ -160,6 +162,19 @@ async function runDoctorHealthFlowWithResult( json: options.json, }); + if (maintenance && (options.repair === true || options.yes === true)) { + const { repairOpenClawStateDatabaseReadabilityForDoctor } = + await import("../state/openclaw-state-db.js"); + // Restore catalog reads before config discovery; versioned migrations remain in its graph. + const readability = repairOpenClawStateDatabaseReadabilityForDoctor({ env: process.env }); + if (readability.warnings.length > 0) { + throw new Error(readability.warnings.join("\n")); + } + for (const change of readability.changes) { + effectiveRuntime.log(change); + } + } + // Keep side-effect-heavy legacy checks before structured contributions until fully migrated. const { maybeRepairUiProtocolFreshness } = await import("../commands/doctor-ui.js"); const { noteSourceInstallIssues } = await import("../commands/doctor-install.js"); diff --git a/src/infra/update-run-reader.ts b/src/infra/update-run-reader.ts index fe4e4b69471f..ffb1261ef744 100644 --- a/src/infra/update-run-reader.ts +++ b/src/infra/update-run-reader.ts @@ -1,5 +1,8 @@ import type { DatabaseSync } from "node:sqlite"; -import type { OpenClawStateDatabaseOptions } from "../state/openclaw-state-db-contract.js"; +import type { + OpenClawStateDatabaseOptions, + OpenClawStateSchemaReadAdmission, +} from "../state/openclaw-state-db-contract.js"; import { withExistingOpenClawStateDatabaseArtifactPreservingReadOnly, withExistingOpenClawStateDatabaseArtifactPreservingReadOnlyAsync, @@ -96,11 +99,13 @@ function readRuns(db: DatabaseSync, input: ListInput): UpdateRunRecord[] { export function listUpdateRuns( input: ListInput = {}, options: OpenClawStateDatabaseOptions = {}, + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission, ): UpdateRunRecord[] { return ( withExistingOpenClawStateDatabaseArtifactPreservingReadOnly( ({ db }) => readRuns(db, input), options, + openStateSchemaReadAdmission, ) ?? [] ); } diff --git a/src/state/openclaw-agent-db-lease.ts b/src/state/openclaw-agent-db-lease.ts index 9fdeaa935038..229cdedf064a 100644 --- a/src/state/openclaw-agent-db-lease.ts +++ b/src/state/openclaw-agent-db-lease.ts @@ -25,8 +25,10 @@ import { prepareAgentDeletionPathFence, } from "./agent-deletion-journal.js"; import { openClawStateDatabaseCache } from "./openclaw-state-db-cache.js"; -import type { OpenClawStateDatabaseOptions } from "./openclaw-state-db-contract.js"; -import { openDanglingWorkshopIndexReadAdmission } from "./openclaw-state-db-dangling-workshop-index.js"; +import type { + OpenClawStateDatabaseOptions, + OpenClawStateSchemaReadAdmission, +} from "./openclaw-state-db-contract.js"; import { runExistingOpenClawStateWriteTransaction } from "./openclaw-state-db-existing-write.js"; import { ensureAgentDatabaseLeaseSchema } from "./openclaw-state-db-schema-additive.js"; import { tableExists } from "./openclaw-state-db-schema-helpers.js"; @@ -350,6 +352,7 @@ function isAgentDatabaseLeaseStale(row: { /** Doctor holds both lifecycle coordinators before checking writers, without schema repair. */ export function assertNoOpenClawAgentDatabaseLeasesReadOnly( options: OpenClawStateDatabaseOptions = {}, + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission, ): void { const pathname = path.resolve(options.path ?? resolveOpenClawStateSqlitePath(options.env)); try { @@ -368,7 +371,7 @@ export function assertNoOpenClawAgentDatabaseLeasesReadOnly( const db = cached?.db ?? openNodeSqliteDatabase(pathname, { readOnly: true }); let closeSchemaReadAdmission: (() => void) | undefined; try { - closeSchemaReadAdmission = openDanglingWorkshopIndexReadAdmission(db); + closeSchemaReadAdmission = openStateSchemaReadAdmission?.(db); runWithSqliteBusyTimeout(db, 250, () => { if (!tableExists(db, "agent_database_leases")) { return; diff --git a/src/state/openclaw-database-preflight.dangling-workshop-index.test.ts b/src/state/openclaw-database-preflight.dangling-workshop-index.test.ts index 77fd1ecc8abf..e7efa0280a89 100644 --- a/src/state/openclaw-database-preflight.dangling-workshop-index.test.ts +++ b/src/state/openclaw-database-preflight.dangling-workshop-index.test.ts @@ -6,6 +6,7 @@ import { requireNodeSqlite } from "../infra/node-sqlite.js"; import { OPENCLAW_AGENT_SCHEMA_VERSION } from "./openclaw-agent-db-contract.js"; import { preflightOpenClawDatabaseSchemas } from "./openclaw-database-preflight.js"; import { OPENCLAW_STATE_SCHEMA_VERSION } from "./openclaw-state-db-contract.js"; +import { openDoctorStateSchemaReadAdmission } from "./openclaw-state-db-doctor-schema.js"; import { closeOpenClawStateDatabaseForTest, openOpenClawStateDatabase, @@ -16,7 +17,7 @@ const tempDirs = useAutoCleanupTempDirTracker(afterEach); afterEach(closeOpenClawStateDatabaseForTest); describe("dangling Workshop index preflight", () => { - it("admits the exact defect for Doctor without mutating the source", async () => { + it("admits the exact defect only for Doctor without mutating the source", async () => { const stateDir = tempDirs.make("openclaw-preflight-dangling-workshop-"); const env = { OPENCLAW_STATE_DIR: stateDir }; const statePath = openOpenClawStateDatabase({ env }).path; @@ -54,10 +55,17 @@ describe("dangling Workshop index preflight", () => { .map((name) => [name, fs.readFileSync(path.join(sourceDir, name))]); const before = snapshot(); + const runtime = await preflightOpenClawDatabaseSchemas({ env, scope: "state" }); + expect(runtime.indeterminate).toEqual([ + expect.objectContaining({ reason: expect.stringMatching(/openclaw doctor --fix/) }), + ]); + expect(snapshot()).toEqual(before); + await expect( preflightOpenClawDatabaseSchemas({ env, scope: "state", + openStateSchemaReadAdmission: openDoctorStateSchemaReadAdmission, supportedVersions: { state: OPENCLAW_STATE_SCHEMA_VERSION, agent: OPENCLAW_AGENT_SCHEMA_VERSION, diff --git a/src/state/openclaw-database-preflight.ts b/src/state/openclaw-database-preflight.ts index 435a8acaad4d..d2e01b9fe303 100644 --- a/src/state/openclaw-database-preflight.ts +++ b/src/state/openclaw-database-preflight.ts @@ -6,7 +6,11 @@ import { resolveUnsuffixedSqliteTargetFromSessionStorePath } from "../config/ses import { resolveConfiguredAgentDatabaseCandidatePaths } from "../config/sessions/targets.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { formatErrorMessage } from "../infra/errors.js"; -import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js"; +import { + clearNodeSqliteKyselyCacheForDatabase, + executeSqliteQuerySync, + getNodeSqliteKysely, +} from "../infra/kysely-sync.js"; import { openNodeSqliteDatabase, resolveImmutableSqliteFileUri } from "../infra/node-sqlite.js"; import { hasNodeErrorCode } from "../infra/path-guards.js"; import { assertSqliteIntegrityInWorker } from "../infra/sqlite-integrity-worker.js"; @@ -59,15 +63,13 @@ import { OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, OPENCLAW_STATE_SCHEMA_VERSION, } from "./openclaw-state-db-contract.js"; -import { - closeWorkshopIndexReadDatabase, - openDanglingWorkshopIndexReadAdmission, -} from "./openclaw-state-db-dangling-workshop-index.js"; +import type { OpenClawStateSchemaReadAdmission } from "./openclaw-state-db-contract.js"; import { assertOpenClawStateDatabaseOwner, assertOpenClawStateDatabaseForMaintenance, openClawStateMigrationAssertions, } from "./openclaw-state-db-maintenance.js"; +import { normalizeOpenClawStateSchemaReadError } from "./openclaw-state-db-schema-migration-required.js"; import { assertCanonicalStateSchemaShape } from "./openclaw-state-db-schema-repair.js"; import { readStateSchemaContentVersion, @@ -354,6 +356,7 @@ export async function preflightOpenClawDatabaseSchemas(options: { ) => readonly { agentId: string; path: string }[]); configuredAgentDatabaseCandidatePaths?: readonly string[]; agentAdmissionConfig?: OpenClawConfig; + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission; }): Promise { options.signal?.throwIfAborted(); const { @@ -397,7 +400,7 @@ export async function preflightOpenClawDatabaseSchemas(options: { stateDatabase = openNodeSqliteDatabase(stateSnapshot.location, { readOnly: true, }); - closeStateSchemaReadAdmission = openDanglingWorkshopIndexReadAdmission(stateDatabase); + closeStateSchemaReadAdmission = options.openStateSchemaReadAdmission?.(stateDatabase); stateDatabase.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`); const stateVersion = readSqliteUserVersion(stateDatabase); const contentVersion = @@ -490,19 +493,25 @@ export async function preflightOpenClawDatabaseSchemas(options: { } catch (error) { // Accepted stop must not turn cancellation or failed cleanup into a // warn-and-continue result that launches the remaining startup runtime. + const failure = normalizeOpenClawStateSchemaReadError(error, statePath); if (options.signal?.aborted || options.requireStartupMigrationReadiness) { - throw error; + throw failure; } result.indeterminate.push({ kind: "state", path: statePath, - reason: formatErrorMessage(error), + reason: formatErrorMessage(failure), }); return result; } finally { try { if (stateDatabase) { - closeWorkshopIndexReadDatabase(stateDatabase, closeStateSchemaReadAdmission); + try { + closeStateSchemaReadAdmission?.(); + } finally { + clearNodeSqliteKyselyCacheForDatabase(stateDatabase); + stateDatabase.close(); + } } } finally { await stateSnapshot?.cleanupAsync(); diff --git a/src/state/openclaw-state-db-contract.ts b/src/state/openclaw-state-db-contract.ts index 36a4b17f4f74..53de054a9e9f 100644 --- a/src/state/openclaw-state-db-contract.ts +++ b/src/state/openclaw-state-db-contract.ts @@ -2,6 +2,8 @@ import type { DatabaseSync } from "node:sqlite"; import type { SqliteWalMaintenance } from "../infra/sqlite-wal.js"; import type { DatabasePathIdentity } from "../infra/sqlite-worker-identity.js"; +export type OpenClawStateSchemaReadAdmission = (database: DatabaseSync) => (() => void) | undefined; + // v17 records one-use prepared worker capacity and node workspace ownership. // v16 makes Skill Workshop ownership directory-based instead of row-provenance-based. // v15 removes redundant agent/session projections from conversation bindings. diff --git a/src/state/openclaw-state-db-dangling-workshop-index.ts b/src/state/openclaw-state-db-dangling-workshop-index.ts deleted file mode 100644 index 9d3acaa80d42..000000000000 --- a/src/state/openclaw-state-db-dangling-workshop-index.ts +++ /dev/null @@ -1,107 +0,0 @@ -import type { DatabaseSync } from "node:sqlite"; -import { clearNodeSqliteKyselyCacheForDatabase } from "../infra/kysely-sync-cache-state.js"; - -const LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX = - "idx_skill_workshop_collection_reviews_workspace_time"; -const LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX_SQL = - "CREATE INDEX idx_skill_workshop_collection_reviews_workspace_time ON skill_workshop_collection_reviews(workspace_dir, create_time DESC, review_id DESC)"; - -function normalizeSqliteCatalogSql(sql: string): string { - return sql - .replace(/\s+/gu, " ") - .replace(/\s*([(),])\s*/gu, "$1") - .trim(); -} - -export function withSqliteWritableSchema(database: DatabaseSync, operation: () => T): T { - database.enableDefensive?.(false); - // sqlite-allow-raw -- Exact legacy catalog admission requires SQLite's writable-schema pragma. - database.exec("PRAGMA writable_schema = ON;"); - try { - return operation(); - } finally { - try { - // sqlite-allow-raw -- Always restore catalog parsing after the bounded legacy inspection. - database.exec("PRAGMA writable_schema = OFF;"); - } finally { - database.enableDefensive?.(true); - } - } -} - -/** Detect only the known v15 review index left behind after its column was retired. */ -export function hasDanglingSkillWorkshopCollectionReviewIndex(database: DatabaseSync): boolean { - return withSqliteWritableSchema(database, () => { - const rawIndex = database // sqlite-allow-raw -- Inspect the exact malformed catalog row before ordinary schema parsing. - .prepare( - "SELECT tbl_name, rootpage, sql FROM sqlite_schema WHERE type = 'index' AND name = ?", - ) - .get(LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX); - // SAFETY: the narrow catalog projection is validated field-by-field below. - const index = rawIndex as { tbl_name?: unknown; rootpage?: unknown; sql?: unknown } | undefined; - if ( - index?.tbl_name !== "skill_workshop_collection_reviews" || - typeof index.rootpage !== "number" || - index.rootpage <= 0 || - typeof index.sql !== "string" || - normalizeSqliteCatalogSql(index.sql) !== - normalizeSqliteCatalogSql(LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX_SQL) - ) { - return false; - } - const rawColumns = database // sqlite-allow-raw -- Validate physical columns without parsing the malformed index. - .prepare("PRAGMA table_info(skill_workshop_collection_reviews)") - .all(); - // SAFETY: PRAGMA table_info rows expose optional names compared as unknown values. - const columns = rawColumns as Array<{ name?: unknown }>; - return ( - columns.some((column) => column.name === "owner_agent_id") && - !columns.some((column) => column.name === "workspace_dir") - ); - }); -} - -/** Keep a read-only connection tolerant of the exact malformed legacy index. */ -export function openDanglingWorkshopIndexReadAdmission( - database: DatabaseSync, -): (() => void) | undefined { - if (!hasDanglingSkillWorkshopCollectionReviewIndex(database)) { - return undefined; - } - database.enableDefensive?.(false); - try { - // sqlite-allow-raw -- Hold exact legacy catalog admission for a bounded read-only operation. - database.exec("PRAGMA writable_schema = ON;"); - } catch (error) { - database.enableDefensive?.(true); - throw error; - } - let open = true; - return () => { - if (!open) { - return; - } - open = false; - try { - // sqlite-allow-raw -- Restore ordinary schema parsing before releasing the read-only handle. - database.exec("PRAGMA writable_schema = OFF;"); - } finally { - database.enableDefensive?.(true); - } - }; -} - -/** Restore schema parsing and release a private read handle even if either cleanup fails. */ -export function closeWorkshopIndexReadDatabase( - database: DatabaseSync, - closeAdmission?: () => void, -): void { - try { - closeAdmission?.(); - } finally { - clearNodeSqliteKyselyCacheForDatabase(database); - database.close(); - } -} - -export { LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX }; diff --git a/src/state/openclaw-state-db-doctor-schema.ts b/src/state/openclaw-state-db-doctor-schema.ts new file mode 100644 index 000000000000..6f3a805325a7 --- /dev/null +++ b/src/state/openclaw-state-db-doctor-schema.ts @@ -0,0 +1,93 @@ +import type { DatabaseSync } from "node:sqlite"; +import { LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX } from "./openclaw-state-db-schema-migration-required.js"; +const LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX_SQL = + "CREATE INDEX idx_skill_workshop_collection_reviews_workspace_time ON skill_workshop_collection_reviews(workspace_dir, create_time DESC, review_id DESC)"; + +function normalizeSqliteCatalogSql(sql: string): string { + return sql + .replace(/\s+/gu, " ") + .replace(/\s*([(),])\s*/gu, "$1") + .trim(); +} + +export function withSqliteWritableSchema(database: DatabaseSync, operation: () => T): T { + database.enableDefensive?.(false); + // sqlite-allow-raw -- Exact legacy catalog admission requires SQLite's writable-schema pragma. + database.exec("PRAGMA writable_schema = ON;"); + try { + return operation(); + } finally { + try { + // sqlite-allow-raw -- OFF retains the schema loaded while malformed rows were ignored. + database.exec("PRAGMA writable_schema = RESET;"); + } finally { + database.enableDefensive?.(true); + } + } +} + +/** Detect only the known v15 review index left behind after its column was retired. */ +export function hasDanglingSkillWorkshopCollectionReviewIndex(database: DatabaseSync): boolean { + return withSqliteWritableSchema(database, () => + inspectSkillWorkshopCollectionReviewIndex(database), + ); +} + +function inspectSkillWorkshopCollectionReviewIndex(database: DatabaseSync): boolean { + const rawIndex = database // sqlite-allow-raw -- Inspect the exact malformed catalog row before ordinary schema parsing. + .prepare("SELECT tbl_name, rootpage, sql FROM sqlite_schema WHERE type = 'index' AND name = ?") + .get(LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX); + // SAFETY: the narrow catalog projection is validated field-by-field below. + const index = rawIndex as { tbl_name?: unknown; rootpage?: unknown; sql?: unknown } | undefined; + if ( + index?.tbl_name !== "skill_workshop_collection_reviews" || + typeof index.rootpage !== "number" || + index.rootpage <= 0 || + typeof index.sql !== "string" || + normalizeSqliteCatalogSql(index.sql) !== + normalizeSqliteCatalogSql(LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX_SQL) + ) { + return false; + } + const rawColumns = database // sqlite-allow-raw -- Validate physical columns without parsing the malformed index. + .prepare("PRAGMA table_info(skill_workshop_collection_reviews)") + .all(); + // SAFETY: PRAGMA table_info rows expose optional names compared as unknown values. + const columns = rawColumns as Array<{ name?: unknown }>; + return ( + columns.some((column) => column.name === "owner_agent_id") && + !columns.some((column) => column.name === "workspace_dir") + ); +} + +/** Doctor can inspect the exact legacy defect before its repair transaction. */ +export function openDoctorStateSchemaReadAdmission( + database: DatabaseSync, +): (() => void) | undefined { + if (!hasDanglingSkillWorkshopCollectionReviewIndex(database)) { + return undefined; + } + database.enableDefensive?.(false); + try { + // sqlite-allow-raw -- Hold exact legacy catalog admission for a bounded read-only operation. + database.exec("PRAGMA writable_schema = ON;"); + } catch (error) { + database.enableDefensive?.(true); + throw error; + } + let open = true; + return () => { + if (!open) { + return; + } + open = false; + try { + // sqlite-allow-raw -- Restore ordinary schema parsing before releasing the read-only handle. + database.exec("PRAGMA writable_schema = RESET;"); + } finally { + database.enableDefensive?.(true); + } + }; +} + +export { LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX }; diff --git a/src/state/openclaw-state-db-maintenance.ts b/src/state/openclaw-state-db-maintenance.ts index 870f7707d0ec..82c98640471c 100644 --- a/src/state/openclaw-state-db-maintenance.ts +++ b/src/state/openclaw-state-db-maintenance.ts @@ -22,7 +22,7 @@ import { hasDanglingSkillWorkshopCollectionReviewIndex, LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX, withSqliteWritableSchema, -} from "./openclaw-state-db-dangling-workshop-index.js"; +} from "./openclaw-state-db-doctor-schema.js"; import { ensureColumn, tableExists, tableHasColumn } from "./openclaw-state-db-schema-helpers.js"; import { migrateJsonCanonicalWideRowsV13 } from "./openclaw-state-db-schema-v13-widerow.js"; import { @@ -541,10 +541,13 @@ export function runStateSchemaMigrationTransaction( pathname: string, migrate: () => T, transactionOptions: SqliteTransactionOptions, + prepareSchema?: () => void, ): T { return runSqliteImmediateTransactionSync( db, () => { + // Doctor restores catalog readability before the publication prelude reads it. + prepareSchema?.(); const publishedVersion = readSqliteUserVersion(db); const blocker = publishedVersion < OPENCLAW_STATE_SCHEMA_VERSION diff --git a/src/state/openclaw-state-db-open.ts b/src/state/openclaw-state-db-open.ts index be584fc3d063..50987991991e 100644 --- a/src/state/openclaw-state-db-open.ts +++ b/src/state/openclaw-state-db-open.ts @@ -30,10 +30,8 @@ import { OPENCLAW_STATE_SCHEMA_VERSION, type OpenClawStateDatabase, } from "./openclaw-state-db-contract.js"; -import { hasDanglingSkillWorkshopCollectionReviewIndex } from "./openclaw-state-db-dangling-workshop-index.js"; import { openTrackedStateDatabase } from "./openclaw-state-db-handle.js"; import { ensureOpenClawStatePermissions } from "./openclaw-state-db-permissions.js"; -import { OpenClawStateDatabaseSchemaMigrationRequiredError } from "./openclaw-state-db-schema-migration-required.js"; import { assertSupportedStateSchemaVersion, readStateSchemaMigrationVersion, @@ -85,12 +83,6 @@ export function openUnpublishedStateDatabase(params: { db, busyTimeoutMs, () => { - if (hasDanglingSkillWorkshopCollectionReviewIndex(db)) { - throw new OpenClawStateDatabaseSchemaMigrationRequiredError( - "legacy-workshop-review-index", - params.pathname, - ); - } assertSupportedStateSchemaVersion(db, params.pathname); assertStateDatabaseIntegrityBeforeMutation(db, params.pathname); configureSqlitePreSchemaPragmas(db, { busyTimeoutMs }); diff --git a/src/state/openclaw-state-db-prepared-reads.test.ts b/src/state/openclaw-state-db-prepared-reads.test.ts new file mode 100644 index 000000000000..8b6c4c610832 --- /dev/null +++ b/src/state/openclaw-state-db-prepared-reads.test.ts @@ -0,0 +1,33 @@ +import { constants } from "node:sqlite"; +import { expect, it } from "vitest"; +import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; +import { withExistingOpenClawStateDatabaseReadOnly } from "./openclaw-state-db-readonly.js"; +import { openOpenClawStateDatabase } from "./openclaw-state-db.js"; +import { assertOpenClawStateWriteAllowed } from "./openclaw-state-ownership.js"; + +it("keeps prepared queries reusable across ordinary reads and ownership checks", async () => { + await withOpenClawTestState({ label: "state-prepared-reads" }, async ({ env }) => { + const { db, path } = openOpenClawStateDatabase({ env }); + db.exec("CREATE TABLE prepared_read (value INTEGER); INSERT INTO prepared_read VALUES (42)"); + let readPreparations = 0; + db.setAuthorizer((action, table) => { + if (action === constants.SQLITE_READ && table === "prepared_read") { + readPreparations++; + } + return constants.SQLITE_OK; + }); + const read = db.prepare("SELECT value FROM prepared_read"); + expect(read.get()).toEqual({ value: 42 }); + expect(readPreparations).toBe(1); + + for (let iteration = 0; iteration < 3; iteration++) { + expect(withExistingOpenClawStateDatabaseReadOnly(() => read.get(), { env })).toEqual({ + value: 42, + }); + assertOpenClawStateWriteAllowed({ database: db, databasePath: path, env }); + expect(read.get()).toEqual({ value: 42 }); + } + + expect(readPreparations).toBe(1); + }); +}); diff --git a/src/state/openclaw-state-db-readonly.test.ts b/src/state/openclaw-state-db-readonly.test.ts index 4bfbb055d402..87d92f247d62 100644 --- a/src/state/openclaw-state-db-readonly.test.ts +++ b/src/state/openclaw-state-db-readonly.test.ts @@ -307,7 +307,7 @@ describe.each(["admission", "explicit", "async"] as const)("%s read-only state r expect(fs.readFileSync(options.path)).toEqual(before); }); }); - it("reads through the exact dangling Workshop index without changing its source", async () => { + it("requires Doctor for the exact dangling Workshop index without changing its source", async () => { await withTempDir("openclaw-state-readonly-dangling-workshop-", async (stateDir) => { const options = createOptions(stateDir); const opened = openOpenClawStateDatabase(options); @@ -339,9 +339,11 @@ describe.each(["admission", "explicit", "async"] as const)("%s read-only state r } const before = fs.readFileSync(options.path); - expect( - await readState(({ db }) => db.prepare("SELECT role FROM schema_meta").get(), options), - ).toEqual({ role: "global" }); + await expect( + Promise.resolve().then(() => + readState(({ db }) => db.prepare("SELECT role FROM schema_meta").get(), options), + ), + ).rejects.toThrow(/legacy-workshop-review-index.*openclaw doctor --fix/); expect(fs.readFileSync(options.path)).toEqual(before); }); }); diff --git a/src/state/openclaw-state-db-readonly.ts b/src/state/openclaw-state-db-readonly.ts index 9cf8e2b8ef63..52cbb45818aa 100644 --- a/src/state/openclaw-state-db-readonly.ts +++ b/src/state/openclaw-state-db-readonly.ts @@ -15,8 +15,8 @@ import { openClawStateDatabaseCache } from "./openclaw-state-db-cache.js"; import type { OpenClawStateDatabaseOptions, OpenClawStateDatabase, + OpenClawStateSchemaReadAdmission, } from "./openclaw-state-db-contract.js"; -import { openDanglingWorkshopIndexReadAdmission } from "./openclaw-state-db-dangling-workshop-index.js"; import { openOpenClawStateReadConnection } from "./openclaw-state-db-read-connection.js"; import { assertSupportedStateSchemaVersion } from "./openclaw-state-db-schema-version.js"; import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js"; @@ -215,18 +215,13 @@ function withOpenClawStateDatabaseReadOnlyIfOpen( return { reused: false }; } try { - const closeSchemaReadAdmission = openDanglingWorkshopIndexReadAdmission(opened.db); - try { - // Process-local terminal failures evict this handle. Persisted quarantine - // is checked on the next physical open so hot reads do not poll metadata. - // A newer build can migrate this file while the handle stays open, so the - // forward-compatibility gate still runs before any reused read. - assertSupportedStateSchemaVersion(opened.db, pathname); - observeOpenClawDatabaseMaintenanceResource(opened.db); - return { reused: true, value: operation(opened) }; - } finally { - closeSchemaReadAdmission?.(); - } + // Process-local terminal failures evict this handle. Persisted quarantine + // is checked on the next physical open so hot reads do not poll metadata. + // A newer build can migrate this file while the handle stays open, so the + // forward-compatibility gate still runs before any reused read. + assertSupportedStateSchemaVersion(opened.db, pathname); + observeOpenClawDatabaseMaintenanceResource(opened.db); + return { reused: true, value: operation(opened) }; } catch (error) { openClawStateDatabaseCache.evictOpenClawStateDatabaseAfterCorruption(opened, error); throw error; @@ -272,46 +267,26 @@ function openOpenClawStateReadOnlyLocation( source: string | PreparedSqliteReadOnlyLocation, ) { const connection = openOpenClawStateReadConnection(pathname, source); - const { db } = connection.database; - let closeSchemaReadAdmission: (() => void) | undefined; - const close = () => { - const errors: unknown[] = []; - let closed = false; - try { - closeSchemaReadAdmission?.(); - } catch (error) { - errors.push(error); - } - try { - closed = connection.close(); - } catch (error) { - errors.push(error); - } - if (errors.length === 1) { - throw errors[0]; - } - if (errors.length > 1) { - throw new AggregateError(errors, "Shared-state reader cleanup failed."); - } - return closed; - }; try { - closeSchemaReadAdmission = openDanglingWorkshopIndexReadAdmission(db); - assertSupportedStateSchemaVersion(db, pathname); + assertSupportedStateSchemaVersion(connection.database.db, pathname); } catch (error) { - close(); + connection.close(); throw error; } - return { database: connection.database, close }; + return connection; } function withOpenClawStateReadOnlyLocation( operation: (database: OpenClawStateReadOnlyDatabase) => T, pathname: string, source: string | PreparedSqliteReadOnlyLocation, + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission, ): T { - const opened = openOpenClawStateReadOnlyLocation(pathname, source); + const opened = openOpenClawStateReadConnection(pathname, source); + let closeAdmission: (() => void) | undefined; try { + closeAdmission = openStateSchemaReadAdmission?.(opened.database.db); + assertSupportedStateSchemaVersion(opened.database.db, pathname); const result = operation(opened.database); const location = typeof source === "string" ? source : source.location; if (location === pathname && isPromiseLike(result)) { @@ -319,7 +294,11 @@ function withOpenClawStateReadOnlyLocation( } return result; } finally { - opened.close(); + try { + closeAdmission?.(); + } finally { + opened.close(); + } } } @@ -397,7 +376,15 @@ export function withExistingOpenClawStateDatabaseReadOnly( export function withExistingOpenClawStateDatabaseArtifactPreservingReadOnly( operation: (database: OpenClawStateReadOnlyDatabase) => T, options: OpenClawStateDatabaseOptions = {}, + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission, ): T | undefined { + if (openStateSchemaReadAdmission) { + return withExistingOpenClawStateDatabaseCurrentReadOnly( + operation, + options, + openStateSchemaReadAdmission, + ); + } return withArtifactPreservingStateReads(() => withExistingOpenClawStateDatabaseReadOnly(operation, options), ); @@ -407,12 +394,16 @@ export function withExistingOpenClawStateDatabaseArtifactPreservingReadOnly( export function withExistingOpenClawStateDatabaseCurrentReadOnly( operation: (database: OpenClawStateReadOnlyDatabase) => T, options: OpenClawStateDatabaseOptions = {}, + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission, ): T | undefined { return stateSnapshotReads.exit(() => { const pathname = resolveReadOnlyPath(options); - const reused = withOpenClawStateDatabaseReadOnlyIfOpen(operation, pathname); - if (reused.reused) { - return reused.value; + // Maintenance admission belongs to a fresh private reader, never a cached writer. + if (!openStateSchemaReadAdmission) { + const reused = withOpenClawStateDatabaseReadOnlyIfOpen(operation, pathname); + if (reused.reused) { + return reused.value; + } } if (existingPathOrUndefined(pathname) === undefined) { return undefined; @@ -425,6 +416,7 @@ export function withExistingOpenClawStateDatabaseCurrentReadOnly( operation, pathname, prepareSqliteReadOnlyLocationSync(pathname), + openStateSchemaReadAdmission, ); }); } diff --git a/src/state/openclaw-state-db-repair.ts b/src/state/openclaw-state-db-repair.ts new file mode 100644 index 000000000000..11f728238092 --- /dev/null +++ b/src/state/openclaw-state-db-repair.ts @@ -0,0 +1,250 @@ +import { clearNodeSqliteKyselyCacheForDatabase } from "../infra/kysely-sync.js"; +import { openNodeSqliteDatabase } from "../infra/node-sqlite.js"; +import { setSqliteBusyTimeout } from "../infra/sqlite-busy-timeout.js"; +import { + repairCanonicalSqliteIndexes, + verifyAndRepairCanonicalSqliteIndexes, +} from "../infra/sqlite-index-schema.js"; +import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js"; +import { assertSqliteSchemaTablesPresent } from "../infra/sqlite-schema-contract.js"; +import { migrateSqliteSchemaToStrictInTransaction } from "../infra/sqlite-strict.js"; +import { runSqliteImmediateTransactionSync } from "../infra/sqlite-transaction.js"; +import { clearOpenClawDatabaseQuarantine } from "./openclaw-quarantine-store.js"; +import { repairAuditEventsSchema } from "./openclaw-state-db-audit-migration.js"; +import { clearOpenClawStateDatabaseOpenFailure } from "./openclaw-state-db-cache.js"; +import { + LAZY_ADDITIVE_STATE_TABLES, + OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + OPENCLAW_STATE_SCHEMA_VERSION, + OPENCLAW_STATE_STRICT_SCHEMA_VERSION, +} from "./openclaw-state-db-contract.js"; +import { assertCurrentStateRuntimeSchema } from "./openclaw-state-db-fast-path.js"; +import { + assertOpenClawStateDatabaseOwner, + markCurrentStateSchemaVersion, + openClawStateMigrationAssertions, + versionedStateMigrations, + runStateSchemaMigrationTransaction, + executeCanonicalStateSchema, + prepareStateDatabaseSchemaRepair, +} from "./openclaw-state-db-maintenance.js"; +import * as operatorApprovalMigration from "./openclaw-state-db-operator-approval-migration.js"; +import { ensureOpenClawStatePermissions } from "./openclaw-state-db-permissions.js"; +import { + ensureAdditiveStateColumns, + ensureFirstUseAdditiveStateColumnsForStrictMigration, +} from "./openclaw-state-db-schema-additive.js"; +import { tableExists } from "./openclaw-state-db-schema-helpers.js"; +import { + assertCanonicalStateSchemaShape, + dropLegacyStateTables, + migrateAgentDatabaseRelativePaths as migrateAgentPaths, + migrateWorkerPlacementExecutionModeSchema, + repairAgentDatabasesCompositePrimaryKey, + repairLegacyGatewayRestartHandoffsForStrictMigration, +} from "./openclaw-state-db-schema-repair.js"; +import { migrateSingletonStateFoldInV12 } from "./openclaw-state-db-schema-v12-foldin.js"; +import { + assertSupportedStateSchemaVersion, + readStateSchemaContentVersion, + readStateSchemaMigrationVersion, +} from "./openclaw-state-db-schema-version.js"; +import * as sessionWatchMigration from "./openclaw-state-db-session-watch-migration.js"; +import * as retirements from "./openclaw-state-db-table-retirements.js"; +import { recoverOrphanTaskDeliveryRows } from "./openclaw-state-db-task-delivery-recovery.js"; +import { describeAgentPathMigration } from "./openclaw-state-db.paths.js"; +import { + assertOpenClawStateWriteAllowed, + OpenClawStateOwnershipError, +} from "./openclaw-state-ownership.js"; +import { getOpenClawStateRuntimeSchema } from "./openclaw-state-schema-compatibility.js"; +import { OPENCLAW_STATE_SCHEMA_SQL } from "./openclaw-state-schema.js"; +import { UpdateSchemaRefusalError } from "./openclaw-update-schema-refusal.js"; + +export function repairStateSchema( + pathname: string, + env: NodeJS.ProcessEnv, + scope: "automatic" | "doctor" | "readability", +): { + changes: string[]; + warnings: string[]; +} { + ensureOpenClawStatePermissions(pathname, env); + // This private handle rebuilds referenced tables and is closed after repair. + const db = openNodeSqliteDatabase(pathname, { enableForeignKeyConstraints: false }); + const rebuiltIndexNames = new Set(); + let ownershipRefused = false; + try { + setSqliteBusyTimeout(db, OPENCLAW_SQLITE_BUSY_TIMEOUT_MS); + let repairAdmittedSchema: (() => string[]) | undefined; + if (scope === "automatic") { + assertSupportedStateSchemaVersion(db, pathname); + } else { + repairAdmittedSchema = prepareStateDatabaseSchemaRepair(db, pathname, env); + if (scope === "readability") { + return { + changes: runSqliteImmediateTransactionSync(db, repairAdmittedSchema, { + busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + databaseLabel: pathname, + operationLabel: "state.schema.readability-repair", + }), + warnings: [], + }; + } + } + const applied: string[] = []; + const changes = runStateSchemaMigrationTransaction( + db, + pathname, + () => { + if (!repairAdmittedSchema) { + assertOpenClawStateWriteAllowed({ database: db, databasePath: pathname, env }); + } + applied.push(...recoverOrphanTaskDeliveryRows(db, pathname)); + const previousVersion = readStateSchemaMigrationVersion(db); + const preAuditSchema = previousVersion === 1 && !tableExists(db, "audit_events"); + if (preAuditSchema) { + assertOpenClawStateDatabaseOwner(db, { pathname }); + } + if (previousVersion === OPENCLAW_STATE_SCHEMA_VERSION) { + for (const name of verifyAndRepairCanonicalSqliteIndexes( + db, + pathname, + OPENCLAW_STATE_SCHEMA_SQL, + { allowMissingColumns: true }, + )) { + rebuiltIndexNames.add(name); + } + // Current-schema doctor repair may normalize recognized columns or + // table options, but it must never recreate a missing table empty. + assertSqliteSchemaTablesPresent(db, pathname, OPENCLAW_STATE_SCHEMA_SQL, { + allowedMissingTables: LAZY_ADDITIVE_STATE_TABLES, + }); + } else { + openClawStateMigrationAssertions.get(previousVersion)?.(db, { pathname }); + assertSqliteIntegrity(db, pathname); + } + dropLegacyStateTables(db); + applied.push(...retirements.runRetiredStateTableMigrations(db, previousVersion)); + if (migrateSingletonStateFoldInV12(db, previousVersion)) { + applied.push("Folded singleton state tables into config_machine_state (v12)"); + } + if (migrateWorkerPlacementExecutionModeSchema(db, previousVersion)) { + applied.push("Migrated cloud worker placements to execution modes"); + } + applied.push( + ...describeAgentPathMigration(migrateAgentPaths(db, previousVersion, pathname)), + ); + if (repairAgentDatabasesCompositePrimaryKey(db)) { + applied.push(`Migrated shared state agent database registry primary key → agent_id,path`); + } + if (repairAuditEventsSchema(db)) { + applied.push( + `Migrated shared state audit event ledger → versioned message lifecycle schema`, + ); + } + applied.push(...operatorApprovalMigration.repairOperatorApprovalSchema(db)); + const needsSessionWatchMigration = + sessionWatchMigration.needsSessionWatchCursorProvenanceMigration(db, previousVersion); + const sessionWatchResult = sessionWatchMigration.migrateSessionWatchCursorProvenance(db); + if (needsSessionWatchMigration) { + applied.push( + `Migrated shared state session watch cursors → provenance column (${sessionWatchResult.migratedAmbientWatches} ambient, ${sessionWatchResult.removedLegacySentinels} sentinels removed)`, + ); + } + assertCanonicalStateSchemaShape(db, pathname); + // Recognized schema-1 stores predate audit; Doctor must finish their schema + // before its later read-only workspace and agent readers can consume it. + if (preAuditSchema || tableExists(db, "audit_events")) { + ensureAdditiveStateColumns(db); + for (const migration of versionedStateMigrations) { + if (migration.migrate(db, previousVersion)) { + applied.push(migration.applied); + } + } + executeCanonicalStateSchema(db, { + includeVersionLazyAdditiveTables: previousVersion !== OPENCLAW_STATE_SCHEMA_VERSION, + }); + if (previousVersion < OPENCLAW_STATE_STRICT_SCHEMA_VERSION) { + repairLegacyGatewayRestartHandoffsForStrictMigration(db); + ensureFirstUseAdditiveStateColumnsForStrictMigration(db); + } + const strictMigration = migrateSqliteSchemaToStrictInTransaction( + db, + getOpenClawStateRuntimeSchema({ + includeVersionLazyAdditiveTables: previousVersion !== OPENCLAW_STATE_SCHEMA_VERSION, + }), + { databaseLabel: pathname }, + ); + if (strictMigration.migratedTables.length > 0) { + applied.push( + `Migrated shared state tables to SQLite STRICT typing (${strictMigration.migratedTables.length})`, + ); + } + for (const name of repairCanonicalSqliteIndexes(db, pathname, OPENCLAW_STATE_SCHEMA_SQL, { + verifyPhysicalIntegrity: false, + })) { + rebuiltIndexNames.add(name); + } + } + markCurrentStateSchemaVersion(db, { + createMetadataIfMissing: previousVersion < OPENCLAW_STATE_SCHEMA_VERSION, + }); + if (readStateSchemaContentVersion(db) === OPENCLAW_STATE_SCHEMA_VERSION) { + assertCurrentStateRuntimeSchema(db, pathname); + } + if (rebuiltIndexNames.size > 0) { + applied.push(`Rebuilt canonical shared-state SQLite indexes (${rebuiltIndexNames.size})`); + } + return applied; + }, + { + busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + databaseLabel: pathname, + operationLabel: "state.schema.repair", + }, + () => { + applied.push(...(repairAdmittedSchema?.() ?? [])); + }, + ); + const quarantineCleared = clearOpenClawDatabaseQuarantine(pathname, { env }); + clearOpenClawStateDatabaseOpenFailure(pathname); + return { + changes, + warnings: quarantineCleared + ? [] + : [ + `Persisted quarantine record for ${pathname} could not be cleared; rerun openclaw doctor --fix so the repaired database is not refused again.`, + ], + }; + } catch (err) { + if (err instanceof UpdateSchemaRefusalError) { + throw err; + } + if (err instanceof OpenClawStateOwnershipError) { + ownershipRefused = true; + throw err; + } + // Reaching this catch inside doctor means repair itself refused or failed, + // so the runtime asserts' "run openclaw doctor --fix" advice is circular here. + const reason = String(err).replace( + /has a legacy ([a-z ]+) schema; run openclaw doctor --fix to migrate it\./u, + "has a legacy $1 schema; automatic repair refused the unrecognized schema shape.", + ); + return { + changes: [], + warnings: [`Failed migrating shared state database schema at ${pathname}: ${reason}`], + }; + } finally { + if (db.isOpen) { + clearNodeSqliteKyselyCacheForDatabase(db); + // Rollback cleanup may have closed the handle after an unrecoverable + // transaction failure; double-close throws ERR_INVALID_STATE and would + // discard the diagnostic warnings returned by the catch above. + db.close(); + } + if (!ownershipRefused) { + ensureOpenClawStatePermissions(pathname, env); + } + } +} diff --git a/src/state/openclaw-state-db-schema-migration-required.ts b/src/state/openclaw-state-db-schema-migration-required.ts index e182119c302a..3024413c0ce4 100644 --- a/src/state/openclaw-state-db-schema-migration-required.ts +++ b/src/state/openclaw-state-db-schema-migration-required.ts @@ -1,5 +1,8 @@ import { StartupMaintenanceRequiredError } from "../infra/startup-maintenance-required.js"; +export const LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX = + "idx_skill_workshop_collection_reviews_workspace_time"; + type OpenClawStateDatabaseSchemaMigrationRequiredKind = | "agent-databases-composite-primary-key" | "audit-events-v2" @@ -17,3 +20,21 @@ export class OpenClawStateDatabaseSchemaMigrationRequiredError extends StartupMa this.name = "OpenClawStateDatabaseSchemaMigrationRequiredError"; } } + +/** Runtime readers report malformed legacy state without entering repair mode. */ +export function normalizeOpenClawStateSchemaReadError(error: unknown, pathname: string): unknown { + if ( + error instanceof Error && + error.message.startsWith( + `malformed database schema (${LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX})`, + ) + ) { + const required = new OpenClawStateDatabaseSchemaMigrationRequiredError( + "legacy-workshop-review-index", + pathname, + ); + required.cause = error; + return required; + } + return error; +} diff --git a/src/state/openclaw-state-db-schema-version.ts b/src/state/openclaw-state-db-schema-version.ts index 5ca002d3d2d9..cec7d4681329 100644 --- a/src/state/openclaw-state-db-schema-version.ts +++ b/src/state/openclaw-state-db-schema-version.ts @@ -7,6 +7,7 @@ import { } from "../infra/sqlite-user-version.js"; import { OPENCLAW_STATE_SCHEMA_VERSION } from "./openclaw-state-db-contract.js"; import { tableExists, tableHasColumn } from "./openclaw-state-db-schema-helpers.js"; +import { normalizeOpenClawStateSchemaReadError } from "./openclaw-state-db-schema-migration-required.js"; import type { DB } from "./openclaw-state-db.generated.js"; // Read-only clients need schema admission without loading updater publication policy. @@ -111,16 +112,20 @@ export function readStateSchemaMigrationVersion(db: DatabaseSync): number { } export function assertSupportedStateSchemaVersion(db: DatabaseSync, pathname: string): number { - const userVersion = readSqliteUserVersion(db); - const contentVersion = - userVersion > OPENCLAW_STATE_SCHEMA_VERSION ? userVersion : readStateSchemaContentVersion(db); - if (contentVersion > OPENCLAW_STATE_SCHEMA_VERSION) { - throw createNewerSqliteSchemaVersionError( - "OpenClaw state database", - pathname, - contentVersion, - OPENCLAW_STATE_SCHEMA_VERSION, - ); + try { + const userVersion = readSqliteUserVersion(db); + const contentVersion = + userVersion > OPENCLAW_STATE_SCHEMA_VERSION ? userVersion : readStateSchemaContentVersion(db); + if (contentVersion > OPENCLAW_STATE_SCHEMA_VERSION) { + throw createNewerSqliteSchemaVersionError( + "OpenClaw state database", + pathname, + contentVersion, + OPENCLAW_STATE_SCHEMA_VERSION, + ); + } + return userVersion; + } catch (error) { + throw normalizeOpenClawStateSchemaReadError(error, pathname); } - return userVersion; } diff --git a/src/state/openclaw-state-db.test.ts b/src/state/openclaw-state-db.test.ts index 4408a6435f81..6ed73e1d6265 100644 --- a/src/state/openclaw-state-db.test.ts +++ b/src/state/openclaw-state-db.test.ts @@ -46,7 +46,7 @@ import { FIRST_USE_STATE_TABLES, OPENCLAW_STATE_SCHEMA_VERSION, } from "./openclaw-state-db-contract.js"; -import { hasDanglingSkillWorkshopCollectionReviewIndex } from "./openclaw-state-db-dangling-workshop-index.js"; +import { hasDanglingSkillWorkshopCollectionReviewIndex } from "./openclaw-state-db-doctor-schema.js"; import { prepareStateDatabaseSchemaRepair } from "./openclaw-state-db-maintenance.js"; import { ensureGitHubPublicationSchema } from "./openclaw-state-db-schema-additive.js"; import { OpenClawStateDatabaseSchemaMigrationRequiredError } from "./openclaw-state-db-schema-migration-required.js"; @@ -1758,56 +1758,80 @@ describe("openclaw state database", () => { ).toEqual({ review_id: "review-v15", backup_id: "backup-v15" }); }); - it("requires Doctor to repair the dangling v15 Workshop review index", () => { - const stateDir = createTempStateDir(); - const options = { env: { OPENCLAW_STATE_DIR: stateDir } }; - const databasePath = materializeCurrentStateDatabase(stateDir); - const { DatabaseSync } = requireNodeSqlite(); - const database = new DatabaseSync(databasePath); - database - .prepare( - `INSERT INTO skill_workshop_collection_reviews ( + it.each([16, OPENCLAW_STATE_SCHEMA_VERSION])( + "requires Doctor to repair the dangling Workshop review index in schema v%i", + (version) => { + const stateDir = createTempStateDir(); + const options = { env: { OPENCLAW_STATE_DIR: stateDir } }; + const databasePath = materializeCurrentStateDatabase(stateDir); + const { DatabaseSync } = requireNodeSqlite(); + const database = new DatabaseSync(databasePath); + database + .prepare( + `INSERT INTO skill_workshop_collection_reviews ( review_id, owner_agent_id, backup_id, create_time, kept_names_json, written_names_json, dropped_json ) VALUES ('review-preserved', 'main', 'backup-preserved', 1, '[]', '[]', '[]')`, - ) - .run(); - database.close(); - const rootpage = createDanglingSkillWorkshopReviewIndex(databasePath); + ) + .run(); + if (version === 16) { + removePreparedWorkerOwnershipColumns(database); + database.exec(` + PRAGMA user_version = 16; + UPDATE schema_meta SET schema_version = 16 WHERE meta_key = 'primary'; + `); + } + database.close(); + const rootpage = createDanglingSkillWorkshopReviewIndex(databasePath); - const defensiveProbe = new DatabaseSync(databasePath); - expect(hasDanglingSkillWorkshopCollectionReviewIndex(defensiveProbe)).toBe(true); - const schemaVersion = readSqliteNumberPragma(defensiveProbe, "schema_version"); - defensiveProbe.exec(`PRAGMA schema_version = ${schemaVersion + 1};`); - expect(readSqliteNumberPragma(defensiveProbe, "schema_version")).toBe(schemaVersion); - defensiveProbe.close(); + const defensiveProbe = new DatabaseSync(databasePath); + expect(hasDanglingSkillWorkshopCollectionReviewIndex(defensiveProbe)).toBe(true); + const schemaVersion = readSqliteNumberPragma(defensiveProbe, "schema_version"); + defensiveProbe.exec(`PRAGMA schema_version = ${schemaVersion + 1};`); + expect(readSqliteNumberPragma(defensiveProbe, "schema_version")).toBe(schemaVersion); + defensiveProbe.close(); - expect(() => openOpenClawStateDatabase(options)).toThrow( - /legacy-workshop-review-index.*openclaw doctor --fix/u, - ); - expect(readDanglingSkillWorkshopReviewIndex(databasePath)).toMatchObject({ rootpage }); + expect(() => openOpenClawStateDatabase(options)).toThrow( + /legacy-workshop-review-index.*openclaw doctor --fix/u, + ); + expect(() => repairOpenClawStateDatabaseSchemaIfNeeded(options)).toThrow( + /legacy-workshop-review-index.*openclaw doctor --fix/u, + ); + expect(readDanglingSkillWorkshopReviewIndex(databasePath)).toMatchObject({ rootpage }); - expect(repairOpenClawStateDatabaseSchema(options)).toEqual({ - changes: ["Removed dangling legacy Skill Workshop review index"], - warnings: [], - }); + expect(repairOpenClawStateDatabaseSchema(options)).toEqual({ + changes: + version === 16 + ? expect.arrayContaining([ + "Removed dangling legacy Skill Workshop review index", + "Recorded prepared worker ownership and one-use lifecycle (v17)", + ]) + : ["Removed dangling legacy Skill Workshop review index"], + warnings: [], + }); - const repaired = new DatabaseSync(databasePath, { readOnly: true }); - try { - expect( - repaired - .prepare( - "SELECT review_id, backup_id FROM skill_workshop_collection_reviews WHERE review_id = 'review-preserved'", - ) - .get(), - ).toEqual({ review_id: "review-preserved", backup_id: "backup-preserved" }); - expect(repaired.prepare("PRAGMA integrity_check").all()).toEqual([{ integrity_check: "ok" }]); - expect(readDanglingSkillWorkshopReviewIndex(databasePath)).toBeUndefined(); - } finally { - repaired.close(); - } - expect(() => openOpenClawStateDatabase(options)).not.toThrow(); - }); + const repaired = new DatabaseSync(databasePath, { readOnly: true }); + try { + expect( + repaired + .prepare( + "SELECT review_id, backup_id FROM skill_workshop_collection_reviews WHERE review_id = 'review-preserved'", + ) + .get(), + ).toEqual({ review_id: "review-preserved", backup_id: "backup-preserved" }); + expect(repaired.prepare("PRAGMA integrity_check").all()).toEqual([ + { integrity_check: "ok" }, + ]); + expect(readSqliteNumberPragma(repaired, "user_version")).toBe( + OPENCLAW_STATE_SCHEMA_VERSION, + ); + expect(readDanglingSkillWorkshopReviewIndex(databasePath)).toBeUndefined(); + } finally { + repaired.close(); + } + expect(() => openOpenClawStateDatabase(options)).not.toThrow(); + }, + ); it("does not repair a dangling Workshop index in a newer unsupported schema", () => { const stateDir = createTempStateDir(); diff --git a/src/state/openclaw-state-db.ts b/src/state/openclaw-state-db.ts index ce7cb0da3e65..69e342aa07ee 100644 --- a/src/state/openclaw-state-db.ts +++ b/src/state/openclaw-state-db.ts @@ -1,8 +1,5 @@ -// OpenClaw state database manages shared persisted state and migrations. import { existsSync } from "node:fs"; import type { DatabaseSync } from "node:sqlite"; -import { clearNodeSqliteKyselyCacheForDatabase } from "../infra/kysely-sync.js"; -import { openNodeSqliteDatabase } from "../infra/node-sqlite.js"; import { normalizeSqliteNonNegativeInteger, readSqliteBusyTimeout, @@ -16,7 +13,6 @@ import { verifyAndRepairCanonicalSqliteIndexes, } from "../infra/sqlite-index-schema.js"; import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js"; -import { assertSqliteSchemaTablesPresent } from "../infra/sqlite-schema-contract.js"; import { prepareSqliteReadOnlyLocation } from "../infra/sqlite-snapshot-source.js"; import { migrateSqliteSchemaToStrictInTransaction } from "../infra/sqlite-strict.js"; import type { SqliteTransactionOptions } from "../infra/sqlite-transaction.js"; @@ -27,26 +23,23 @@ import { } from "../infra/state-database-coordinator.js"; import { migrateLegacyCronRunLogsToTaskRuns } from "../infra/state-migrations.cron-run-logs.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; -import { clearOpenClawDatabaseQuarantine } from "./openclaw-quarantine-store.js"; import { getOpenClawDatabaseMaintenanceScope, observeOpenClawDatabaseMaintenanceResource, } from "./openclaw-state-db-async-lifecycle.js"; -import { repairAuditEventsSchema } from "./openclaw-state-db-audit-migration.js"; import { openClawStateDatabaseCache as stateDbCache, recordOpenClawStateDatabaseOpenFailure, - clearOpenClawStateDatabaseOpenFailure, } from "./openclaw-state-db-cache.js"; import { OPENCLAW_DATABASE_SCHEMA_DOCS_URL, - LAZY_ADDITIVE_STATE_TABLES, OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, OPENCLAW_STATE_SCHEMA_VERSION, OPENCLAW_STATE_STRICT_SCHEMA_VERSION, type OpenClawStateDatabase, type OpenClawStateDatabaseOptions, } from "./openclaw-state-db-contract.js"; +import { openDoctorStateSchemaReadAdmission } from "./openclaw-state-db-doctor-schema.js"; import { assertCurrentStateRuntimeSchema, isOpenClawStateSchemaFastPathEligible, @@ -54,7 +47,6 @@ import { } from "./openclaw-state-db-fast-path.js"; import { assertOpenClawStateDatabaseForMaintenance, - assertOpenClawStateDatabaseOwner, markCurrentStateSchemaVersion, openClawStateMigrationAssertions, resolveDatabasePath, @@ -62,25 +54,22 @@ import { runStateSchemaMigrationTransaction, writeCurrentStateSchemaMetadata, executeCanonicalStateSchema, - prepareStateDatabaseSchemaRepair, } from "./openclaw-state-db-maintenance.js"; import { openUnpublishedStateDatabase } from "./openclaw-state-db-open.js"; -import * as operatorApprovalMigration from "./openclaw-state-db-operator-approval-migration.js"; import { ensureOpenClawStatePermissions } from "./openclaw-state-db-permissions.js"; import { openOpenClawStateReadConnection } from "./openclaw-state-db-read-connection.js"; import { withExistingOpenClawStateDatabaseReadOnly } from "./openclaw-state-db-readonly.js"; +import { repairStateSchema } from "./openclaw-state-db-repair.js"; import { ensureAdditiveStateColumns, ensureFirstUseAdditiveStateColumnsForStrictMigration, } from "./openclaw-state-db-schema-additive.js"; -import { tableExists } from "./openclaw-state-db-schema-helpers.js"; import { type AgentDatabasePathMigrationSummary as AgentPathSummary, assertCanonicalStateSchemaShape, dropLegacyStateTables, migrateAgentDatabaseRelativePaths as migrateAgentPaths, migrateWorkerPlacementExecutionModeSchema, - repairAgentDatabasesCompositePrimaryKey, repairLegacyGatewayRestartHandoffsForStrictMigration, } from "./openclaw-state-db-schema-repair.js"; import { migrateSingletonStateFoldInV12 } from "./openclaw-state-db-schema-v12-foldin.js"; @@ -96,16 +85,14 @@ import { withOpenClawStateStartupCheckpointConnection, } from "./openclaw-state-db-startup-checkpoint.js"; import * as retirements from "./openclaw-state-db-table-retirements.js"; -import { recoverOrphanTaskDeliveryRows } from "./openclaw-state-db-task-delivery-recovery.js"; import { runCoordinatedStateTransaction, withSharedStateWriteCoordinator, } from "./openclaw-state-db-write-coordination.js"; -import { describeAgentPathMigration, warnAgentPathMigration } from "./openclaw-state-db.paths.js"; +import { warnAgentPathMigration } from "./openclaw-state-db.paths.js"; import { assertOpenClawStateWriteAllowed, isOpenClawStateWriteContentionError, - OpenClawStateOwnershipError, runWithOpenClawStateWriteAccess, } from "./openclaw-state-ownership.js"; import { getOpenClawStateRuntimeSchema } from "./openclaw-state-schema-compatibility.js"; @@ -114,7 +101,7 @@ import { type StateSchemaPublicationBlocker, } from "./openclaw-state-schema-publication.js"; import { OPENCLAW_STATE_SCHEMA_SQL } from "./openclaw-state-schema.js"; -import { UpdateSchemaRefusalError } from "./openclaw-update-schema-refusal.js"; + export { registerOpenClawStateDatabaseLifecycleListener } from "./openclaw-state-db-cache.js"; export { OPENCLAW_DATABASE_SCHEMA_DOCS_URL, OPENCLAW_SQLITE_BUSY_TIMEOUT_MS }; @@ -138,167 +125,6 @@ function assertOpenClawStateDatabaseFreshOpenAllowed( const stateDbLog = createSubsystemLogger("state/db"); const deferredStateDatabases = new WeakSet(); -function repairStateSchema(pathname: string, env: NodeJS.ProcessEnv) { - ensureOpenClawStatePermissions(pathname, env); - const db = openNodeSqliteDatabase(pathname); - const rebuiltIndexNames = new Set(); - let ownershipRefused = false; - try { - db.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`); - const repairAdmittedSchema = prepareStateDatabaseSchemaRepair(db, pathname, env); - db.exec("PRAGMA foreign_keys = OFF;"); - const changes = runStateSchemaMigrationTransaction( - db, - pathname, - () => { - const applied = repairAdmittedSchema(); - applied.push(...recoverOrphanTaskDeliveryRows(db, pathname)); - const previousVersion = readStateSchemaMigrationVersion(db); - const preAuditSchema = previousVersion === 1 && !tableExists(db, "audit_events"); - if (preAuditSchema) { - assertOpenClawStateDatabaseOwner(db, { pathname }); - } - if (previousVersion === OPENCLAW_STATE_SCHEMA_VERSION) { - for (const name of verifyAndRepairCanonicalSqliteIndexes( - db, - pathname, - OPENCLAW_STATE_SCHEMA_SQL, - { allowMissingColumns: true }, - )) { - rebuiltIndexNames.add(name); - } - // Current-schema doctor repair may normalize recognized columns or - // table options, but it must never recreate a missing table empty. - assertSqliteSchemaTablesPresent(db, pathname, OPENCLAW_STATE_SCHEMA_SQL, { - allowedMissingTables: LAZY_ADDITIVE_STATE_TABLES, - }); - } else { - openClawStateMigrationAssertions.get(previousVersion)?.(db, { pathname }); - assertSqliteIntegrity(db, pathname); - } - dropLegacyStateTables(db); - applied.push(...retirements.runRetiredStateTableMigrations(db, previousVersion)); - if (migrateSingletonStateFoldInV12(db, previousVersion)) { - applied.push("Folded singleton state tables into config_machine_state (v12)"); - } - if (migrateWorkerPlacementExecutionModeSchema(db, previousVersion)) { - applied.push("Migrated cloud worker placements to execution modes"); - } - applied.push( - ...describeAgentPathMigration(migrateAgentPaths(db, previousVersion, pathname)), - ); - if (repairAgentDatabasesCompositePrimaryKey(db)) { - applied.push(`Migrated shared state agent database registry primary key → agent_id,path`); - } - if (repairAuditEventsSchema(db)) { - applied.push( - `Migrated shared state audit event ledger → versioned message lifecycle schema`, - ); - } - applied.push(...operatorApprovalMigration.repairOperatorApprovalSchema(db)); - const needsSessionWatchMigration = - sessionWatchMigration.needsSessionWatchCursorProvenanceMigration(db, previousVersion); - const sessionWatchResult = sessionWatchMigration.migrateSessionWatchCursorProvenance(db); - if (needsSessionWatchMigration) { - applied.push( - `Migrated shared state session watch cursors → provenance column (${sessionWatchResult.migratedAmbientWatches} ambient, ${sessionWatchResult.removedLegacySentinels} sentinels removed)`, - ); - } - assertCanonicalStateSchemaShape(db, pathname); - // Recognized schema-1 stores predate audit; Doctor must finish their schema - // before its later read-only workspace and agent readers can consume it. - if (preAuditSchema || tableExists(db, "audit_events")) { - ensureAdditiveStateColumns(db); - for (const migration of versionedStateMigrations) { - if (migration.migrate(db, previousVersion)) { - applied.push(migration.applied); - } - } - executeCanonicalStateSchema(db, { - includeVersionLazyAdditiveTables: previousVersion !== OPENCLAW_STATE_SCHEMA_VERSION, - }); - if (previousVersion < OPENCLAW_STATE_STRICT_SCHEMA_VERSION) { - repairLegacyGatewayRestartHandoffsForStrictMigration(db); - ensureFirstUseAdditiveStateColumnsForStrictMigration(db); - } - const strictMigration = migrateSqliteSchemaToStrictInTransaction( - db, - getOpenClawStateRuntimeSchema({ - includeVersionLazyAdditiveTables: previousVersion !== OPENCLAW_STATE_SCHEMA_VERSION, - }), - { databaseLabel: pathname }, - ); - if (strictMigration.migratedTables.length > 0) { - applied.push( - `Migrated shared state tables to SQLite STRICT typing (${strictMigration.migratedTables.length})`, - ); - } - for (const name of repairCanonicalSqliteIndexes(db, pathname, OPENCLAW_STATE_SCHEMA_SQL, { - verifyPhysicalIntegrity: false, - })) { - rebuiltIndexNames.add(name); - } - } - markCurrentStateSchemaVersion(db, { - createMetadataIfMissing: previousVersion < OPENCLAW_STATE_SCHEMA_VERSION, - }); - if (readStateSchemaContentVersion(db) === OPENCLAW_STATE_SCHEMA_VERSION) { - assertCurrentStateRuntimeSchema(db, pathname); - } - if (rebuiltIndexNames.size > 0) { - applied.push(`Rebuilt canonical shared-state SQLite indexes (${rebuiltIndexNames.size})`); - } - return applied; - }, - { - busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, - databaseLabel: pathname, - operationLabel: "state.schema.repair", - }, - ); - const quarantineCleared = clearOpenClawDatabaseQuarantine(pathname, { env }); - clearOpenClawStateDatabaseOpenFailure(pathname); - return { - changes, - warnings: quarantineCleared - ? [] - : [ - `Persisted quarantine record for ${pathname} could not be cleared; rerun openclaw doctor --fix so the repaired database is not refused again.`, - ], - }; - } catch (err) { - if (err instanceof UpdateSchemaRefusalError) { - throw err; - } - if (err instanceof OpenClawStateOwnershipError) { - ownershipRefused = true; - throw err; - } - // Reaching this catch inside doctor means repair itself refused or failed, - // so the runtime asserts' "run openclaw doctor --fix" advice is circular here. - const reason = String(err).replace( - /has a legacy ([a-z ]+) schema; run openclaw doctor --fix to migrate it\./u, - "has a legacy $1 schema; automatic repair refused the unrecognized schema shape.", - ); - return { - changes: [], - warnings: [`Failed migrating shared state database schema at ${pathname}: ${reason}`], - }; - } finally { - if (db.isOpen) { - db.exec("PRAGMA foreign_keys = ON;"); - clearNodeSqliteKyselyCacheForDatabase(db); - // Rollback cleanup may have closed the handle after an unrecoverable - // transaction failure; double-close throws ERR_INVALID_STATE and would - // discard the diagnostic warnings returned by the catch above. - db.close(); - } - if (!ownershipRefused) { - ensureOpenClawStatePermissions(pathname, env); - } - } -} - export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabaseOptions = {}): { changes: string[]; warnings: string[]; @@ -309,9 +135,39 @@ export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabase return { changes: [], warnings: [] }; } return runWithOpenClawStateWriteAccess( - { databasePath: pathname, env }, + { + databasePath: pathname, + env, + openStateSchemaReadAdmission: openDoctorStateSchemaReadAdmission, + }, "state schema repair", - () => withStateSchemaFence({ databasePath: pathname }, () => repairStateSchema(pathname, env)), + () => + withStateSchemaFence({ databasePath: pathname }, () => + repairStateSchema(pathname, env, "doctor"), + ), + ); +} + +/** Make exact legacy catalog damage readable before Doctor loads config-dependent state. */ +export function repairOpenClawStateDatabaseReadabilityForDoctor( + options: OpenClawStateDatabaseOptions = {}, +): { changes: string[]; warnings: string[] } { + const env = options.env ?? process.env; + const pathname = resolveDatabasePath(options); + if (!existsSync(pathname)) { + return { changes: [], warnings: [] }; + } + return runWithOpenClawStateWriteAccess( + { + databasePath: pathname, + env, + openStateSchemaReadAdmission: openDoctorStateSchemaReadAdmission, + }, + "Doctor state readability repair", + () => + withStateSchemaFence({ databasePath: pathname }, () => + repairStateSchema(pathname, env, "readability"), + ), ); } @@ -333,7 +189,9 @@ export function repairOpenClawStateDatabaseSchemaIfNeeded( "state schema repair preflight/repair", () => needsOpenClawStateDatabaseSchemaRepair(pathname) - ? withStateSchemaFence({ databasePath: pathname }, () => repairStateSchema(pathname, env)) + ? withStateSchemaFence({ databasePath: pathname }, () => + repairStateSchema(pathname, env, "automatic"), + ) : { changes: [], warnings: [] }, ); } diff --git a/src/state/openclaw-state-ownership.ts b/src/state/openclaw-state-ownership.ts index 425f1e2787fb..13f0c3779b69 100644 --- a/src/state/openclaw-state-ownership.ts +++ b/src/state/openclaw-state-ownership.ts @@ -19,9 +19,12 @@ import { acquireStateDatabaseCoordinator, StateDatabaseCoordinatorContentionError, } from "../infra/state-database-coordinator.js"; -import { OPENCLAW_SQLITE_BUSY_TIMEOUT_MS } from "./openclaw-state-db-contract.js"; -import { openDanglingWorkshopIndexReadAdmission } from "./openclaw-state-db-dangling-workshop-index.js"; +import { + OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + type OpenClawStateSchemaReadAdmission, +} from "./openclaw-state-db-contract.js"; import { tableExists } from "./openclaw-state-db-schema-helpers.js"; +import { normalizeOpenClawStateSchemaReadError } from "./openclaw-state-db-schema-migration-required.js"; export const STATE_SUPERVISION_KEY = "gateway.supervision"; const MAX_OWNERSHIP_TIMESTAMP_MS = 8_640_000_000_000_000; @@ -120,8 +123,6 @@ export function inspectOpenClawStateOwnershipFromDatabase( databasePath: string, configMachineStateTableReady = false, ): OpenClawExternalStateOwnership | null { - database.enableDefensive?.(false); - database.exec("PRAGMA writable_schema = ON;"); try { if (!configMachineStateTableReady && !tableExists(database, "config_machine_state")) { return null; @@ -136,30 +137,27 @@ export function inspectOpenClawStateOwnershipFromDatabase( throw new OpenClawStateOwnershipMetadataError(databasePath, "reserved value is not text"); } return parseExternalOwnership(row.value_json, databasePath); - } finally { - try { - database.exec("PRAGMA writable_schema = OFF;"); - } finally { - database.enableDefensive?.(true); - } + } catch (error) { + throw normalizeOpenClawStateSchemaReadError(error, databasePath); } } function inspectOwnershipThroughConnection( location: string, databasePath: string, + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission, ): OpenClawExternalStateOwnership | null { const database = openNodeSqliteDatabase(location, { readOnly: true }); - let closeSchemaReadAdmission: (() => void) | undefined; + let closeAdmission: (() => void) | undefined; try { - closeSchemaReadAdmission = openDanglingWorkshopIndexReadAdmission(database); + closeAdmission = openStateSchemaReadAdmission?.(database); database.exec( `PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS}; PRAGMA query_only = ON; PRAGMA trusted_schema = OFF;`, ); return inspectOpenClawStateOwnershipFromDatabase(database, databasePath); } finally { try { - closeSchemaReadAdmission?.(); + closeAdmission?.(); } finally { database.close(); } @@ -177,7 +175,11 @@ function inspectJournalAwarePublicOwnership( } } -function inspectOwnershipWhileCoordinatorHeld(databasePath: string, busyTimeoutMs: number) { +function inspectOwnershipWhileCoordinatorHeld( + databasePath: string, + busyTimeoutMs: number, + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission, +) { const resolvedPath = path.resolve(databasePath); if (!existsSync(resolvedPath)) { return null; @@ -185,11 +187,17 @@ function inspectOwnershipWhileCoordinatorHeld(databasePath: string, busyTimeoutM // Write admission owns locking and recovery while the coordinator is held. // Inspect the live committed view without cloning a potentially busy family. const database = openNodeSqliteDatabase(resolveExistingSqliteFileUri(resolvedPath)); + let closeAdmission: (() => void) | undefined; try { + closeAdmission = openStateSchemaReadAdmission?.(database); database.exec(`PRAGMA busy_timeout = ${busyTimeoutMs}; PRAGMA trusted_schema = OFF;`); return inspectOpenClawStateOwnershipFromDatabase(database, resolvedPath); } finally { - database.close(); + try { + closeAdmission?.(); + } finally { + database.close(); + } } } @@ -238,6 +246,7 @@ function acquireOpenClawStateWriteAccess(options: { databasePath: string; busyTimeoutMs?: number; env?: NodeJS.ProcessEnv; + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission; }): { release: () => void } { const resolvedPath = path.resolve(options.databasePath); const busyTimeoutMs = normalizeSqliteNonNegativeInteger( @@ -248,7 +257,11 @@ function acquireOpenClawStateWriteAccess(options: { try { quarantineOrphanedSqliteSidecars(resolvedPath); assertOwnershipAllowsWrite( - inspectOwnershipWhileCoordinatorHeld(resolvedPath, busyTimeoutMs), + inspectOwnershipWhileCoordinatorHeld( + resolvedPath, + busyTimeoutMs, + options.openStateSchemaReadAdmission, + ), resolvedPath, options.env ?? process.env, ); @@ -274,7 +287,12 @@ function acquireOpenClawStateWriteAccess(options: { } export function runWithOpenClawStateWriteAccess( - options: { databasePath: string; busyTimeoutMs?: number; env?: NodeJS.ProcessEnv }, + options: { + databasePath: string; + busyTimeoutMs?: number; + env?: NodeJS.ProcessEnv; + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission; + }, operationLabel: string, operation: () => T, ): T { @@ -291,6 +309,7 @@ export async function assertOpenClawStateWriteAllowedAtPath(options: { env?: NodeJS.ProcessEnv; recoverOrphanedSidecars?: boolean; signal?: AbortSignal; + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission; }): Promise { options.signal?.throwIfAborted(); const databasePath = path.resolve(options.databasePath); @@ -317,7 +336,11 @@ export async function assertOpenClawStateWriteAllowedAtPath(options: { try { options.signal?.throwIfAborted(); assertOwnershipAllowsWrite( - inspectOwnershipThroughConnection(prepared.location, databasePath), + inspectOwnershipThroughConnection( + prepared.location, + databasePath, + options.openStateSchemaReadAdmission, + ), databasePath, env, ); diff --git a/test/scripts/docker-e2e-plan.test.ts b/test/scripts/docker-e2e-plan.test.ts index eccbe6d5e57c..8af1ccef977f 100644 --- a/test/scripts/docker-e2e-plan.test.ts +++ b/test/scripts/docker-e2e-plan.test.ts @@ -1412,6 +1412,29 @@ await import('./scripts/check-docker-e2e-boundaries.mts');`, } }); + it("pins opt-in Workshop Doctor recovery to published 9.4 without credentials", () => { + const plan = planFor({ + selectedLaneNames: ["published-upgrade-survivor"], + upgradeSurvivorBaselines: "2026.9.3 2026.9.4 2026.9.5", + upgradeSurvivorScenarios: "workshop-doctor-recovery", + }); + const name = "published-upgrade-survivor-2026.9.4-workshop-doctor-recovery"; + expect(plan.lanes.map(summarizeLane)).toEqual([ + publishedUpgradeSurvivorLane(name, "openclaw@2026.9.4", "workshop-doctor-recovery"), + ]); + expect(plan.requiredPrepublishPluginPackages).toEqual([]); + expect(plan.credentials).toEqual([]); + for (const alias of ["reported-issues", "far-reaching"]) { + expect( + planFor({ + selectedLaneNames: ["published-upgrade-survivor"], + upgradeSurvivorBaselines: "2026.9.4", + upgradeSurvivorScenarios: alias, + }).lanes.map((lane) => lane.name), + ).not.toContain(name); + } + }); + it("runs sibling-source canaries from published 9.4 without provider or registry fixtures", () => { const plan = planFor({ selectedLaneNames: ["published-upgrade-survivor"], diff --git a/test/scripts/upgrade-survivor-assertions.test.ts b/test/scripts/upgrade-survivor-assertions.test.ts index 19d81685b422..0b3b04c4543c 100644 --- a/test/scripts/upgrade-survivor-assertions.test.ts +++ b/test/scripts/upgrade-survivor-assertions.test.ts @@ -394,27 +394,32 @@ describe("upgrade recovery result assertions", () => { ); }); - it("accepts clean updates for baselines that already have consent", () => { - const result = { - status: "ok", - after: { version: "2026.8.1" }, - steps: [{ name: "global update", exitCode: 0 }], - }; - expect(runJsonAssertion("assert-successful-update-json", result, "2026.8.1").status).toBe(0); - expect( - runJsonAssertion( - "assert-successful-update-json", - { - ...result, - steps: [{ name: "global update", exitCode: 1 }], - }, - "2026.8.1", - ).status, - ).not.toBe(0); - expect( - runPrefixedJsonAssertion("assert-successful-update-json", result, "2026.8.1").status, - ).toBe(0); - }); + it.each(["base", "workshop-doctor-recovery"])( + "accepts clean updates for baselines that already have consent (%s)", + (scenario) => + withEnv({ OPENCLAW_UPGRADE_SURVIVOR_SCENARIO: scenario }, () => { + const result = { + status: "ok", + after: { version: "2026.8.1" }, + steps: [{ name: "global update", exitCode: 0 }], + }; + const update = runJsonAssertion("assert-successful-update-json", result, "2026.8.1"); + expect(update.status, update.stderr).toBe(0); + expect( + runJsonAssertion( + "assert-successful-update-json", + { + ...result, + steps: [{ name: "global update", exitCode: 1 }], + }, + "2026.8.1", + ).status, + ).not.toBe(0); + expect( + runPrefixedJsonAssertion("assert-successful-update-json", result, "2026.8.1").status, + ).toBe(0); + }), + ); describe("missing Codex migration update result", () => { const scenarioEnv = { diff --git a/test/scripts/upgrade-survivor-migration-order.test.ts b/test/scripts/upgrade-survivor-migration-order.test.ts index 8cb1c02c1804..04c740f17cc4 100644 --- a/test/scripts/upgrade-survivor-migration-order.test.ts +++ b/test/scripts/upgrade-survivor-migration-order.test.ts @@ -1,7 +1,17 @@ -import { spawnSync } from "node:child_process"; -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { execFileSync, spawnSync } from "node:child_process"; +import { cpSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; +import { DatabaseSync } from "node:sqlite"; import { afterEach, describe, expect, it } from "vitest"; +import { + assertWorkshopDoctorRepair, + assertWorkshopRecoveredUpgrade, + assertWorkshopUpdateRefusal, + captureWorkshopBaseline, + captureWorkshopCandidate, + completeWorkshopRecovery, + seedWorkshopIndex, +} from "../../scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs"; import { resolveTestNodeExecPath } from "../../src/test-utils/node-process.js"; import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; @@ -24,10 +34,22 @@ function runFirstHop(scenario: string, automatic: boolean) { CURRENT_PHASE="$1" shift case "$CURRENT_PHASE" in - update-candidate) + install-baseline) baseline_spec="$OPENCLAW_UPGRADE_SURVIVOR_BASELINE" ;; + prepare-workshop-baseline) printf 'baseline-doctor\\n' >>"$HOME/events" ;; + capture-workshop-candidate) printf 'candidate-identity\\n' >>"$HOME/events" ;; + seed-workshop-baseline-index|seed-workshop-candidate-index) printf 'seed\\n' >>"$HOME/events" ;; + update-candidate|update-workshop-recovered-state) printf 'update\\n' >>"$HOME/events" if [ "$FIXTURE_AUTOMATIC" = 1 ]; then touch "$HOME/migrated"; fi ;; + repair-workshop-baseline|repair-workshop-candidate) printf 'explicit-doctor\\n' >>"$HOME/events" ;; + assert-workshop-published-refusal) + printf 'refusal\\n' >>"$HOME/events" + if [ "$FIXTURE_AUTOMATIC" = 1 ]; then return 42; fi + ;; + assert-workshop-baseline-repair|assert-workshop-candidate-repair|assert-workshop-recovered-upgrade) + printf 'verify\\n' >>"$HOME/events" + ;; doctor) printf 'doctor\\n' >>"$HOME/events" touch "$HOME/migrated" @@ -59,7 +81,8 @@ trap 'case "$BASH_COMMAND" in "phase "*) install_fixture_phases ;; esac' DEBUG OPENCLAW_CONFIG_PATH: join(state, "openclaw.json"), OPENCLAW_UPGRADE_SURVIVOR_RUNTIME_ROOT: join(home, "runtime"), OPENCLAW_UPGRADE_SURVIVOR_SUMMARY_JSON: summary, - OPENCLAW_UPGRADE_SURVIVOR_BASELINE: "openclaw@2026.7.1-2", + OPENCLAW_UPGRADE_SURVIVOR_BASELINE: + scenario === "workshop-doctor-recovery" ? "openclaw@2026.9.4" : "openclaw@2026.7.1-2", OPENCLAW_UPGRADE_SURVIVOR_SCENARIO: scenario, BASH_ENV: prelude, FIXTURE_AUTOMATIC: automatic ? "1" : "0", @@ -94,4 +117,251 @@ describe.skipIf(process.platform === "win32")("survivor first-hop observation", expect(events).toEqual(["update", "observe", "doctor", "observe", "consent"]); expect(summary.status).toBe("passed"); }); + + it.each([false, true])( + "keeps published refusal before explicit Workshop recovery (unexpected success=%s)", + (unexpectedSuccess) => { + const { result, events, summary } = runFirstHop( + "workshop-doctor-recovery", + unexpectedSuccess, + ); + expect(result.status, result.stderr).toBe(unexpectedSuccess ? 42 : 0); + expect(events).toEqual([ + "baseline-doctor", + "candidate-identity", + "seed", + "refusal", + ...(unexpectedSuccess + ? [] + : ["explicit-doctor", "verify", "update", "verify", "seed", "explicit-doctor", "verify"]), + ]); + expect(summary.status).toBe(unexpectedSuccess ? "failed" : "passed"); + expect(summary.updateRecovery).toBeNull(); + expect(summary.firstHopPostCore.availability).toBe("unavailable"); + if (unexpectedSuccess) { + expect(summary.failure.phase).toBe("assert-workshop-published-refusal"); + } + }, + ); +}); + +const workshopIndex = "idx_skill_workshop_collection_reviews_workspace_time"; + +function workshopFixture() { + const root = tempDirs.make("survivor-workshop-evidence-"); + const state = join(root, "state"); + const artifacts = join(root, "artifacts"); + const packageRoot = join(root, "installed"); + const candidateRoot = join(root, "package"); + for (const directory of [ + join(state, "state"), + artifacts, + join(packageRoot, "dist"), + join(candidateRoot, "dist"), + ]) { + mkdirSync(directory, { recursive: true }); + } + for (const directory of [packageRoot, candidateRoot]) { + writeFileSync( + join(directory, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.9.4" }), + ); + writeFileSync( + join(directory, "dist", "build-info.json"), + JSON.stringify({ buildId: directory === packageRoot ? "baseline" : "candidate" }), + ); + } + const baseline = captureWorkshopBaseline(packageRoot, artifacts); + const tarball = join(root, "candidate.tgz"); + execFileSync("tar", ["-czf", tarball, "package"], { cwd: root }); + const candidate = captureWorkshopCandidate(tarball, artifacts, "2026.9.4"); + const filename = join(state, "state", "openclaw.sqlite"); + const initial = new DatabaseSync(filename); + initial.exec(`CREATE TABLE skill_workshop_collection_reviews ( + review_id TEXT PRIMARY KEY, owner_agent_id TEXT, backup_id TEXT, create_time INTEGER, + kept_names_json TEXT, written_names_json TEXT, dropped_json TEXT + ); CREATE TABLE unrelated_records (value TEXT); INSERT INTO unrelated_records VALUES ('preserved');`); + initial.close(); + const seeded = seedWorkshopIndex(state, artifacts, "baseline"); + writeFileSync( + join(artifacts, "update.json"), + JSON.stringify({ + ok: false, + error: { + type: "cli_error", + message: `SQLite integrity_check failed: Page ${seeded.rootpage}: never used`, + }, + }), + ); + return { state, artifacts, packageRoot, candidateRoot, baseline, candidate, filename }; +} + +type WorkshopIdentity = { version: string; buildInfoSha256: string }; +function recordProcess( + observations: string, + pid: number, + role: "update" | "doctor", + identity: WorkshopIdentity, + malformedAtStart: boolean, + updateInProgress: boolean, + exitCode: number, + nativeReceipt = true, +) { + mkdirSync(join(observations, "diagnostics"), { recursive: true }); + const witness = { + role, + identity, + malformedAtStart, + updateInProgress, + exitCode, + pid, + parentPid: 100, + }; + writeFileSync(join(observations, `workshop-process-${pid}.json`), JSON.stringify(witness)); + if (nativeReceipt) { + writeFileSync( + join(observations, "diagnostics", `process-${pid}-exited.json`), + JSON.stringify({ + role, + packageVersion: identity.version, + pid, + parentPid: witness.parentPid, + exitCode, + }), + ); + } +} + +// Simulate command outcomes to test evidence rejection; real repair belongs to the Docker proof. +function simulateWorkshopRepair(filename: string) { + const database = new DatabaseSync(filename); + database.enableDefensive?.(false); + database.exec("PRAGMA writable_schema = ON;"); + database + .prepare("UPDATE sqlite_schema SET sql = ? WHERE name = ?") + .run( + `CREATE INDEX ${workshopIndex} ON skill_workshop_collection_reviews(review_id, create_time DESC)`, + workshopIndex, + ); + const version = database.prepare("PRAGMA schema_version").get()?.schema_version; + database.exec( + `PRAGMA writable_schema = OFF; PRAGMA schema_version = ${Number(version) + 1}; DROP INDEX ${workshopIndex};`, + ); + database.close(); +} + +describe("Workshop Doctor recovery evidence", () => { + it.each(["refused", "unexpected-success", "unrelated-data-changed", "installed-build-changed"])( + "validates the published updater's %s outcome without accepting repair", + (outcome) => { + const fixture = workshopFixture(); + const observations = join(fixture.artifacts, "first-update"); + recordProcess( + observations, + 101, + "update", + fixture.baseline, + true, + false, + outcome === "unexpected-success" ? 0 : 1, + ); + if (outcome === "unrelated-data-changed") { + const database = new DatabaseSync(fixture.filename); + database.enableDefensive?.(false); + database.exec( + "PRAGMA writable_schema = ON; UPDATE unrelated_records SET value = 'changed';", + ); + database.close(); + } + if (outcome === "installed-build-changed") { + cpSync(fixture.candidateRoot, fixture.packageRoot, { recursive: true }); + } + const verify = () => + assertWorkshopUpdateRefusal( + fixture.state, + fixture.artifacts, + observations, + fixture.packageRoot, + outcome === "unexpected-success" ? 0 : 1, + ); + if (outcome === "refused") { + expect(verify()).toMatchObject({ + status: "refused-before-candidate", + automaticRepair: false, + }); + } else { + expect(verify).toThrow( + outcome === "unexpected-success" + ? /must refuse/ + : outcome === "unrelated-data-changed" + ? /changed state/ + : /changed installed build/, + ); + } + }, + ); + + it.each([ + "intact", + "review-changed", + "already-repaired", + "missing-receipt", + "same-version-wrong-build", + "update-marker", + ])("requires explicit candidate Doctor evidence: %s", (outcome) => { + const fixture = workshopFixture(); + simulateWorkshopRepair(fixture.filename); + if (outcome === "review-changed") { + const database = new DatabaseSync(fixture.filename); + database.exec("UPDATE skill_workshop_collection_reviews SET backup_id = 'wrong-backup';"); + database.close(); + } + const observations = join(fixture.artifacts, "candidate-doctor"); + recordProcess( + observations, + 102, + "doctor", + outcome === "same-version-wrong-build" ? fixture.baseline : fixture.candidate, + outcome !== "already-repaired", + outcome === "update-marker", + 0, + outcome !== "missing-receipt", + ); + const verify = () => + assertWorkshopDoctorRepair(fixture.state, fixture.artifacts, observations, "candidate"); + if (outcome === "intact") { + expect(verify()).toMatchObject({ status: "explicit-doctor-repaired" }); + } else { + expect(verify).toThrow( + outcome === "review-changed" ? /retained Workshop review/ : /Missing matching doctor/, + ); + } + }); + + it("keeps the refused first attempt distinct from both repairs and the recovered upgrade", () => { + const fixture = workshopFixture(); + const first = join(fixture.artifacts, "first-update"); + recordProcess(first, 101, "update", fixture.baseline, true, false, 1); + assertWorkshopUpdateRefusal(fixture.state, fixture.artifacts, first, fixture.packageRoot, 1); + simulateWorkshopRepair(fixture.filename); + const baselineDoctor = join(fixture.artifacts, "baseline-doctor"); + recordProcess(baselineDoctor, 102, "doctor", fixture.baseline, true, false, 0); + assertWorkshopDoctorRepair(fixture.state, fixture.artifacts, baselineDoctor, "baseline"); + cpSync(fixture.candidateRoot, fixture.packageRoot, { recursive: true }); + const upgraded = join(fixture.artifacts, "recovered-upgrade"); + recordProcess(upgraded, 103, "update", fixture.baseline, false, false, 0); + recordProcess(upgraded, 104, "doctor", fixture.candidate, false, true, 0); + assertWorkshopRecoveredUpgrade(fixture.state, fixture.artifacts, upgraded, fixture.packageRoot); + seedWorkshopIndex(fixture.state, fixture.artifacts, "candidate"); + const candidateDoctor = join(fixture.artifacts, "candidate-doctor"); + simulateWorkshopRepair(fixture.filename); + recordProcess(candidateDoctor, 105, "doctor", fixture.candidate, true, false, 0); + assertWorkshopDoctorRepair(fixture.state, fixture.artifacts, candidateDoctor, "candidate"); + expect(completeWorkshopRecovery(fixture.state, fixture.artifacts)).toMatchObject({ + firstAttempt: { status: "refused-before-candidate", automaticRepair: false }, + baselineDoctor: { status: "explicit-doctor-repaired" }, + upgrade: { status: "upgraded-after-explicit-repair" }, + candidateDoctor: { status: "explicit-doctor-repaired" }, + }); + }); }); diff --git a/test/scripts/upgrade-survivor-plugin-registry.test.ts b/test/scripts/upgrade-survivor-plugin-registry.test.ts index 1d694e8f1aa8..64384f9d281b 100644 --- a/test/scripts/upgrade-survivor-plugin-registry.test.ts +++ b/test/scripts/upgrade-survivor-plugin-registry.test.ts @@ -241,6 +241,24 @@ describe("standalone upgrade survivor plugin registry", () => { ); }); + it.each([ + ["custom-plugin-siblings", "openclaw@2026.9.4"], + ["abandoned-update", "openclaw@2026.9.2"], + ["workshop-doctor-recovery", "openclaw@2026.9.4"], + ])("follows the planner's no-registry decision for %s", (scenario, baseline) => { + const { captureDir, result } = runSurvivor({ + OPENCLAW_UPGRADE_SURVIVOR_SCENARIO: scenario, + OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC: baseline, + }); + + expect(result.status, result.stderr).toBe(0); + expect(existsSync(join(captureDir, "node-args"))).toBe(false); + expect(existsSync(join(captureDir, "docker-run-args"))).toBe(true); + expect(readFileSync(join(captureDir, "docker-args"), "utf8")).not.toContain( + "/tmp/openclaw-prepublish-plugin-registry", + ); + }); + it("does not prepare a registry for a published candidate", () => { const { captureDir, packageTarball, result } = runSurvivor({ OPENCLAW_CURRENT_PACKAGE_TGZ: undefined, diff --git a/test/vitest/vitest.database-worker-core-paths.mjs b/test/vitest/vitest.database-worker-core-paths.mjs index 642ec882068f..df24943cfa23 100644 --- a/test/vitest/vitest.database-worker-core-paths.mjs +++ b/test/vitest/vitest.database-worker-core-paths.mjs @@ -59,6 +59,7 @@ export const databaseWorkerCoreTestFiles = [ "src/agents/sessions/sdk.auth-migration.test.ts", "src/agents/subagents/completion/subagent-completion-admission.store.test.ts", "src/commands/doctor-maintenance.worker.test.ts", + "src/flows/doctor-health.dangling-workshop-index.test.ts", "src/entry.memory-json.test.ts", "src/gateway/server-methods/memory-search.test.ts", "src/logging/diagnostic-session-context.test.ts",