diff --git a/docs/reference/database-schemas/integrity-and-recovery.md b/docs/reference/database-schemas/integrity-and-recovery.md index bda09f9c0d23..4618b0cf5f83 100644 --- a/docs/reference/database-schemas/integrity-and-recovery.md +++ b/docs/reference/database-schemas/integrity-and-recovery.md @@ -126,6 +126,21 @@ The heartbeat proves ownership, not migration progress. A live but stuck mainten `SQLite read-only worker` failures append `code` and numeric SQLite `errcode` diagnostics when the underlying error supplies valid values, including through a bounded cause chain. Report the full code suffix when investigating a failure. Snapshot and integrity-child timeout errors include the applied budget and source file size; snapshot timeouts report an unknown size if the source stat failed. Integrity-child timeouts also retain `lastObservedPhase`. A generic `disk I/O error` or `SQLITE_IOERR` alone does not prove the disk is full. +### A legacy Workshop index prevents shared-state reads + +The `legacy-workshop-review-index` error requires `openclaw doctor --fix`. +Ordinary Gateway reads and automatic migration do not enter the legacy catalog +repair path. Healthy reads retain their prepared SQLite queries. + +With OpenClaw 2026.9.4, run Doctor before retrying `openclaw update`: the installed +updater checks database integrity before it can launch the target version. + +Doctor checks database versions and active owners before repairing the exact +known index. It restores catalog readability before loading dependent config +and plugin state, then continues its normal migration and verification flow. +The readability repair preserves review rows and schema-version markers; +unrecognized damage and newer databases remain refused. + ### The shared-state WAL keeps growing The running Gateway records the result of its existing WAL maintenance pass, diff --git a/scripts/e2e/lib/upgrade-survivor/assertions.mjs b/scripts/e2e/lib/upgrade-survivor/assertions.mjs index 552b37cfa097..68380cdea92c 100644 --- a/scripts/e2e/lib/upgrade-survivor/assertions.mjs +++ b/scripts/e2e/lib/upgrade-survivor/assertions.mjs @@ -31,6 +31,7 @@ const SCENARIOS = new Set([ "msteams-polls", "abandoned-update", "legacy-operator-state", + "workshop-doctor-recovery", "mobile-pairing-reconnect", "acpx-openclaw-tools-bridge", "feishu-channel", diff --git a/scripts/e2e/lib/upgrade-survivor/diagnostics.mjs b/scripts/e2e/lib/upgrade-survivor/diagnostics.mjs index 452a0543120d..499d6a78007e 100644 --- a/scripts/e2e/lib/upgrade-survivor/diagnostics.mjs +++ b/scripts/e2e/lib/upgrade-survivor/diagnostics.mjs @@ -25,6 +25,11 @@ const logNames = [ "post-update-validate.err", "doctor.log", "baseline-doctor.log", + "workshop-doctor-recovery.json", + "workshop-published-refusal.json", + "workshop-baseline-doctor.json", + "workshop-recovered-upgrade.json", + "workshop-candidate-doctor.json", "gateway.log", "gateway.log.doctor", "baseline-service-install.err", @@ -912,10 +917,14 @@ function publishedSuccessSummary(artifactRoot, sanitize) { return { phase: sanitize(event.phase, "phase"), status: event.status, at: event.at }; }), logs: Object.fromEntries( - ["update.json", "repair.json", "recovery-update.json"].map((name) => [ - name, - sanitize(readOwned(artifactRoot, name, name), name), - ]), + [ + "update.json", + "repair.json", + "recovery-update.json", + ...(snapshot.scenario === "workshop-doctor-recovery" + ? ["workshop-doctor-recovery.json", "baseline-doctor.log", "doctor.log"] + : []), + ].map((name) => [name, sanitize(readOwned(artifactRoot, name, name), name)]), ), omissions, }; diff --git a/scripts/e2e/lib/upgrade-survivor/run.sh b/scripts/e2e/lib/upgrade-survivor/run.sh index 7737ee72ac09..51e2cc5cf6f5 100644 --- a/scripts/e2e/lib/upgrade-survivor/run.sh +++ b/scripts/e2e/lib/upgrade-survivor/run.sh @@ -133,6 +133,7 @@ update_restart_source="" update_repair_required="0" initial_update_observation_root="" last_update_observation_root="" +workshop_doctor_observation_root="" idempotence_seconds="" run_completed="0" update_outcome="" @@ -247,6 +248,13 @@ validate_update_restart_mode() { return 1 ;; esac + if [ "$SCENARIO" = "workshop-doctor-recovery" ] && { + [ "$LIVE_OPENAI" != "0" ] || [ "$ROOT_MANAGED_VPS" != "0" ] || + [ "$UPDATE_RESTART_MODE" != "manual" ] || [ "$CANDIDATE_KIND" != "tarball" ]; + }; then + echo "workshop-doctor-recovery requires a candidate tarball, manual restart, and no live provider or managed VPS" >&2 + return 1 + fi } json_event() { @@ -344,6 +352,9 @@ const summary = { updateRecovery: process.env.SUMMARY_UPDATE_REPAIR_REQUIRED === "1" ? "capability-consent" : null, updateRestartSource: process.env.SUMMARY_UPDATE_RESTART_SOURCE || null, firstHopPostCore, + workshopDoctorRecovery: process.env.SUMMARY_SCENARIO === "workshop-doctor-recovery" + ? readJsonOrNull(path.join(path.dirname(process.env.SUMMARY_JSON), "workshop-doctor-recovery.json")) + : undefined, restartFixture: readJsonOrNull(process.env.SUMMARY_RESTART_FIXTURE), restartRuntimeFixture: readJsonOrNull(process.env.SUMMARY_RESTART_RUNTIME_FIXTURE), restartInference: process.env.SUMMARY_RESTART_INFERENCE || null, @@ -1408,9 +1419,14 @@ update_candidate() { if [ "$ROOT_MANAGED_VPS" != "1" ]; then update_env+=(OPENCLAW_ALLOW_ROOT=1) fi + local update_node_options="${NODE_OPTIONS:+$NODE_OPTIONS }--import=$PWD/scripts/e2e/lib/upgrade-survivor/diagnostics.mjs" + if [ "$SCENARIO" = "workshop-doctor-recovery" ]; then + update_node_options+=" --import=$PWD/scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs" + update_env+=("OPENCLAW_UPGRADE_SURVIVOR_WORKSHOP_STATE_DIR=$OPENCLAW_STATE_DIR") + fi update_env+=( "OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_ROOT=$observation_root" - "NODE_OPTIONS=${NODE_OPTIONS:+$NODE_OPTIONS }--import=$PWD/scripts/e2e/lib/upgrade-survivor/diagnostics.mjs" + "NODE_OPTIONS=$update_node_options" ) local update_status=0 if [ "$SCENARIO" = "recovery-cleanup" ]; then @@ -1466,6 +1482,25 @@ update_candidate() { fi } +assert_workshop_published_refusal() { + local refusal_exit=0 + update_candidate || refusal_exit=$? + node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs refusal \ + "$initial_update_observation_root" "$(package_root)" "$refusal_exit" || return "$?" + update_outcome="refused-before-candidate" +} + +run_workshop_doctor() { + local stage="$1" log="$2" + workshop_doctor_observation_root="$(mktemp -d "$ARTIFACT_ROOT/workshop-$stage-doctor.XXXXXX")" + local doctor_node_options="${NODE_OPTIONS:+$NODE_OPTIONS }--import=$PWD/scripts/e2e/lib/upgrade-survivor/diagnostics.mjs --import=$PWD/scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs" + openclaw_e2e_maybe_timeout "$COMMAND_TIMEOUT" env -u OPENCLAW_UPDATE_IN_PROGRESS \ + "OPENCLAW_UPGRADE_SURVIVOR_WORKSHOP_STATE_DIR=$OPENCLAW_STATE_DIR" \ + "OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_ROOT=$workshop_doctor_observation_root" \ + "NODE_OPTIONS=$doctor_node_options" \ + openclaw doctor --fix --non-interactive >"$log" 2>&1 +} + replace_historical_mobile_pairing_candidate() { local update_spec update_spec="$(candidate_update_spec)" @@ -1907,6 +1942,30 @@ phase validate-update-restart-mode validate_update_restart_mode phase reset-run-state reset_run_state phase install-baseline install_baseline phase initialize-state initialize_state +if [ "$SCENARIO" = "workshop-doctor-recovery" ]; then + if [ "$baseline_spec" != "openclaw@2026.9.4" ]; then + echo "workshop-doctor-recovery requires the exact published openclaw@2026.9.4 baseline" >&2 + exit 2 + fi + phase configure-workshop-baseline node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs configure + phase prepare-workshop-baseline openclaw_e2e_maybe_timeout "$COMMAND_TIMEOUT" openclaw doctor --fix --non-interactive + phase resolve-workshop-candidate resolve_candidate_version + phase capture-workshop-baseline node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs baseline "$(package_root)" + phase capture-workshop-candidate node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs candidate "$CANDIDATE_SPEC" "$candidate_version" + phase seed-workshop-baseline-index node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs seed baseline + phase assert-workshop-published-refusal assert_workshop_published_refusal + phase repair-workshop-baseline run_workshop_doctor baseline "$ARTIFACT_ROOT/baseline-doctor.log" + phase assert-workshop-baseline-repair node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs doctor "$workshop_doctor_observation_root" baseline + phase update-workshop-recovered-state update_candidate 1 + phase assert-workshop-recovered-upgrade node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs upgrade "$last_update_observation_root" "$(package_root)" + phase seed-workshop-candidate-index node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs seed candidate + phase repair-workshop-candidate run_workshop_doctor candidate "$DOCTOR_LOG" + phase assert-workshop-candidate-repair node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs doctor "$workshop_doctor_observation_root" candidate + phase assert-workshop-recovery node scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs complete + run_completed="1" + echo "Workshop Doctor recovery passed: published updater refused unchanged malformed state; explicit baseline Doctor, recovered upgrade, and explicit candidate Doctor succeeded." + exit 0 +fi if [ "$SCENARIO" = "custom-plugin-siblings" ]; then phase seed-sibling-plugin node scripts/e2e/lib/upgrade-survivor/custom-plugin-siblings.mjs seed phase validate-baseline-config validate_baseline_config diff --git a/scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs b/scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs new file mode 100644 index 000000000000..0df20d2407ff --- /dev/null +++ b/scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs @@ -0,0 +1,422 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { pathToFileURL } from "node:url"; +import { isMainThread } from "node:worker_threads"; + +const INDEX = "idx_skill_workshop_collection_reviews_workspace_time"; +const INDEX_SQL = `CREATE INDEX ${INDEX} ON skill_workshop_collection_reviews(workspace_dir, create_time DESC, review_id DESC)`; +const REVIEW = { + review_id: "survivor-workshop-review", + owner_agent_id: "main", + backup_id: "survivor-workshop-backup", + create_time: 1, + kept_names_json: '["retained-skill"]', + written_names_json: "[]", + dropped_json: "[]", +}; + +const readJson = (filename) => JSON.parse(fs.readFileSync(filename, "utf8")); +const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex"); + +function writeJson(filename, value) { + fs.mkdirSync(path.dirname(filename), { recursive: true }); + fs.writeFileSync(filename, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 }); +} + +function databasePath(stateDir) { + const filename = path.join(stateDir, "state", "openclaw.sqlite"); + assert(fs.statSync(filename).isFile(), "Published baseline did not create shared SQLite state"); + return filename; +} + +function buildIdentity(manifest, buildInfo) { + assert.equal(manifest.name, "openclaw"); + assert.equal(typeof manifest.version, "string"); + JSON.parse(buildInfo.toString("utf8")); + return { version: manifest.version, buildInfoSha256: sha256(buildInfo) }; +} + +function installedIdentity(packageRoot) { + return buildIdentity( + readJson(path.join(packageRoot, "package.json")), + fs.readFileSync(path.join(packageRoot, "dist", "build-info.json")), + ); +} + +export function captureWorkshopBaseline(packageRoot, artifactRoot) { + const identity = installedIdentity(packageRoot); + assert.equal(identity.version, "2026.9.4"); + writeJson(path.join(artifactRoot, "workshop-baseline.json"), identity); + return identity; +} + +export function captureWorkshopCandidate(tarball, artifactRoot, candidateVersion) { + const readPackedFile = (relative) => + execFileSync("tar", ["-xOf", tarball.replace(/^file:/u, ""), `package/${relative}`], { + maxBuffer: 1024 * 1024, + }); + const identity = buildIdentity( + JSON.parse(readPackedFile("package.json").toString("utf8")), + readPackedFile("dist/build-info.json"), + ); + assert.equal(identity.version, candidateVersion, "Candidate artifact version changed"); + const baseline = readJson(path.join(artifactRoot, "workshop-baseline.json")); + assert.notEqual( + identity.buildInfoSha256, + baseline.buildInfoSha256, + "Candidate must be a distinct build", + ); + writeJson(path.join(artifactRoot, "workshop-candidate.json"), identity); + return identity; +} + +function hasMalformedWorkshopIndex(filename) { + const database = new DatabaseSync(filename, { readOnly: true }); + try { + database.prepare("SELECT review_id FROM skill_workshop_collection_reviews LIMIT 1").get(); + return false; + } catch (error) { + if ( + error instanceof Error && + error.message.includes("malformed database schema") && + error.message.includes(INDEX) + ) { + return true; + } + throw error; + } finally { + database.close(); + } +} + +function inspectMalformedState(filename) { + assert(hasMalformedWorkshopIndex(filename), "Legacy Workshop fixture is not malformed"); + const database = new DatabaseSync(filename, { readOnly: true }); + try { + database.enableDefensive?.(false); + database.exec("PRAGMA writable_schema = ON;"); + const catalog = database + .prepare("SELECT type, name, tbl_name, rootpage, sql FROM sqlite_schema ORDER BY type, name") + .all(); + const index = catalog.find((entry) => entry.name === INDEX); + assert.equal(index?.sql, INDEX_SQL); + assert( + typeof index.rootpage === "number" && index.rootpage > 0, + "Malformed index must retain its physical b-tree", + ); + // Compare logical state, including the ledger, without mistaking WAL housekeeping for writes. + const digest = createHash("sha256").update(JSON.stringify(catalog)); + for (const pragma of ["user_version", "schema_version", "application_id"]) { + digest.update(JSON.stringify(database.prepare(`PRAGMA ${pragma}`).get())); + } + for (const table of catalog.filter((entry) => entry.type === "table")) { + const rows = database + .prepare(`SELECT * FROM "${table.name.replaceAll('"', '""')}"`) + .all() + .map((row) => JSON.stringify(row)) + .toSorted(); + digest.update(JSON.stringify([table.name, rows])); + } + const review = database + .prepare("SELECT * FROM skill_workshop_collection_reviews WHERE review_id = ?") + .get(REVIEW.review_id); + assert.deepEqual({ ...review }, REVIEW, "Retained Workshop review changed"); + return { + index: INDEX, + sql: index.sql, + rootpage: index.rootpage, + stateSha256: digest.digest("hex"), + }; + } finally { + database.close(); + } +} + +export function seedWorkshopIndex(stateDir, artifactRoot, stage) { + assert(["baseline", "candidate"].includes(stage)); + const filename = databasePath(stateDir); + const database = new DatabaseSync(filename); + try { + if (stage === "baseline") { + database + .prepare( + `INSERT INTO skill_workshop_collection_reviews ( + review_id, owner_agent_id, backup_id, create_time, kept_names_json, written_names_json, dropped_json + ) VALUES (?, ?, ?, ?, ?, ?, ?)`, + ) + .run(...Object.values(REVIEW)); + } else { + assert.deepEqual( + { + ...database + .prepare("SELECT * FROM skill_workshop_collection_reviews WHERE review_id = ?") + .get(REVIEW.review_id), + }, + REVIEW, + "Candidate reseeding must preserve the upgraded review", + ); + } + database.exec( + `CREATE INDEX ${INDEX} ON skill_workshop_collection_reviews(review_id, create_time DESC);`, + ); + database.enableDefensive?.(false); + database.exec("PRAGMA writable_schema = ON;"); + database + .prepare("UPDATE sqlite_schema SET sql = ? WHERE type = 'index' AND name = ?") + .run(INDEX_SQL, INDEX); + const { schema_version } = database.prepare("PRAGMA schema_version").get(); + database.exec(`PRAGMA writable_schema = OFF; PRAGMA schema_version = ${schema_version + 1};`); + } finally { + database.close(); + } + const seeded = inspectMalformedState(filename); + writeJson(path.join(artifactRoot, `workshop-${stage}-seeded.json`), seeded); + return seeded; +} + +function observeProcess() { + const role = process.argv[2]; + const stateDir = process.env.OPENCLAW_STATE_DIR; + const observations = process.env.OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_ROOT; + if ( + !isMainThread || + !["update", "doctor"].includes(role) || + !observations || + !stateDir || + stateDir !== process.env.OPENCLAW_UPGRADE_SURVIVOR_WORKSHOP_STATE_DIR + ) { + return; + } + let identity = null; + try { + let directory = path.dirname(fs.realpathSync(process.argv[1])); + // Installed CLI entrypoints are at the package root or inside dist. + for (let depth = 0; depth < 3; depth++, directory = path.dirname(directory)) { + const manifest = path.join(directory, "package.json"); + if (fs.existsSync(manifest) && readJson(manifest).name === "openclaw") { + identity = installedIdentity(directory); + break; + } + } + } catch { + // Missing identity rejects the evidence without changing the observed CLI. + } + const evidence = { + role, + pid: process.pid, + parentPid: process.ppid, + identity, + updateInProgress: process.env.OPENCLAW_UPDATE_IN_PROGRESS === "1", + malformedAtStart: hasMalformedWorkshopIndex(databasePath(stateDir)), + }; + const filename = path.join(observations, `workshop-process-${process.pid}.json`); + writeJson(filename, evidence); + process.once("exit", (exitCode) => writeJson(filename, { ...evidence, exitCode })); +} + +function processWitness(observations, identity, expected) { + assert.match(identity.buildInfoSha256, /^[a-f0-9]{64}$/u, "Missing build identity"); + const receipts = fs + .readdirSync(path.join(observations, "diagnostics")) + .filter((name) => /^process-\d+-exited\.json$/u.test(name)) + .map((name) => readJson(path.join(observations, "diagnostics", name))); + const witnesses = fs + .readdirSync(observations) + .filter((name) => /^workshop-process-\d+\.json$/u.test(name)) + .map((name) => readJson(path.join(observations, name))); + const witness = witnesses.find( + (entry) => + entry.identity?.version === identity.version && + entry.identity?.buildInfoSha256 === identity.buildInfoSha256 && + Object.entries(expected).every(([key, value]) => entry[key] === value) && + receipts.some( + (receipt) => + receipt.role === entry.role && + receipt.packageVersion === identity.version && + receipt.pid === entry.pid && + receipt.parentPid === entry.parentPid && + receipt.exitCode === entry.exitCode, + ), + ); + assert(witness, `Missing matching ${expected.role} build, source-state, and exit evidence`); + return witness; +} + +function assertRepairedState(stateDir) { + const database = new DatabaseSync(databasePath(stateDir), { readOnly: true }); + try { + assert.equal( + database.prepare("SELECT name FROM sqlite_schema WHERE name = ?").get(INDEX), + undefined, + "Doctor left the malformed Workshop index behind", + ); + assert.deepEqual( + { + ...database + .prepare("SELECT * FROM skill_workshop_collection_reviews WHERE review_id = ?") + .get(REVIEW.review_id), + }, + REVIEW, + "Doctor changed the retained Workshop review", + ); + assert.deepEqual( + database + .prepare("PRAGMA integrity_check") + .all() + .map((row) => row.integrity_check), + ["ok"], + ); + } finally { + database.close(); + } +} + +export function assertWorkshopUpdateRefusal( + stateDir, + artifactRoot, + observations, + packageRoot, + exitCode, +) { + assert.equal(exitCode, 1, "Published updater must refuse before installing the candidate"); + const baseline = readJson(path.join(artifactRoot, "workshop-baseline.json")); + assert.deepEqual( + installedIdentity(packageRoot), + baseline, + "Refused updater changed installed build", + ); + const seeded = readJson(path.join(artifactRoot, "workshop-baseline-seeded.json")); + assert.deepEqual( + inspectMalformedState(databasePath(stateDir)), + seeded, + "Refused updater changed state", + ); + const result = readJson(path.join(artifactRoot, "update.json")); + assert.equal(result.ok, false); + assert.equal(result.error?.type, "cli_error"); + assert(result.error.message.includes("SQLite integrity_check failed")); + assert(result.error.message.includes(`Page ${seeded.rootpage}: never used`)); + const updater = processWitness(observations, baseline, { + role: "update", + exitCode: 1, + malformedAtStart: true, + }); + const doctorStarted = fs + .readdirSync(path.join(observations, "diagnostics")) + .filter((name) => /^process-\d+-started\.json$/u.test(name)) + .some((name) => + ["doctor", "post-core"].includes(readJson(path.join(observations, "diagnostics", name)).role), + ); + assert.equal(doctorStarted, false, "Published refusal must precede the candidate handoff"); + const refusal = { + status: "refused-before-candidate", + automaticRepair: false, + stateSha256: seeded.stateSha256, + updater, + }; + writeJson(path.join(artifactRoot, "workshop-published-refusal.json"), refusal); + return refusal; +} + +export function assertWorkshopDoctorRepair(stateDir, artifactRoot, observations, stage) { + assert(["baseline", "candidate"].includes(stage)); + const identity = readJson(path.join(artifactRoot, `workshop-${stage}.json`)); + assertRepairedState(stateDir); + const doctor = processWitness(observations, identity, { + role: "doctor", + exitCode: 0, + malformedAtStart: true, + updateInProgress: false, + }); + const repair = { status: "explicit-doctor-repaired", doctor }; + writeJson(path.join(artifactRoot, `workshop-${stage}-doctor.json`), repair); + return repair; +} + +export function assertWorkshopRecoveredUpgrade(stateDir, artifactRoot, observations, packageRoot) { + const baseline = readJson(path.join(artifactRoot, "workshop-baseline.json")); + const candidate = readJson(path.join(artifactRoot, "workshop-candidate.json")); + assert.deepEqual( + installedIdentity(packageRoot), + candidate, + "Updater did not install the exact candidate", + ); + assertRepairedState(stateDir); + const updater = processWitness(observations, baseline, { + role: "update", + exitCode: 0, + malformedAtStart: false, + }); + const doctor = processWitness(observations, candidate, { + role: "doctor", + exitCode: 0, + malformedAtStart: false, + updateInProgress: true, + }); + const upgraded = { status: "upgraded-after-explicit-repair", updater, doctor }; + writeJson(path.join(artifactRoot, "workshop-recovered-upgrade.json"), upgraded); + return upgraded; +} + +export function completeWorkshopRecovery(stateDir, artifactRoot) { + assertRepairedState(stateDir); + const firstAttempt = readJson(path.join(artifactRoot, "workshop-published-refusal.json")); + const baselineDoctor = readJson(path.join(artifactRoot, "workshop-baseline-doctor.json")); + const upgrade = readJson(path.join(artifactRoot, "workshop-recovered-upgrade.json")); + const candidateDoctor = readJson(path.join(artifactRoot, "workshop-candidate-doctor.json")); + assert.equal(firstAttempt.status, "refused-before-candidate"); + assert.equal(firstAttempt.automaticRepair, false); + assert.equal(baselineDoctor.status, "explicit-doctor-repaired"); + assert.equal(upgrade.status, "upgraded-after-explicit-repair"); + assert.equal(candidateDoctor.status, "explicit-doctor-repaired"); + const result = { firstAttempt, baselineDoctor, upgrade, candidateDoctor }; + writeJson(path.join(artifactRoot, "workshop-doctor-recovery.json"), result); + return result; +} + +const direct = + process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url; +if (direct) { + const [mode, first, second, third] = process.argv.slice(2); + const stateDir = process.env.OPENCLAW_STATE_DIR; + const artifacts = process.env.OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_ROOT; + assert(stateDir && artifacts, "Missing isolated survivor paths"); + if (mode === "configure") { + assert(process.env.OPENCLAW_CONFIG_PATH, "Missing isolated config path"); + writeJson(process.env.OPENCLAW_CONFIG_PATH, { + gateway: { + mode: "local", + bind: "loopback", + auth: { mode: "token", token: "upgrade-survivor-token" }, + controlUi: { enabled: false }, + }, + agents: { + ownership: "explicit", + entries: { main: { workspace: path.join(stateDir, "workspace") } }, + }, + plugins: { enabled: false }, + }); + } else if (mode === "baseline") { + captureWorkshopBaseline(first, artifacts); + } else if (mode === "candidate") { + captureWorkshopCandidate(first, artifacts, second); + } else if (mode === "seed") { + seedWorkshopIndex(stateDir, artifacts, first); + } else if (mode === "refusal") { + assertWorkshopUpdateRefusal(stateDir, artifacts, first, second, Number(third)); + } else if (mode === "doctor") { + assertWorkshopDoctorRepair(stateDir, artifacts, first, second); + } else if (mode === "upgrade") { + assertWorkshopRecoveredUpgrade(stateDir, artifacts, first, second); + } else if (mode === "complete") { + completeWorkshopRecovery(stateDir, artifacts); + } else { + throw new Error(`Unknown Workshop recovery fixture mode: ${mode}`); + } +} else { + observeProcess(); +} diff --git a/scripts/e2e/upgrade-survivor-docker.sh b/scripts/e2e/upgrade-survivor-docker.sh index 463b736add8f..fef670b3ec9b 100755 --- a/scripts/e2e/upgrade-survivor-docker.sh +++ b/scripts/e2e/upgrade-survivor-docker.sh @@ -218,6 +218,14 @@ if [ "$SCENARIO" = "abandoned-update" ] && { exit 1 fi +if [ "$SCENARIO" = "workshop-doctor-recovery" ] && { + [ "${OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE:-0}" != "1" ] || + [ "$UPDATE_RESTART_MODE" != "manual" ] || [ "$ROOT_MANAGED_VPS" != "0" ] || [ "$LIVE_OPENAI" != "0" ]; +}; then + echo "workshop-doctor-recovery requires the published baseline, manual restart, and no live provider" >&2 + exit 1 +fi + resolve_lane_artifact_suffix() { if [ -n "${OPENCLAW_DOCKER_ALL_LANE_NAME:-}" ]; then printf "%s" "$OPENCLAW_DOCKER_ALL_LANE_NAME" @@ -327,17 +335,31 @@ if [ "${OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE:-0}" = "1" ]; then CANDIDATE_SPEC="$(normalize_npm_candidate "$CANDIDATE_RAW")" fi - if [ "$CANDIDATE_IS_CURRENT" = "1" ] && [ "$SCENARIO" != "custom-plugin-siblings" ] && [ -z "${OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR:-}" ]; then - AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT="$( - mktemp -d "${TMPDIR:-/tmp}/openclaw-upgrade-survivor-plugin-registry.XXXXXX" + if [ "$CANDIDATE_IS_CURRENT" = "1" ] && [ -z "${OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR:-}" ]; then + registry_required="$( + OPENCLAW_DOCKER_ALL_LANES=published-upgrade-survivor \ + OPENCLAW_DOCKER_ALL_TIMINGS=0 \ + OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS="$BASELINE_SPEC" \ + OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS="$SCENARIO" \ + node "$HARNESS_ROOT_DIR/scripts/test-docker-all.mjs" --plan-json | node -e ' + const plan = JSON.parse(require("node:fs").readFileSync(0, "utf8")); + const required = plan.needs?.prepublishPluginRegistry; + if (typeof required !== "boolean") throw new Error("Docker planner omitted plugin registry requirements"); + process.stdout.write(required ? "1" : "0"); + ' )" - OPENCLAW_DOCKER_ALL_LANES=published-upgrade-survivor \ - OPENCLAW_DOCKER_ALL_LOG_DIR="$AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT" \ - OPENCLAW_DOCKER_ALL_TIMINGS=0 \ - OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS="$BASELINE_SPEC" \ - OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS="$SCENARIO" \ - node "$HARNESS_ROOT_DIR/scripts/test-docker-all.mjs" --prepare-plugin-registry - export OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR="$AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT/prepublish-plugin-registry" + if [ "$registry_required" = "1" ]; then + AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT="$( + mktemp -d "${TMPDIR:-/tmp}/openclaw-upgrade-survivor-plugin-registry.XXXXXX" + )" + OPENCLAW_DOCKER_ALL_LANES=published-upgrade-survivor \ + OPENCLAW_DOCKER_ALL_LOG_DIR="$AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT" \ + OPENCLAW_DOCKER_ALL_TIMINGS=0 \ + OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS="$BASELINE_SPEC" \ + OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS="$SCENARIO" \ + node "$HARNESS_ROOT_DIR/scripts/test-docker-all.mjs" --prepare-plugin-registry + export OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR="$AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT/prepublish-plugin-registry" + fi fi if [ -n "$PACKAGE_TGZ" ]; then diff --git a/scripts/lib/docker-e2e-plan.mts b/scripts/lib/docker-e2e-plan.mts index 829f325273c3..749146b5dd82 100644 --- a/scripts/lib/docker-e2e-plan.mts +++ b/scripts/lib/docker-e2e-plan.mts @@ -133,6 +133,10 @@ const UPGRADE_SURVIVOR_RUNTIME_COMPANION_PACKAGES = ["@openclaw/codex"]; // Pre-protocol catalogs are content-addressed. Unknown legacy blocks fail // closed instead of requiring a dependency or reimplementing a JavaScript parser. const LEGACY_UPGRADE_SURVIVOR_SCENARIO_CATALOGS = new Map([ + [ + "7d9d7520c2c34d51fff78e542a7f539b77080bfd04648438e95aec4af3fe362e", + "base msteams-polls abandoned-update legacy-operator-state workshop-doctor-recovery mobile-pairing-reconnect acpx-openclaw-tools-bridge feishu-channel bootstrap-persona channel-post-core-restore codex-allowlist-survival plugin-deps-cleanup configured-plugin-installs missing-configured-plugin-migration custom-plugin-siblings stale-source-plugin-shadow prerelease-plugin-registry tilde-log-path meeting-transcripts-sqlite versioned-runtime-deps cron-scheduled-authority sqlite-volume recovery-cleanup auth-profile-v2026-7-2-beta-5 watchos-direct-node", + ], [ "5e8821538f3722fdf0dc3b3917ae639853f305e4c7a9b84febb8905601a9b5c7", "base msteams-polls abandoned-update legacy-operator-state mobile-pairing-reconnect acpx-openclaw-tools-bridge feishu-channel bootstrap-persona channel-post-core-restore codex-allowlist-survival plugin-deps-cleanup configured-plugin-installs missing-configured-plugin-migration custom-plugin-siblings stale-source-plugin-shadow prerelease-plugin-registry tilde-log-path meeting-transcripts-sqlite versioned-runtime-deps cron-scheduled-authority sqlite-volume recovery-cleanup auth-profile-v2026-7-2-beta-5 watchos-direct-node", @@ -682,7 +686,12 @@ export function requiredPrepublishPluginPackagesForLanes(poolLanes: DockerE2eLan requiredPackages.add(packageName); } const scenario = upgradeSurvivorScenarioForLane(poolLane); - if (!scenario || scenario === "abandoned-update" || scenario === "custom-plugin-siblings") { + if ( + !scenario || + scenario === "abandoned-update" || + scenario === "custom-plugin-siblings" || + scenario === "workshop-doctor-recovery" + ) { continue; } if (scenario === "legacy-operator-state") { diff --git a/scripts/lib/upgrade-survivor-policy.mjs b/scripts/lib/upgrade-survivor-policy.mjs index fef94051c684..b56cb92e96b4 100644 --- a/scripts/lib/upgrade-survivor-policy.mjs +++ b/scripts/lib/upgrade-survivor-policy.mjs @@ -3,6 +3,7 @@ const UPGRADE_SURVIVOR_SCENARIOS = Object.freeze([ "msteams-polls", "abandoned-update", "legacy-operator-state", + "workshop-doctor-recovery", "mobile-pairing-reconnect", "acpx-openclaw-tools-bridge", "feishu-channel", @@ -39,7 +40,11 @@ const scenarioMinimumBaselines = new Map([ // These black-box scenarios are implemented entirely by the current trusted // release harness and treat the selected tree only as the package under test. -const TRUSTED_HARNESS_OWNED_SCENARIOS = new Set(["mobile-pairing-reconnect", "abandoned-update"]); +const TRUSTED_HARNESS_OWNED_SCENARIOS = new Set([ + "mobile-pairing-reconnect", + "abandoned-update", + "workshop-doctor-recovery", +]); export function isTrustedHarnessOwnedUpgradeSurvivorScenario(scenario) { return TRUSTED_HARNESS_OWNED_SCENARIOS.has(scenario); @@ -55,6 +60,7 @@ const aggregateScenarios = UPGRADE_SURVIVOR_SCENARIOS.filter( scenario !== "msteams-polls" && scenario !== "abandoned-update" && scenario !== "missing-configured-plugin-migration" && + scenario !== "workshop-doctor-recovery" && scenario !== "mobile-pairing-reconnect" && scenario !== "watchos-direct-node" && scenario !== "prerelease-plugin-registry" && @@ -153,6 +159,9 @@ export function supportsUpgradeSurvivorScenarioAtBaseline(scenario, baselineSpec if (scenario === "abandoned-update" || scenario === "missing-configured-plugin-migration") { return baselineSpec === "openclaw@2026.9.2"; } + if (scenario === "workshop-doctor-recovery") { + return baselineSpec === "openclaw@2026.9.4"; + } const minimumBaseline = scenarioMinimumBaselines.get(scenario); return ( !minimumBaseline || diff --git a/src/cli/run-main.exit.test.ts b/src/cli/run-main.exit.test.ts index d9434e83c8cb..f468901456ae 100644 --- a/src/cli/run-main.exit.test.ts +++ b/src/cli/run-main.exit.test.ts @@ -2449,15 +2449,22 @@ describe("runCli exit behavior", () => { expect(startProxyMock).toHaveBeenCalledWith(undefined); }); - it("reads source-only proxy config before doctor lint owns plugin-aware validation", async () => { + it.each([ + ["lint", ["--lint", "--json"]], + ["repair", ["--fix", "--non-interactive"]], + ["diagnosis", []], + ])("reads source-only proxy config before Doctor %s owns state access", async (_mode, args) => { tryRouteCliMock.mockResolvedValueOnce(true); - readSourceConfigBestEffortMock.mockResolvedValueOnce({ proxy: { selected: "doctor-lint" } }); + readSourceConfigBestEffortMock.mockResolvedValueOnce({ proxy: { selected: "doctor" } }); + loadConfigMock.mockImplementation(() => { + throw new Error("Shared state requires Doctor repair"); + }); - await runCli(["node", "openclaw", "doctor", "--lint", "--json"]); + await runCli(["node", "openclaw", "doctor", ...args]); expect(readSourceConfigBestEffortMock).toHaveBeenCalledOnce(); expect(loadConfigMock).not.toHaveBeenCalled(); - expect(startProxyMock).toHaveBeenCalledWith({ selected: "doctor-lint" }); + expect(startProxyMock).toHaveBeenCalledWith({ selected: "doctor" }); }); it.each([ diff --git a/src/cli/run-main.ts b/src/cli/run-main.ts index 66a2b67e8e8c..7e49179f9bb2 100644 --- a/src/cli/run-main.ts +++ b/src/cli/run-main.ts @@ -22,7 +22,6 @@ import type { PluginCliLoadSession } from "../plugins/cli-registry-loader.js"; import { createPluginCache, getPluginCache, withPluginCache } from "../plugins/plugin-cache.js"; import { resolveCliArgvInvocation } from "./argv-invocation.js"; import { - hasFlag, normalizeGeneratedHelpCommandArgv, normalizeRootHelpTargetArgv, normalizeRootLogLevelArgv, @@ -1209,7 +1208,7 @@ async function runCliWithPreparedOutputMode( const useSourceOnlyBestEffortConfig = !(await isCurrentRuntimeSupported()) || normalizedInvocation.primary === "update" || - (normalizedInvocation.primary === "doctor" && hasFlag(normalizedArgv, "--lint")); + normalizedInvocation.primary === "doctor"; const readBestEffortCliConfig = async (): Promise => { if (!bestEffortConfigPromise) { bestEffortConfigPromise = import("../config/io.js").then(async (configIo) => { diff --git a/src/commands/doctor-maintenance.finish-revalidation.test.ts b/src/commands/doctor-maintenance.finish-revalidation.test.ts index 7fddda2e05d9..84458657f04c 100644 --- a/src/commands/doctor-maintenance.finish-revalidation.test.ts +++ b/src/commands/doctor-maintenance.finish-revalidation.test.ts @@ -1,18 +1,26 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; import path from "node:path"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import type { GatewayService } from "../daemon/service.js"; import { createMockGatewayService, mockSystemAccountHome } from "../daemon/service.test-helpers.js"; +import { openNodeSqliteDatabase } from "../infra/node-sqlite.js"; import * as updateRunDriver from "../infra/update-run-driver.js"; import { readUpdateRunDriver } from "../infra/update-run-driver.js"; import { createUpdateRun, getUpdateRun, + listUpdateRuns, recordUpdateRunPhase, recordUpdateRunStep, finishUpdateRun, } from "../infra/update-run-ledger.js"; -import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { OPENCLAW_STATE_SCHEMA_VERSION } from "../state/openclaw-state-db-contract.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; import { withEnvAsync } from "../test-utils/env.js"; import { mockProcessPlatform } from "../test-utils/vitest-spies.js"; import { beginDoctorMaintenance } from "./doctor-maintenance.js"; @@ -102,10 +110,18 @@ type Continuation = | "lost-before-restart" | "dead-before-restart" | "terminal-dead-before-restart"; +type LegacyCatalog = + | "exact" + | "unknown" + | "future-version" + | "future-content" + | "conflict-on-recheck" + | "different-state"; async function runDoctorFinishForStoppedUnit( scenario: StoppedUnitState, continuation?: Continuation, + legacyCatalog?: LegacyCatalog, ): Promise<{ finishError: unknown; restartCalls: number; @@ -176,6 +192,79 @@ async function runDoctorFinishForStoppedUnit( ); } } + let assertCatalogUnchanged = () => {}; + let assertPreStopArtifactsUnchanged = () => {}; + let activateCompetingUpdate: (() => void) | undefined; + if (legacyCatalog) { + const lateRun = + legacyCatalog === "conflict-on-recheck" + ? createUpdateRun({ trigger: "cli", origin: { driver: readUpdateRunDriver() } }) + : undefined; + if (lateRun) { + finishUpdateRun(lateRun.runId, { status: "skipped" }); + } + const pathname = openOpenClawStateDatabase().path; + closeOpenClawStateDatabaseForTest(); + const db = openNodeSqliteDatabase(pathname); + try { + db.exec( + "CREATE INDEX idx_skill_workshop_collection_reviews_workspace_time ON skill_workshop_collection_reviews(review_id, create_time DESC);", + ); + if (legacyCatalog === "future-version") { + db.exec(`PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION + 1}`); + } + if (legacyCatalog === "future-content") { + db.prepare( + "INSERT INTO config_machine_state (state_key, value_json, updated_at_ms) VALUES ('state.schema.contentVersion', ?, 1)", + ).run(String(OPENCLAW_STATE_SCHEMA_VERSION + 1)); + } + db.enableDefensive?.(false); + db.exec("PRAGMA writable_schema = ON"); + db.prepare("UPDATE sqlite_schema SET sql = ? WHERE type = 'index' AND name = ?").run( + `CREATE INDEX idx_skill_workshop_collection_reviews_workspace_time ON skill_workshop_collection_reviews(${legacyCatalog === "unknown" ? "unexpected_column" : "workspace_dir"}, create_time DESC, review_id DESC)`, + "idx_skill_workshop_collection_reviews_workspace_time", + ); + const schema = db.prepare("PRAGMA schema_version").get() as { schema_version: number }; + db.exec( + `PRAGMA writable_schema = OFF; PRAGMA schema_version = ${schema.schema_version + 1}`, + ); + } finally { + db.close(); + } + const readArtifacts = () => + [pathname, `${pathname}-wal`, `${pathname}-shm`].map((file) => + fs.existsSync(file) + ? createHash("sha256").update(fs.readFileSync(file)).digest("hex") + : undefined, + ); + const beforeArtifacts = readArtifacts(); + const beforeCatalog = fs.readFileSync(pathname); + assertCatalogUnchanged = () => + expect(fs.readFileSync(pathname).equals(beforeCatalog)).toBe(true); + assertPreStopArtifactsUnchanged = () => expect(readArtifacts()).toEqual(beforeArtifacts); + expect(() => listUpdateRuns()).toThrow( + legacyCatalog === "future-version" + ? /uses newer schema version/ + : /legacy-workshop-review-index.*doctor --fix/, + ); + assertPreStopArtifactsUnchanged(); + if (lateRun) { + activateCompetingUpdate = () => { + const writer = openNodeSqliteDatabase(pathname); + try { + writer.enableDefensive?.(false); + writer.exec("PRAGMA writable_schema = ON"); + writer + .prepare( + "UPDATE update_runs SET status = 'running', phase = 'validating', finished_at_ms = NULL WHERE run_id = ?", + ) + .run(lateRun.runId); + } finally { + writer.close(); + } + }; + } + } mockProcessPlatform("linux"); let running = continuation !== "parked" && @@ -191,7 +280,9 @@ async function runDoctorFinishForStoppedUnit( "--port", "18789", ], - environment: { HOME: home }, + environment: { + HOME: legacyCatalog === "different-state" ? path.join(home, "other") : home, + }, }; const restart = vi.fn(async () => { if (scenario === "restart-failed") { @@ -206,8 +297,11 @@ async function runDoctorFinishForStoppedUnit( hasInstalledDefinition: async () => true, isLoaded: async () => scenario === "retained", readCommand: async (_env, opts) => { - if (continuation === "lost-before-stop" && ++commandReads === 2 && runId) { - createUpdateRun({ trigger: "cli", origin: { driver: readUpdateRunDriver() } }); + if (++commandReads === 2) { + activateCompetingUpdate?.(); + if (continuation === "lost-before-stop" && runId) { + createUpdateRun({ trigger: "cli", origin: { driver: readUpdateRunDriver() } }); + } } if ( stopObserved && @@ -241,6 +335,7 @@ async function runDoctorFinishForStoppedUnit( : { status: "stopped" }; }, stop: vi.fn(async () => { + assertPreStopArtifactsUnchanged(); mocks.stops += 1; running = false; stopObserved = true; @@ -259,8 +354,18 @@ async function runDoctorFinishForStoppedUnit( error: () => {}, exit: () => {}, }, + }).finally(() => { + if (!activateCompetingUpdate) { + assertCatalogUnchanged(); + if (mocks.stops === 0) { + assertPreStopArtifactsUnchanged(); + } + } }); expect(maintenance).toBeDefined(); + if (legacyCatalog) { + expect(() => maintenance?.run(() => listUpdateRuns())).toThrow(); + } if (continuation === "lost-before-restart" && runId) { createUpdateRun({ trigger: "cli", origin: { driver: readUpdateRunDriver() } }); } @@ -282,20 +387,63 @@ async function runDoctorFinishForStoppedUnit( } catch (error) { finishError = error; } + assertCatalogUnchanged(); + const savedRun = runId && !legacyCatalog ? getUpdateRun(runId) : undefined; return { finishError, restartCalls: restart.mock.calls.length, logs, - takeoverSteps: runId - ? (getUpdateRun(runId)?.steps.filter((step) => step.step === "finalize:repair-takeover") - .length ?? 0) - : 0, - runStatus: runId ? getUpdateRun(runId)?.status : undefined, + takeoverSteps: + savedRun?.steps.filter((step) => step.step === "finalize:repair-takeover").length ?? 0, + runStatus: savedRun?.status, }; }, ); } +it("admits exact legacy catalog reads for an owned running service without repairing it", async () => { + const result = await runDoctorFinishForStoppedUnit("retained", undefined, "exact"); + expect(result.finishError).toBeUndefined(); + expect(mocks.stops).toBe(1); + expect(result.restartCalls).toBe(1); +}); + +it("preserves the existing malformed continuation writer refusal before stopping the service", async () => { + await expect(runDoctorFinishForStoppedUnit("retained", "own", "exact")).rejects.toThrow( + "schema migration required", + ); + expect(mocks.stops).toBe(0); +}); + +it.each([ + { catalog: "exact", continuation: "manual", message: "is still in progress" }, + { catalog: "conflict-on-recheck", continuation: undefined, message: "is still in progress" }, + { + catalog: "future-version", + continuation: undefined, + message: `uses newer schema version ${OPENCLAW_STATE_SCHEMA_VERSION + 1}`, + }, + { + catalog: "future-content", + continuation: undefined, + message: `uses newer schema version ${OPENCLAW_STATE_SCHEMA_VERSION + 1}`, + }, + { + catalog: "different-state", + continuation: undefined, + message: "non-default state dir or config path", + }, + { catalog: "unknown", continuation: undefined, message: "schema migration required" }, +] as const)( + "refuses $catalog/$continuation before stopping the service", + async ({ catalog, continuation, message }) => { + await expect(runDoctorFinishForStoppedUnit("retained", continuation, catalog)).rejects.toThrow( + message, + ); + expect(mocks.stops).toBe(0); + }, +); + it.each(["own", "parked", "normal-update-parked", "unrecorded-parked"] as const)( "continues owning-run Doctor maintenance with service %s", async (continuation) => { diff --git a/src/commands/doctor-maintenance.ts b/src/commands/doctor-maintenance.ts index 52351f71754c..b88c7e781e7c 100644 --- a/src/commands/doctor-maintenance.ts +++ b/src/commands/doctor-maintenance.ts @@ -18,6 +18,7 @@ import { createOpenClawDatabaseMaintenanceScope, type OpenClawDatabaseMaintenanceScope, } from "../state/openclaw-state-db-async-lifecycle.js"; +import { openDoctorStateSchemaReadAdmission } from "../state/openclaw-state-db-doctor-schema.js"; import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; import type { DoctorOptions } from "./doctor-prompter.js"; import { isDoctorUpdateRepairMode, resolveDoctorRepairMode } from "./doctor-repair-mode.js"; @@ -136,6 +137,7 @@ export async function beginDoctorMaintenance(params: { const runs = listUpdateRuns( { active: true, limit: 100, includeRunId: inheritedRunId }, { env }, + openDoctorStateSchemaReadAdmission, ); const admission = inspectUpdateRepairDriverAdmission(runs, inheritedRunId); if (admission.kind === "conflict") { @@ -209,7 +211,7 @@ export async function beginDoctorMaintenance(params: { const { assertNoOpenClawAgentDatabaseLeasesReadOnly, OpenClawAgentDatabaseLeaseActiveError } = await import("../state/openclaw-agent-db-lease.js"); try { - assertNoOpenClawAgentDatabaseLeasesReadOnly({ env }); + assertNoOpenClawAgentDatabaseLeasesReadOnly({ env }, openDoctorStateSchemaReadAdmission); } catch (error) { if (error instanceof OpenClawAgentDatabaseLeaseActiveError) { throw error; @@ -220,6 +222,7 @@ export async function beginDoctorMaintenance(params: { const schemas = await preflightOpenClawDatabaseSchemas({ env, scope: "state", + openStateSchemaReadAdmission: openDoctorStateSchemaReadAdmission, }); const unreadable = schemas.indeterminate.find((database) => database.kind === "state"); if (unreadable) { diff --git a/src/commands/doctor-update-schema-guard.ts b/src/commands/doctor-update-schema-guard.ts index 02ac20318287..b45f6efba2c8 100644 --- a/src/commands/doctor-update-schema-guard.ts +++ b/src/commands/doctor-update-schema-guard.ts @@ -8,6 +8,7 @@ import { preflightOpenClawDatabaseSchemas, type OpenClawDatabaseSchemaPreflight, } from "../state/openclaw-database-preflight.js"; +import { openDoctorStateSchemaReadAdmission } from "../state/openclaw-state-db-doctor-schema.js"; import { tableExists } from "../state/openclaw-state-db-schema-helpers.js"; import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; import { readStateSchemaPublicationBlocker } from "../state/openclaw-state-schema-publication.js"; @@ -28,7 +29,9 @@ async function readDrivingUpdater(): Promise< }); try { const database = openNodeSqliteDatabase(snapshot.location, { readOnly: true }); + let closeSchemaReadAdmission: (() => void) | undefined; try { + closeSchemaReadAdmission = openDoctorStateSchemaReadAdmission(database); const blocker = readStateSchemaPublicationBlocker(database); return blocker ? { @@ -37,8 +40,12 @@ async function readDrivingUpdater(): Promise< } : undefined; } finally { - clearNodeSqliteKyselyCacheForDatabase(database); - database.close(); + try { + closeSchemaReadAdmission?.(); + } finally { + clearNodeSqliteKyselyCacheForDatabase(database); + database.close(); + } } } finally { await snapshot.cleanupAsync(); @@ -58,6 +65,7 @@ export async function guardUpdateDoctorSchemaUpgrade(options: { options.schemas ?? (await preflightOpenClawDatabaseSchemas({ env: process.env, + openStateSchemaReadAdmission: openDoctorStateSchemaReadAdmission, })); if (!schemas.pendingMigrations?.length) { return; diff --git a/src/flows/doctor-health.dangling-workshop-index.test.ts b/src/flows/doctor-health.dangling-workshop-index.test.ts new file mode 100644 index 000000000000..887b671b8ecd --- /dev/null +++ b/src/flows/doctor-health.dangling-workshop-index.test.ts @@ -0,0 +1,163 @@ +import fs from "node:fs"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { doctorCommand } from "../commands/doctor.js"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { readSqliteNumberPragma } from "../infra/sqlite-pragma.test-support.js"; +import { OPENCLAW_AGENT_SCHEMA_VERSION } from "../state/openclaw-agent-db-contract.js"; +import { closeOpenClawStateDatabaseAsync } from "../state/openclaw-state-db-cache.js"; +import { OPENCLAW_STATE_SCHEMA_VERSION } from "../state/openclaw-state-db-contract.js"; +import { openOpenClawStateDatabase } from "../state/openclaw-state-db.js"; +import { claimOpenClawStateOwnership } from "../state/openclaw-state-ownership-operations.js"; +import { + withOpenClawTestState, + type OpenClawTestState, +} from "../test-utils/openclaw-test-state.js"; +import { mocks } from "./doctor-health.test-support.js"; + +// Keep the real Doctor config/migration chain; the shared harness isolates service and UI checks. +vi.doUnmock("../commands/doctor-config-flow.js"); +vi.doUnmock("../commands/doctor-prompter.js"); +vi.doUnmock("../commands/doctor/shared/plugin-runtime-symlinks.js"); + +beforeEach(() => { + mocks.packageRoot.mockReturnValue(undefined); + mocks.outro.mockClear(); + mocks.runContributions.mockReset(); +}); +afterEach(() => vi.restoreAllMocks()); + +async function seedState(state: OpenClawTestState) { + await state.writeConfig({ + agents: { ownership: "explicit", entries: { main: { workspace: state.workspaceDir } } }, + gateway: { mode: "local" }, + plugins: { enabled: false }, + }); + const database = openOpenClawStateDatabase({ env: state.env }); + database.db.exec(` + INSERT INTO skill_workshop_collection_reviews ( + review_id, owner_agent_id, backup_id, create_time, kept_names_json, written_names_json, dropped_json + ) VALUES ('review-preserved', 'main', 'backup-preserved', 1, '[]', '[]', '[]'); + `); + return database; +} + +function damageWorkshopIndex(databasePath: string): void { + const { DatabaseSync } = requireNodeSqlite(); + const database = new DatabaseSync(databasePath); + try { + database.exec( + "CREATE INDEX idx_skill_workshop_collection_reviews_workspace_time ON skill_workshop_collection_reviews(review_id, create_time DESC);", + ); + database.enableDefensive?.(false); + database.exec("PRAGMA writable_schema = ON;"); + database + .prepare( + `UPDATE sqlite_schema + SET sql = 'CREATE INDEX idx_skill_workshop_collection_reviews_workspace_time + ON skill_workshop_collection_reviews(workspace_dir, create_time DESC, review_id DESC)' + WHERE type = 'index' AND name = 'idx_skill_workshop_collection_reviews_workspace_time'`, + ) + .run(); + const schemaVersion = readSqliteNumberPragma(database, "schema_version"); + database.exec(`PRAGMA writable_schema = OFF; PRAGMA schema_version = ${schemaVersion + 1};`); + } finally { + database.close(); + } +} + +describe("Doctor malformed Workshop catalog recovery", () => { + it("restores readability before the real config and schema repair chain", async () => { + await withOpenClawTestState({ scenario: "minimal" }, async (state) => { + const database = await seedState(state); + database.db.exec("DROP INDEX idx_task_runs_status;"); + await closeOpenClawStateDatabaseAsync(); + damageWorkshopIndex(database.path); + const runtime = { log: vi.fn(), error: vi.fn(), exit: vi.fn() }; + + await doctorCommand(runtime, { repair: true, nonInteractive: true }); + + expect(runtime.exit).not.toHaveBeenCalled(); + expect(mocks.outro).toHaveBeenCalledWith("Doctor complete."); + expect(runtime.log).toHaveBeenCalledWith( + "Removed dangling legacy Skill Workshop review index", + ); + const { DatabaseSync } = requireNodeSqlite(); + const repaired = new DatabaseSync(database.path, { readOnly: true }); + try { + expect( + repaired + .prepare("SELECT review_id, backup_id FROM skill_workshop_collection_reviews") + .all(), + ).toEqual([{ review_id: "review-preserved", backup_id: "backup-preserved" }]); + expect( + repaired + .prepare( + "SELECT name FROM sqlite_schema WHERE name = 'idx_skill_workshop_collection_reviews_workspace_time'", + ) + .get(), + ).toBeUndefined(); + expect( + repaired.prepare("SELECT name FROM pragma_index_info('idx_task_runs_status')").all(), + ).toEqual([{ name: "status" }]); + expect(readSqliteNumberPragma(repaired, "user_version")).toBe( + OPENCLAW_STATE_SCHEMA_VERSION, + ); + expect(repaired.prepare("PRAGMA integrity_check").all()).toEqual([ + { integrity_check: "ok" }, + ]); + } finally { + repaired.close(); + } + }); + }); + + it.each(["shared-schema", "custom-agent-schema", "external-owner"] as const)( + "refuses %s before changing the malformed source", + async (reason) => { + await withOpenClawTestState({ scenario: "minimal" }, async (state) => { + const database = await seedState(state); + if (reason === "shared-schema") { + database.db.exec(`PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION + 1};`); + } else if (reason === "custom-agent-schema") { + const customPath = state.path("custom", "sessions.sqlite"); + fs.mkdirSync(path.dirname(customPath)); + const { DatabaseSync } = requireNodeSqlite(); + const custom = new DatabaseSync(customPath); + custom.exec(` + PRAGMA user_version = ${OPENCLAW_AGENT_SCHEMA_VERSION + 1}; + CREATE TABLE schema_meta (meta_key TEXT PRIMARY KEY, agent_id TEXT); + INSERT INTO schema_meta VALUES ('primary', 'main'); + `); + custom.close(); + await state.writeConfig({ + agents: { ownership: "explicit", entries: { main: { workspace: state.workspaceDir } } }, + session: { store: customPath }, + gateway: { mode: "local" }, + plugins: { enabled: false }, + }); + } else { + claimOpenClawStateOwnership("fixture-manager", { + env: { ...state.env, OPENCLAW_SUPERVISOR_MODE: "external" }, + }); + } + await closeOpenClawStateDatabaseAsync(); + damageWorkshopIndex(database.path); + const before = fs.readFileSync(database.path); + const configBefore = fs.readFileSync(state.configPath); + + await expect( + doctorCommand( + { log: vi.fn(), error: vi.fn(), exit: vi.fn() }, + { repair: true, nonInteractive: true }, + ), + ).rejects.toThrow(reason === "external-owner" ? /externally supervised/ : /newer/); + + expect(fs.readFileSync(database.path)).toEqual(before); + expect(fs.readFileSync(state.configPath)).toEqual(configBefore); + expect(mocks.runContributions).not.toHaveBeenCalled(); + expect(mocks.outro).not.toHaveBeenCalledWith("Doctor complete."); + }); + }, + ); +}); diff --git a/src/flows/doctor-health.migration-refusal.test.ts b/src/flows/doctor-health.migration-refusal.test.ts index 5408d1c4448d..b5eeaeefda57 100644 --- a/src/flows/doctor-health.migration-refusal.test.ts +++ b/src/flows/doctor-health.migration-refusal.test.ts @@ -203,7 +203,7 @@ describe("Doctor maintenance admission", () => { }); describe("Doctor agent lease admission", () => { - it("admits the exact dangling Workshop index without mutating state", async () => { + it("reserves dangling Workshop index admission for Doctor without mutating state", async () => { await withOpenClawTestState({ scenario: "minimal" }, async (state) => { const opened = openOpenClawStateDatabase({ env: state.env }); const pathname = opened.path; @@ -231,8 +231,21 @@ describe("Doctor agent lease admission", () => { } const before = fs.readFileSync(pathname); - expect(() => assertNoOpenClawAgentDatabaseLeasesReadOnly({ env: state.env })).not.toThrow(); + expect(() => assertNoOpenClawAgentDatabaseLeasesReadOnly({ env: state.env })).toThrow( + /malformed database schema/, + ); expect(fs.readFileSync(pathname)).toEqual(before); + const doctor = await doctorMaintenance.beginDoctorMaintenance({ + options: { repair: true, nonInteractive: true }, + root: null, + runtime: { log: vi.fn(), error: vi.fn(), exit: vi.fn() }, + }); + try { + expect(doctor).toBeDefined(); + expect(fs.readFileSync(pathname)).toEqual(before); + } finally { + await doctor?.release(); + } }); }); diff --git a/src/flows/doctor-health.ts b/src/flows/doctor-health.ts index 8e4bfdbb3798..dbb4d0e5ad9d 100644 --- a/src/flows/doctor-health.ts +++ b/src/flows/doctor-health.ts @@ -33,9 +33,10 @@ const loadConfigModule = createLazyRuntimeModule(() => import("../config/config. async function assertDoctorDatabaseSchemasCompatible(scope?: "state") { const databasePreflight = await import("../state/openclaw-database-preflight.js"); - const [{ createConfigIO }, targets] = await Promise.all([ + const [{ createConfigIO }, targets, { openDoctorStateSchemaReadAdmission }] = await Promise.all([ import("../config/io.js"), import("../config/sessions/targets.js"), + import("../state/openclaw-state-db-doctor-schema.js"), ]); const snapshot = await createConfigIO({ env: { ...process.env }, @@ -46,6 +47,7 @@ async function assertDoctorDatabaseSchemasCompatible(scope?: "state") { const databaseSchemas = await databasePreflight.preflightOpenClawDatabaseSchemas({ env: process.env, scope, + openStateSchemaReadAdmission: openDoctorStateSchemaReadAdmission, configuredAgentDatabaseTargets: (registeredDatabases) => targets.resolveConfiguredAgentDatabaseTargets(cfg, { env: process.env, registeredDatabases }), configuredAgentDatabaseCandidatePaths: targets.resolveConfiguredAgentDatabaseCandidatePaths( @@ -160,6 +162,19 @@ async function runDoctorHealthFlowWithResult( json: options.json, }); + if (maintenance && (options.repair === true || options.yes === true)) { + const { repairOpenClawStateDatabaseReadabilityForDoctor } = + await import("../state/openclaw-state-db.js"); + // Restore catalog reads before config discovery; versioned migrations remain in its graph. + const readability = repairOpenClawStateDatabaseReadabilityForDoctor({ env: process.env }); + if (readability.warnings.length > 0) { + throw new Error(readability.warnings.join("\n")); + } + for (const change of readability.changes) { + effectiveRuntime.log(change); + } + } + // Keep side-effect-heavy legacy checks before structured contributions until fully migrated. const { maybeRepairUiProtocolFreshness } = await import("../commands/doctor-ui.js"); const { noteSourceInstallIssues } = await import("../commands/doctor-install.js"); diff --git a/src/infra/update-run-reader.ts b/src/infra/update-run-reader.ts index fe4e4b69471f..ffb1261ef744 100644 --- a/src/infra/update-run-reader.ts +++ b/src/infra/update-run-reader.ts @@ -1,5 +1,8 @@ import type { DatabaseSync } from "node:sqlite"; -import type { OpenClawStateDatabaseOptions } from "../state/openclaw-state-db-contract.js"; +import type { + OpenClawStateDatabaseOptions, + OpenClawStateSchemaReadAdmission, +} from "../state/openclaw-state-db-contract.js"; import { withExistingOpenClawStateDatabaseArtifactPreservingReadOnly, withExistingOpenClawStateDatabaseArtifactPreservingReadOnlyAsync, @@ -96,11 +99,13 @@ function readRuns(db: DatabaseSync, input: ListInput): UpdateRunRecord[] { export function listUpdateRuns( input: ListInput = {}, options: OpenClawStateDatabaseOptions = {}, + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission, ): UpdateRunRecord[] { return ( withExistingOpenClawStateDatabaseArtifactPreservingReadOnly( ({ db }) => readRuns(db, input), options, + openStateSchemaReadAdmission, ) ?? [] ); } diff --git a/src/state/openclaw-agent-db-lease.ts b/src/state/openclaw-agent-db-lease.ts index 9fdeaa935038..229cdedf064a 100644 --- a/src/state/openclaw-agent-db-lease.ts +++ b/src/state/openclaw-agent-db-lease.ts @@ -25,8 +25,10 @@ import { prepareAgentDeletionPathFence, } from "./agent-deletion-journal.js"; import { openClawStateDatabaseCache } from "./openclaw-state-db-cache.js"; -import type { OpenClawStateDatabaseOptions } from "./openclaw-state-db-contract.js"; -import { openDanglingWorkshopIndexReadAdmission } from "./openclaw-state-db-dangling-workshop-index.js"; +import type { + OpenClawStateDatabaseOptions, + OpenClawStateSchemaReadAdmission, +} from "./openclaw-state-db-contract.js"; import { runExistingOpenClawStateWriteTransaction } from "./openclaw-state-db-existing-write.js"; import { ensureAgentDatabaseLeaseSchema } from "./openclaw-state-db-schema-additive.js"; import { tableExists } from "./openclaw-state-db-schema-helpers.js"; @@ -350,6 +352,7 @@ function isAgentDatabaseLeaseStale(row: { /** Doctor holds both lifecycle coordinators before checking writers, without schema repair. */ export function assertNoOpenClawAgentDatabaseLeasesReadOnly( options: OpenClawStateDatabaseOptions = {}, + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission, ): void { const pathname = path.resolve(options.path ?? resolveOpenClawStateSqlitePath(options.env)); try { @@ -368,7 +371,7 @@ export function assertNoOpenClawAgentDatabaseLeasesReadOnly( const db = cached?.db ?? openNodeSqliteDatabase(pathname, { readOnly: true }); let closeSchemaReadAdmission: (() => void) | undefined; try { - closeSchemaReadAdmission = openDanglingWorkshopIndexReadAdmission(db); + closeSchemaReadAdmission = openStateSchemaReadAdmission?.(db); runWithSqliteBusyTimeout(db, 250, () => { if (!tableExists(db, "agent_database_leases")) { return; diff --git a/src/state/openclaw-database-preflight.dangling-workshop-index.test.ts b/src/state/openclaw-database-preflight.dangling-workshop-index.test.ts index 77fd1ecc8abf..e7efa0280a89 100644 --- a/src/state/openclaw-database-preflight.dangling-workshop-index.test.ts +++ b/src/state/openclaw-database-preflight.dangling-workshop-index.test.ts @@ -6,6 +6,7 @@ import { requireNodeSqlite } from "../infra/node-sqlite.js"; import { OPENCLAW_AGENT_SCHEMA_VERSION } from "./openclaw-agent-db-contract.js"; import { preflightOpenClawDatabaseSchemas } from "./openclaw-database-preflight.js"; import { OPENCLAW_STATE_SCHEMA_VERSION } from "./openclaw-state-db-contract.js"; +import { openDoctorStateSchemaReadAdmission } from "./openclaw-state-db-doctor-schema.js"; import { closeOpenClawStateDatabaseForTest, openOpenClawStateDatabase, @@ -16,7 +17,7 @@ const tempDirs = useAutoCleanupTempDirTracker(afterEach); afterEach(closeOpenClawStateDatabaseForTest); describe("dangling Workshop index preflight", () => { - it("admits the exact defect for Doctor without mutating the source", async () => { + it("admits the exact defect only for Doctor without mutating the source", async () => { const stateDir = tempDirs.make("openclaw-preflight-dangling-workshop-"); const env = { OPENCLAW_STATE_DIR: stateDir }; const statePath = openOpenClawStateDatabase({ env }).path; @@ -54,10 +55,17 @@ describe("dangling Workshop index preflight", () => { .map((name) => [name, fs.readFileSync(path.join(sourceDir, name))]); const before = snapshot(); + const runtime = await preflightOpenClawDatabaseSchemas({ env, scope: "state" }); + expect(runtime.indeterminate).toEqual([ + expect.objectContaining({ reason: expect.stringMatching(/openclaw doctor --fix/) }), + ]); + expect(snapshot()).toEqual(before); + await expect( preflightOpenClawDatabaseSchemas({ env, scope: "state", + openStateSchemaReadAdmission: openDoctorStateSchemaReadAdmission, supportedVersions: { state: OPENCLAW_STATE_SCHEMA_VERSION, agent: OPENCLAW_AGENT_SCHEMA_VERSION, diff --git a/src/state/openclaw-database-preflight.ts b/src/state/openclaw-database-preflight.ts index 435a8acaad4d..d2e01b9fe303 100644 --- a/src/state/openclaw-database-preflight.ts +++ b/src/state/openclaw-database-preflight.ts @@ -6,7 +6,11 @@ import { resolveUnsuffixedSqliteTargetFromSessionStorePath } from "../config/ses import { resolveConfiguredAgentDatabaseCandidatePaths } from "../config/sessions/targets.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { formatErrorMessage } from "../infra/errors.js"; -import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js"; +import { + clearNodeSqliteKyselyCacheForDatabase, + executeSqliteQuerySync, + getNodeSqliteKysely, +} from "../infra/kysely-sync.js"; import { openNodeSqliteDatabase, resolveImmutableSqliteFileUri } from "../infra/node-sqlite.js"; import { hasNodeErrorCode } from "../infra/path-guards.js"; import { assertSqliteIntegrityInWorker } from "../infra/sqlite-integrity-worker.js"; @@ -59,15 +63,13 @@ import { OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, OPENCLAW_STATE_SCHEMA_VERSION, } from "./openclaw-state-db-contract.js"; -import { - closeWorkshopIndexReadDatabase, - openDanglingWorkshopIndexReadAdmission, -} from "./openclaw-state-db-dangling-workshop-index.js"; +import type { OpenClawStateSchemaReadAdmission } from "./openclaw-state-db-contract.js"; import { assertOpenClawStateDatabaseOwner, assertOpenClawStateDatabaseForMaintenance, openClawStateMigrationAssertions, } from "./openclaw-state-db-maintenance.js"; +import { normalizeOpenClawStateSchemaReadError } from "./openclaw-state-db-schema-migration-required.js"; import { assertCanonicalStateSchemaShape } from "./openclaw-state-db-schema-repair.js"; import { readStateSchemaContentVersion, @@ -354,6 +356,7 @@ export async function preflightOpenClawDatabaseSchemas(options: { ) => readonly { agentId: string; path: string }[]); configuredAgentDatabaseCandidatePaths?: readonly string[]; agentAdmissionConfig?: OpenClawConfig; + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission; }): Promise { options.signal?.throwIfAborted(); const { @@ -397,7 +400,7 @@ export async function preflightOpenClawDatabaseSchemas(options: { stateDatabase = openNodeSqliteDatabase(stateSnapshot.location, { readOnly: true, }); - closeStateSchemaReadAdmission = openDanglingWorkshopIndexReadAdmission(stateDatabase); + closeStateSchemaReadAdmission = options.openStateSchemaReadAdmission?.(stateDatabase); stateDatabase.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`); const stateVersion = readSqliteUserVersion(stateDatabase); const contentVersion = @@ -490,19 +493,25 @@ export async function preflightOpenClawDatabaseSchemas(options: { } catch (error) { // Accepted stop must not turn cancellation or failed cleanup into a // warn-and-continue result that launches the remaining startup runtime. + const failure = normalizeOpenClawStateSchemaReadError(error, statePath); if (options.signal?.aborted || options.requireStartupMigrationReadiness) { - throw error; + throw failure; } result.indeterminate.push({ kind: "state", path: statePath, - reason: formatErrorMessage(error), + reason: formatErrorMessage(failure), }); return result; } finally { try { if (stateDatabase) { - closeWorkshopIndexReadDatabase(stateDatabase, closeStateSchemaReadAdmission); + try { + closeStateSchemaReadAdmission?.(); + } finally { + clearNodeSqliteKyselyCacheForDatabase(stateDatabase); + stateDatabase.close(); + } } } finally { await stateSnapshot?.cleanupAsync(); diff --git a/src/state/openclaw-state-db-contract.ts b/src/state/openclaw-state-db-contract.ts index 36a4b17f4f74..53de054a9e9f 100644 --- a/src/state/openclaw-state-db-contract.ts +++ b/src/state/openclaw-state-db-contract.ts @@ -2,6 +2,8 @@ import type { DatabaseSync } from "node:sqlite"; import type { SqliteWalMaintenance } from "../infra/sqlite-wal.js"; import type { DatabasePathIdentity } from "../infra/sqlite-worker-identity.js"; +export type OpenClawStateSchemaReadAdmission = (database: DatabaseSync) => (() => void) | undefined; + // v17 records one-use prepared worker capacity and node workspace ownership. // v16 makes Skill Workshop ownership directory-based instead of row-provenance-based. // v15 removes redundant agent/session projections from conversation bindings. diff --git a/src/state/openclaw-state-db-dangling-workshop-index.ts b/src/state/openclaw-state-db-dangling-workshop-index.ts deleted file mode 100644 index 9d3acaa80d42..000000000000 --- a/src/state/openclaw-state-db-dangling-workshop-index.ts +++ /dev/null @@ -1,107 +0,0 @@ -import type { DatabaseSync } from "node:sqlite"; -import { clearNodeSqliteKyselyCacheForDatabase } from "../infra/kysely-sync-cache-state.js"; - -const LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX = - "idx_skill_workshop_collection_reviews_workspace_time"; -const LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX_SQL = - "CREATE INDEX idx_skill_workshop_collection_reviews_workspace_time ON skill_workshop_collection_reviews(workspace_dir, create_time DESC, review_id DESC)"; - -function normalizeSqliteCatalogSql(sql: string): string { - return sql - .replace(/\s+/gu, " ") - .replace(/\s*([(),])\s*/gu, "$1") - .trim(); -} - -export function withSqliteWritableSchema(database: DatabaseSync, operation: () => T): T { - database.enableDefensive?.(false); - // sqlite-allow-raw -- Exact legacy catalog admission requires SQLite's writable-schema pragma. - database.exec("PRAGMA writable_schema = ON;"); - try { - return operation(); - } finally { - try { - // sqlite-allow-raw -- Always restore catalog parsing after the bounded legacy inspection. - database.exec("PRAGMA writable_schema = OFF;"); - } finally { - database.enableDefensive?.(true); - } - } -} - -/** Detect only the known v15 review index left behind after its column was retired. */ -export function hasDanglingSkillWorkshopCollectionReviewIndex(database: DatabaseSync): boolean { - return withSqliteWritableSchema(database, () => { - const rawIndex = database // sqlite-allow-raw -- Inspect the exact malformed catalog row before ordinary schema parsing. - .prepare( - "SELECT tbl_name, rootpage, sql FROM sqlite_schema WHERE type = 'index' AND name = ?", - ) - .get(LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX); - // SAFETY: the narrow catalog projection is validated field-by-field below. - const index = rawIndex as { tbl_name?: unknown; rootpage?: unknown; sql?: unknown } | undefined; - if ( - index?.tbl_name !== "skill_workshop_collection_reviews" || - typeof index.rootpage !== "number" || - index.rootpage <= 0 || - typeof index.sql !== "string" || - normalizeSqliteCatalogSql(index.sql) !== - normalizeSqliteCatalogSql(LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX_SQL) - ) { - return false; - } - const rawColumns = database // sqlite-allow-raw -- Validate physical columns without parsing the malformed index. - .prepare("PRAGMA table_info(skill_workshop_collection_reviews)") - .all(); - // SAFETY: PRAGMA table_info rows expose optional names compared as unknown values. - const columns = rawColumns as Array<{ name?: unknown }>; - return ( - columns.some((column) => column.name === "owner_agent_id") && - !columns.some((column) => column.name === "workspace_dir") - ); - }); -} - -/** Keep a read-only connection tolerant of the exact malformed legacy index. */ -export function openDanglingWorkshopIndexReadAdmission( - database: DatabaseSync, -): (() => void) | undefined { - if (!hasDanglingSkillWorkshopCollectionReviewIndex(database)) { - return undefined; - } - database.enableDefensive?.(false); - try { - // sqlite-allow-raw -- Hold exact legacy catalog admission for a bounded read-only operation. - database.exec("PRAGMA writable_schema = ON;"); - } catch (error) { - database.enableDefensive?.(true); - throw error; - } - let open = true; - return () => { - if (!open) { - return; - } - open = false; - try { - // sqlite-allow-raw -- Restore ordinary schema parsing before releasing the read-only handle. - database.exec("PRAGMA writable_schema = OFF;"); - } finally { - database.enableDefensive?.(true); - } - }; -} - -/** Restore schema parsing and release a private read handle even if either cleanup fails. */ -export function closeWorkshopIndexReadDatabase( - database: DatabaseSync, - closeAdmission?: () => void, -): void { - try { - closeAdmission?.(); - } finally { - clearNodeSqliteKyselyCacheForDatabase(database); - database.close(); - } -} - -export { LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX }; diff --git a/src/state/openclaw-state-db-doctor-schema.ts b/src/state/openclaw-state-db-doctor-schema.ts new file mode 100644 index 000000000000..6f3a805325a7 --- /dev/null +++ b/src/state/openclaw-state-db-doctor-schema.ts @@ -0,0 +1,93 @@ +import type { DatabaseSync } from "node:sqlite"; +import { LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX } from "./openclaw-state-db-schema-migration-required.js"; +const LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX_SQL = + "CREATE INDEX idx_skill_workshop_collection_reviews_workspace_time ON skill_workshop_collection_reviews(workspace_dir, create_time DESC, review_id DESC)"; + +function normalizeSqliteCatalogSql(sql: string): string { + return sql + .replace(/\s+/gu, " ") + .replace(/\s*([(),])\s*/gu, "$1") + .trim(); +} + +export function withSqliteWritableSchema(database: DatabaseSync, operation: () => T): T { + database.enableDefensive?.(false); + // sqlite-allow-raw -- Exact legacy catalog admission requires SQLite's writable-schema pragma. + database.exec("PRAGMA writable_schema = ON;"); + try { + return operation(); + } finally { + try { + // sqlite-allow-raw -- OFF retains the schema loaded while malformed rows were ignored. + database.exec("PRAGMA writable_schema = RESET;"); + } finally { + database.enableDefensive?.(true); + } + } +} + +/** Detect only the known v15 review index left behind after its column was retired. */ +export function hasDanglingSkillWorkshopCollectionReviewIndex(database: DatabaseSync): boolean { + return withSqliteWritableSchema(database, () => + inspectSkillWorkshopCollectionReviewIndex(database), + ); +} + +function inspectSkillWorkshopCollectionReviewIndex(database: DatabaseSync): boolean { + const rawIndex = database // sqlite-allow-raw -- Inspect the exact malformed catalog row before ordinary schema parsing. + .prepare("SELECT tbl_name, rootpage, sql FROM sqlite_schema WHERE type = 'index' AND name = ?") + .get(LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX); + // SAFETY: the narrow catalog projection is validated field-by-field below. + const index = rawIndex as { tbl_name?: unknown; rootpage?: unknown; sql?: unknown } | undefined; + if ( + index?.tbl_name !== "skill_workshop_collection_reviews" || + typeof index.rootpage !== "number" || + index.rootpage <= 0 || + typeof index.sql !== "string" || + normalizeSqliteCatalogSql(index.sql) !== + normalizeSqliteCatalogSql(LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX_SQL) + ) { + return false; + } + const rawColumns = database // sqlite-allow-raw -- Validate physical columns without parsing the malformed index. + .prepare("PRAGMA table_info(skill_workshop_collection_reviews)") + .all(); + // SAFETY: PRAGMA table_info rows expose optional names compared as unknown values. + const columns = rawColumns as Array<{ name?: unknown }>; + return ( + columns.some((column) => column.name === "owner_agent_id") && + !columns.some((column) => column.name === "workspace_dir") + ); +} + +/** Doctor can inspect the exact legacy defect before its repair transaction. */ +export function openDoctorStateSchemaReadAdmission( + database: DatabaseSync, +): (() => void) | undefined { + if (!hasDanglingSkillWorkshopCollectionReviewIndex(database)) { + return undefined; + } + database.enableDefensive?.(false); + try { + // sqlite-allow-raw -- Hold exact legacy catalog admission for a bounded read-only operation. + database.exec("PRAGMA writable_schema = ON;"); + } catch (error) { + database.enableDefensive?.(true); + throw error; + } + let open = true; + return () => { + if (!open) { + return; + } + open = false; + try { + // sqlite-allow-raw -- Restore ordinary schema parsing before releasing the read-only handle. + database.exec("PRAGMA writable_schema = RESET;"); + } finally { + database.enableDefensive?.(true); + } + }; +} + +export { LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX }; diff --git a/src/state/openclaw-state-db-maintenance.ts b/src/state/openclaw-state-db-maintenance.ts index 870f7707d0ec..82c98640471c 100644 --- a/src/state/openclaw-state-db-maintenance.ts +++ b/src/state/openclaw-state-db-maintenance.ts @@ -22,7 +22,7 @@ import { hasDanglingSkillWorkshopCollectionReviewIndex, LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX, withSqliteWritableSchema, -} from "./openclaw-state-db-dangling-workshop-index.js"; +} from "./openclaw-state-db-doctor-schema.js"; import { ensureColumn, tableExists, tableHasColumn } from "./openclaw-state-db-schema-helpers.js"; import { migrateJsonCanonicalWideRowsV13 } from "./openclaw-state-db-schema-v13-widerow.js"; import { @@ -541,10 +541,13 @@ export function runStateSchemaMigrationTransaction( pathname: string, migrate: () => T, transactionOptions: SqliteTransactionOptions, + prepareSchema?: () => void, ): T { return runSqliteImmediateTransactionSync( db, () => { + // Doctor restores catalog readability before the publication prelude reads it. + prepareSchema?.(); const publishedVersion = readSqliteUserVersion(db); const blocker = publishedVersion < OPENCLAW_STATE_SCHEMA_VERSION diff --git a/src/state/openclaw-state-db-open.ts b/src/state/openclaw-state-db-open.ts index be584fc3d063..50987991991e 100644 --- a/src/state/openclaw-state-db-open.ts +++ b/src/state/openclaw-state-db-open.ts @@ -30,10 +30,8 @@ import { OPENCLAW_STATE_SCHEMA_VERSION, type OpenClawStateDatabase, } from "./openclaw-state-db-contract.js"; -import { hasDanglingSkillWorkshopCollectionReviewIndex } from "./openclaw-state-db-dangling-workshop-index.js"; import { openTrackedStateDatabase } from "./openclaw-state-db-handle.js"; import { ensureOpenClawStatePermissions } from "./openclaw-state-db-permissions.js"; -import { OpenClawStateDatabaseSchemaMigrationRequiredError } from "./openclaw-state-db-schema-migration-required.js"; import { assertSupportedStateSchemaVersion, readStateSchemaMigrationVersion, @@ -85,12 +83,6 @@ export function openUnpublishedStateDatabase(params: { db, busyTimeoutMs, () => { - if (hasDanglingSkillWorkshopCollectionReviewIndex(db)) { - throw new OpenClawStateDatabaseSchemaMigrationRequiredError( - "legacy-workshop-review-index", - params.pathname, - ); - } assertSupportedStateSchemaVersion(db, params.pathname); assertStateDatabaseIntegrityBeforeMutation(db, params.pathname); configureSqlitePreSchemaPragmas(db, { busyTimeoutMs }); diff --git a/src/state/openclaw-state-db-prepared-reads.test.ts b/src/state/openclaw-state-db-prepared-reads.test.ts new file mode 100644 index 000000000000..8b6c4c610832 --- /dev/null +++ b/src/state/openclaw-state-db-prepared-reads.test.ts @@ -0,0 +1,33 @@ +import { constants } from "node:sqlite"; +import { expect, it } from "vitest"; +import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; +import { withExistingOpenClawStateDatabaseReadOnly } from "./openclaw-state-db-readonly.js"; +import { openOpenClawStateDatabase } from "./openclaw-state-db.js"; +import { assertOpenClawStateWriteAllowed } from "./openclaw-state-ownership.js"; + +it("keeps prepared queries reusable across ordinary reads and ownership checks", async () => { + await withOpenClawTestState({ label: "state-prepared-reads" }, async ({ env }) => { + const { db, path } = openOpenClawStateDatabase({ env }); + db.exec("CREATE TABLE prepared_read (value INTEGER); INSERT INTO prepared_read VALUES (42)"); + let readPreparations = 0; + db.setAuthorizer((action, table) => { + if (action === constants.SQLITE_READ && table === "prepared_read") { + readPreparations++; + } + return constants.SQLITE_OK; + }); + const read = db.prepare("SELECT value FROM prepared_read"); + expect(read.get()).toEqual({ value: 42 }); + expect(readPreparations).toBe(1); + + for (let iteration = 0; iteration < 3; iteration++) { + expect(withExistingOpenClawStateDatabaseReadOnly(() => read.get(), { env })).toEqual({ + value: 42, + }); + assertOpenClawStateWriteAllowed({ database: db, databasePath: path, env }); + expect(read.get()).toEqual({ value: 42 }); + } + + expect(readPreparations).toBe(1); + }); +}); diff --git a/src/state/openclaw-state-db-readonly.test.ts b/src/state/openclaw-state-db-readonly.test.ts index 4bfbb055d402..87d92f247d62 100644 --- a/src/state/openclaw-state-db-readonly.test.ts +++ b/src/state/openclaw-state-db-readonly.test.ts @@ -307,7 +307,7 @@ describe.each(["admission", "explicit", "async"] as const)("%s read-only state r expect(fs.readFileSync(options.path)).toEqual(before); }); }); - it("reads through the exact dangling Workshop index without changing its source", async () => { + it("requires Doctor for the exact dangling Workshop index without changing its source", async () => { await withTempDir("openclaw-state-readonly-dangling-workshop-", async (stateDir) => { const options = createOptions(stateDir); const opened = openOpenClawStateDatabase(options); @@ -339,9 +339,11 @@ describe.each(["admission", "explicit", "async"] as const)("%s read-only state r } const before = fs.readFileSync(options.path); - expect( - await readState(({ db }) => db.prepare("SELECT role FROM schema_meta").get(), options), - ).toEqual({ role: "global" }); + await expect( + Promise.resolve().then(() => + readState(({ db }) => db.prepare("SELECT role FROM schema_meta").get(), options), + ), + ).rejects.toThrow(/legacy-workshop-review-index.*openclaw doctor --fix/); expect(fs.readFileSync(options.path)).toEqual(before); }); }); diff --git a/src/state/openclaw-state-db-readonly.ts b/src/state/openclaw-state-db-readonly.ts index 9cf8e2b8ef63..52cbb45818aa 100644 --- a/src/state/openclaw-state-db-readonly.ts +++ b/src/state/openclaw-state-db-readonly.ts @@ -15,8 +15,8 @@ import { openClawStateDatabaseCache } from "./openclaw-state-db-cache.js"; import type { OpenClawStateDatabaseOptions, OpenClawStateDatabase, + OpenClawStateSchemaReadAdmission, } from "./openclaw-state-db-contract.js"; -import { openDanglingWorkshopIndexReadAdmission } from "./openclaw-state-db-dangling-workshop-index.js"; import { openOpenClawStateReadConnection } from "./openclaw-state-db-read-connection.js"; import { assertSupportedStateSchemaVersion } from "./openclaw-state-db-schema-version.js"; import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js"; @@ -215,18 +215,13 @@ function withOpenClawStateDatabaseReadOnlyIfOpen( return { reused: false }; } try { - const closeSchemaReadAdmission = openDanglingWorkshopIndexReadAdmission(opened.db); - try { - // Process-local terminal failures evict this handle. Persisted quarantine - // is checked on the next physical open so hot reads do not poll metadata. - // A newer build can migrate this file while the handle stays open, so the - // forward-compatibility gate still runs before any reused read. - assertSupportedStateSchemaVersion(opened.db, pathname); - observeOpenClawDatabaseMaintenanceResource(opened.db); - return { reused: true, value: operation(opened) }; - } finally { - closeSchemaReadAdmission?.(); - } + // Process-local terminal failures evict this handle. Persisted quarantine + // is checked on the next physical open so hot reads do not poll metadata. + // A newer build can migrate this file while the handle stays open, so the + // forward-compatibility gate still runs before any reused read. + assertSupportedStateSchemaVersion(opened.db, pathname); + observeOpenClawDatabaseMaintenanceResource(opened.db); + return { reused: true, value: operation(opened) }; } catch (error) { openClawStateDatabaseCache.evictOpenClawStateDatabaseAfterCorruption(opened, error); throw error; @@ -272,46 +267,26 @@ function openOpenClawStateReadOnlyLocation( source: string | PreparedSqliteReadOnlyLocation, ) { const connection = openOpenClawStateReadConnection(pathname, source); - const { db } = connection.database; - let closeSchemaReadAdmission: (() => void) | undefined; - const close = () => { - const errors: unknown[] = []; - let closed = false; - try { - closeSchemaReadAdmission?.(); - } catch (error) { - errors.push(error); - } - try { - closed = connection.close(); - } catch (error) { - errors.push(error); - } - if (errors.length === 1) { - throw errors[0]; - } - if (errors.length > 1) { - throw new AggregateError(errors, "Shared-state reader cleanup failed."); - } - return closed; - }; try { - closeSchemaReadAdmission = openDanglingWorkshopIndexReadAdmission(db); - assertSupportedStateSchemaVersion(db, pathname); + assertSupportedStateSchemaVersion(connection.database.db, pathname); } catch (error) { - close(); + connection.close(); throw error; } - return { database: connection.database, close }; + return connection; } function withOpenClawStateReadOnlyLocation( operation: (database: OpenClawStateReadOnlyDatabase) => T, pathname: string, source: string | PreparedSqliteReadOnlyLocation, + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission, ): T { - const opened = openOpenClawStateReadOnlyLocation(pathname, source); + const opened = openOpenClawStateReadConnection(pathname, source); + let closeAdmission: (() => void) | undefined; try { + closeAdmission = openStateSchemaReadAdmission?.(opened.database.db); + assertSupportedStateSchemaVersion(opened.database.db, pathname); const result = operation(opened.database); const location = typeof source === "string" ? source : source.location; if (location === pathname && isPromiseLike(result)) { @@ -319,7 +294,11 @@ function withOpenClawStateReadOnlyLocation( } return result; } finally { - opened.close(); + try { + closeAdmission?.(); + } finally { + opened.close(); + } } } @@ -397,7 +376,15 @@ export function withExistingOpenClawStateDatabaseReadOnly( export function withExistingOpenClawStateDatabaseArtifactPreservingReadOnly( operation: (database: OpenClawStateReadOnlyDatabase) => T, options: OpenClawStateDatabaseOptions = {}, + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission, ): T | undefined { + if (openStateSchemaReadAdmission) { + return withExistingOpenClawStateDatabaseCurrentReadOnly( + operation, + options, + openStateSchemaReadAdmission, + ); + } return withArtifactPreservingStateReads(() => withExistingOpenClawStateDatabaseReadOnly(operation, options), ); @@ -407,12 +394,16 @@ export function withExistingOpenClawStateDatabaseArtifactPreservingReadOnly( export function withExistingOpenClawStateDatabaseCurrentReadOnly( operation: (database: OpenClawStateReadOnlyDatabase) => T, options: OpenClawStateDatabaseOptions = {}, + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission, ): T | undefined { return stateSnapshotReads.exit(() => { const pathname = resolveReadOnlyPath(options); - const reused = withOpenClawStateDatabaseReadOnlyIfOpen(operation, pathname); - if (reused.reused) { - return reused.value; + // Maintenance admission belongs to a fresh private reader, never a cached writer. + if (!openStateSchemaReadAdmission) { + const reused = withOpenClawStateDatabaseReadOnlyIfOpen(operation, pathname); + if (reused.reused) { + return reused.value; + } } if (existingPathOrUndefined(pathname) === undefined) { return undefined; @@ -425,6 +416,7 @@ export function withExistingOpenClawStateDatabaseCurrentReadOnly( operation, pathname, prepareSqliteReadOnlyLocationSync(pathname), + openStateSchemaReadAdmission, ); }); } diff --git a/src/state/openclaw-state-db-repair.ts b/src/state/openclaw-state-db-repair.ts new file mode 100644 index 000000000000..11f728238092 --- /dev/null +++ b/src/state/openclaw-state-db-repair.ts @@ -0,0 +1,250 @@ +import { clearNodeSqliteKyselyCacheForDatabase } from "../infra/kysely-sync.js"; +import { openNodeSqliteDatabase } from "../infra/node-sqlite.js"; +import { setSqliteBusyTimeout } from "../infra/sqlite-busy-timeout.js"; +import { + repairCanonicalSqliteIndexes, + verifyAndRepairCanonicalSqliteIndexes, +} from "../infra/sqlite-index-schema.js"; +import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js"; +import { assertSqliteSchemaTablesPresent } from "../infra/sqlite-schema-contract.js"; +import { migrateSqliteSchemaToStrictInTransaction } from "../infra/sqlite-strict.js"; +import { runSqliteImmediateTransactionSync } from "../infra/sqlite-transaction.js"; +import { clearOpenClawDatabaseQuarantine } from "./openclaw-quarantine-store.js"; +import { repairAuditEventsSchema } from "./openclaw-state-db-audit-migration.js"; +import { clearOpenClawStateDatabaseOpenFailure } from "./openclaw-state-db-cache.js"; +import { + LAZY_ADDITIVE_STATE_TABLES, + OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + OPENCLAW_STATE_SCHEMA_VERSION, + OPENCLAW_STATE_STRICT_SCHEMA_VERSION, +} from "./openclaw-state-db-contract.js"; +import { assertCurrentStateRuntimeSchema } from "./openclaw-state-db-fast-path.js"; +import { + assertOpenClawStateDatabaseOwner, + markCurrentStateSchemaVersion, + openClawStateMigrationAssertions, + versionedStateMigrations, + runStateSchemaMigrationTransaction, + executeCanonicalStateSchema, + prepareStateDatabaseSchemaRepair, +} from "./openclaw-state-db-maintenance.js"; +import * as operatorApprovalMigration from "./openclaw-state-db-operator-approval-migration.js"; +import { ensureOpenClawStatePermissions } from "./openclaw-state-db-permissions.js"; +import { + ensureAdditiveStateColumns, + ensureFirstUseAdditiveStateColumnsForStrictMigration, +} from "./openclaw-state-db-schema-additive.js"; +import { tableExists } from "./openclaw-state-db-schema-helpers.js"; +import { + assertCanonicalStateSchemaShape, + dropLegacyStateTables, + migrateAgentDatabaseRelativePaths as migrateAgentPaths, + migrateWorkerPlacementExecutionModeSchema, + repairAgentDatabasesCompositePrimaryKey, + repairLegacyGatewayRestartHandoffsForStrictMigration, +} from "./openclaw-state-db-schema-repair.js"; +import { migrateSingletonStateFoldInV12 } from "./openclaw-state-db-schema-v12-foldin.js"; +import { + assertSupportedStateSchemaVersion, + readStateSchemaContentVersion, + readStateSchemaMigrationVersion, +} from "./openclaw-state-db-schema-version.js"; +import * as sessionWatchMigration from "./openclaw-state-db-session-watch-migration.js"; +import * as retirements from "./openclaw-state-db-table-retirements.js"; +import { recoverOrphanTaskDeliveryRows } from "./openclaw-state-db-task-delivery-recovery.js"; +import { describeAgentPathMigration } from "./openclaw-state-db.paths.js"; +import { + assertOpenClawStateWriteAllowed, + OpenClawStateOwnershipError, +} from "./openclaw-state-ownership.js"; +import { getOpenClawStateRuntimeSchema } from "./openclaw-state-schema-compatibility.js"; +import { OPENCLAW_STATE_SCHEMA_SQL } from "./openclaw-state-schema.js"; +import { UpdateSchemaRefusalError } from "./openclaw-update-schema-refusal.js"; + +export function repairStateSchema( + pathname: string, + env: NodeJS.ProcessEnv, + scope: "automatic" | "doctor" | "readability", +): { + changes: string[]; + warnings: string[]; +} { + ensureOpenClawStatePermissions(pathname, env); + // This private handle rebuilds referenced tables and is closed after repair. + const db = openNodeSqliteDatabase(pathname, { enableForeignKeyConstraints: false }); + const rebuiltIndexNames = new Set(); + let ownershipRefused = false; + try { + setSqliteBusyTimeout(db, OPENCLAW_SQLITE_BUSY_TIMEOUT_MS); + let repairAdmittedSchema: (() => string[]) | undefined; + if (scope === "automatic") { + assertSupportedStateSchemaVersion(db, pathname); + } else { + repairAdmittedSchema = prepareStateDatabaseSchemaRepair(db, pathname, env); + if (scope === "readability") { + return { + changes: runSqliteImmediateTransactionSync(db, repairAdmittedSchema, { + busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + databaseLabel: pathname, + operationLabel: "state.schema.readability-repair", + }), + warnings: [], + }; + } + } + const applied: string[] = []; + const changes = runStateSchemaMigrationTransaction( + db, + pathname, + () => { + if (!repairAdmittedSchema) { + assertOpenClawStateWriteAllowed({ database: db, databasePath: pathname, env }); + } + applied.push(...recoverOrphanTaskDeliveryRows(db, pathname)); + const previousVersion = readStateSchemaMigrationVersion(db); + const preAuditSchema = previousVersion === 1 && !tableExists(db, "audit_events"); + if (preAuditSchema) { + assertOpenClawStateDatabaseOwner(db, { pathname }); + } + if (previousVersion === OPENCLAW_STATE_SCHEMA_VERSION) { + for (const name of verifyAndRepairCanonicalSqliteIndexes( + db, + pathname, + OPENCLAW_STATE_SCHEMA_SQL, + { allowMissingColumns: true }, + )) { + rebuiltIndexNames.add(name); + } + // Current-schema doctor repair may normalize recognized columns or + // table options, but it must never recreate a missing table empty. + assertSqliteSchemaTablesPresent(db, pathname, OPENCLAW_STATE_SCHEMA_SQL, { + allowedMissingTables: LAZY_ADDITIVE_STATE_TABLES, + }); + } else { + openClawStateMigrationAssertions.get(previousVersion)?.(db, { pathname }); + assertSqliteIntegrity(db, pathname); + } + dropLegacyStateTables(db); + applied.push(...retirements.runRetiredStateTableMigrations(db, previousVersion)); + if (migrateSingletonStateFoldInV12(db, previousVersion)) { + applied.push("Folded singleton state tables into config_machine_state (v12)"); + } + if (migrateWorkerPlacementExecutionModeSchema(db, previousVersion)) { + applied.push("Migrated cloud worker placements to execution modes"); + } + applied.push( + ...describeAgentPathMigration(migrateAgentPaths(db, previousVersion, pathname)), + ); + if (repairAgentDatabasesCompositePrimaryKey(db)) { + applied.push(`Migrated shared state agent database registry primary key → agent_id,path`); + } + if (repairAuditEventsSchema(db)) { + applied.push( + `Migrated shared state audit event ledger → versioned message lifecycle schema`, + ); + } + applied.push(...operatorApprovalMigration.repairOperatorApprovalSchema(db)); + const needsSessionWatchMigration = + sessionWatchMigration.needsSessionWatchCursorProvenanceMigration(db, previousVersion); + const sessionWatchResult = sessionWatchMigration.migrateSessionWatchCursorProvenance(db); + if (needsSessionWatchMigration) { + applied.push( + `Migrated shared state session watch cursors → provenance column (${sessionWatchResult.migratedAmbientWatches} ambient, ${sessionWatchResult.removedLegacySentinels} sentinels removed)`, + ); + } + assertCanonicalStateSchemaShape(db, pathname); + // Recognized schema-1 stores predate audit; Doctor must finish their schema + // before its later read-only workspace and agent readers can consume it. + if (preAuditSchema || tableExists(db, "audit_events")) { + ensureAdditiveStateColumns(db); + for (const migration of versionedStateMigrations) { + if (migration.migrate(db, previousVersion)) { + applied.push(migration.applied); + } + } + executeCanonicalStateSchema(db, { + includeVersionLazyAdditiveTables: previousVersion !== OPENCLAW_STATE_SCHEMA_VERSION, + }); + if (previousVersion < OPENCLAW_STATE_STRICT_SCHEMA_VERSION) { + repairLegacyGatewayRestartHandoffsForStrictMigration(db); + ensureFirstUseAdditiveStateColumnsForStrictMigration(db); + } + const strictMigration = migrateSqliteSchemaToStrictInTransaction( + db, + getOpenClawStateRuntimeSchema({ + includeVersionLazyAdditiveTables: previousVersion !== OPENCLAW_STATE_SCHEMA_VERSION, + }), + { databaseLabel: pathname }, + ); + if (strictMigration.migratedTables.length > 0) { + applied.push( + `Migrated shared state tables to SQLite STRICT typing (${strictMigration.migratedTables.length})`, + ); + } + for (const name of repairCanonicalSqliteIndexes(db, pathname, OPENCLAW_STATE_SCHEMA_SQL, { + verifyPhysicalIntegrity: false, + })) { + rebuiltIndexNames.add(name); + } + } + markCurrentStateSchemaVersion(db, { + createMetadataIfMissing: previousVersion < OPENCLAW_STATE_SCHEMA_VERSION, + }); + if (readStateSchemaContentVersion(db) === OPENCLAW_STATE_SCHEMA_VERSION) { + assertCurrentStateRuntimeSchema(db, pathname); + } + if (rebuiltIndexNames.size > 0) { + applied.push(`Rebuilt canonical shared-state SQLite indexes (${rebuiltIndexNames.size})`); + } + return applied; + }, + { + busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + databaseLabel: pathname, + operationLabel: "state.schema.repair", + }, + () => { + applied.push(...(repairAdmittedSchema?.() ?? [])); + }, + ); + const quarantineCleared = clearOpenClawDatabaseQuarantine(pathname, { env }); + clearOpenClawStateDatabaseOpenFailure(pathname); + return { + changes, + warnings: quarantineCleared + ? [] + : [ + `Persisted quarantine record for ${pathname} could not be cleared; rerun openclaw doctor --fix so the repaired database is not refused again.`, + ], + }; + } catch (err) { + if (err instanceof UpdateSchemaRefusalError) { + throw err; + } + if (err instanceof OpenClawStateOwnershipError) { + ownershipRefused = true; + throw err; + } + // Reaching this catch inside doctor means repair itself refused or failed, + // so the runtime asserts' "run openclaw doctor --fix" advice is circular here. + const reason = String(err).replace( + /has a legacy ([a-z ]+) schema; run openclaw doctor --fix to migrate it\./u, + "has a legacy $1 schema; automatic repair refused the unrecognized schema shape.", + ); + return { + changes: [], + warnings: [`Failed migrating shared state database schema at ${pathname}: ${reason}`], + }; + } finally { + if (db.isOpen) { + clearNodeSqliteKyselyCacheForDatabase(db); + // Rollback cleanup may have closed the handle after an unrecoverable + // transaction failure; double-close throws ERR_INVALID_STATE and would + // discard the diagnostic warnings returned by the catch above. + db.close(); + } + if (!ownershipRefused) { + ensureOpenClawStatePermissions(pathname, env); + } + } +} diff --git a/src/state/openclaw-state-db-schema-migration-required.ts b/src/state/openclaw-state-db-schema-migration-required.ts index e182119c302a..3024413c0ce4 100644 --- a/src/state/openclaw-state-db-schema-migration-required.ts +++ b/src/state/openclaw-state-db-schema-migration-required.ts @@ -1,5 +1,8 @@ import { StartupMaintenanceRequiredError } from "../infra/startup-maintenance-required.js"; +export const LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX = + "idx_skill_workshop_collection_reviews_workspace_time"; + type OpenClawStateDatabaseSchemaMigrationRequiredKind = | "agent-databases-composite-primary-key" | "audit-events-v2" @@ -17,3 +20,21 @@ export class OpenClawStateDatabaseSchemaMigrationRequiredError extends StartupMa this.name = "OpenClawStateDatabaseSchemaMigrationRequiredError"; } } + +/** Runtime readers report malformed legacy state without entering repair mode. */ +export function normalizeOpenClawStateSchemaReadError(error: unknown, pathname: string): unknown { + if ( + error instanceof Error && + error.message.startsWith( + `malformed database schema (${LEGACY_SKILL_WORKSHOP_COLLECTION_REVIEWS_INDEX})`, + ) + ) { + const required = new OpenClawStateDatabaseSchemaMigrationRequiredError( + "legacy-workshop-review-index", + pathname, + ); + required.cause = error; + return required; + } + return error; +} diff --git a/src/state/openclaw-state-db-schema-version.ts b/src/state/openclaw-state-db-schema-version.ts index 5ca002d3d2d9..cec7d4681329 100644 --- a/src/state/openclaw-state-db-schema-version.ts +++ b/src/state/openclaw-state-db-schema-version.ts @@ -7,6 +7,7 @@ import { } from "../infra/sqlite-user-version.js"; import { OPENCLAW_STATE_SCHEMA_VERSION } from "./openclaw-state-db-contract.js"; import { tableExists, tableHasColumn } from "./openclaw-state-db-schema-helpers.js"; +import { normalizeOpenClawStateSchemaReadError } from "./openclaw-state-db-schema-migration-required.js"; import type { DB } from "./openclaw-state-db.generated.js"; // Read-only clients need schema admission without loading updater publication policy. @@ -111,16 +112,20 @@ export function readStateSchemaMigrationVersion(db: DatabaseSync): number { } export function assertSupportedStateSchemaVersion(db: DatabaseSync, pathname: string): number { - const userVersion = readSqliteUserVersion(db); - const contentVersion = - userVersion > OPENCLAW_STATE_SCHEMA_VERSION ? userVersion : readStateSchemaContentVersion(db); - if (contentVersion > OPENCLAW_STATE_SCHEMA_VERSION) { - throw createNewerSqliteSchemaVersionError( - "OpenClaw state database", - pathname, - contentVersion, - OPENCLAW_STATE_SCHEMA_VERSION, - ); + try { + const userVersion = readSqliteUserVersion(db); + const contentVersion = + userVersion > OPENCLAW_STATE_SCHEMA_VERSION ? userVersion : readStateSchemaContentVersion(db); + if (contentVersion > OPENCLAW_STATE_SCHEMA_VERSION) { + throw createNewerSqliteSchemaVersionError( + "OpenClaw state database", + pathname, + contentVersion, + OPENCLAW_STATE_SCHEMA_VERSION, + ); + } + return userVersion; + } catch (error) { + throw normalizeOpenClawStateSchemaReadError(error, pathname); } - return userVersion; } diff --git a/src/state/openclaw-state-db.test.ts b/src/state/openclaw-state-db.test.ts index 4408a6435f81..6ed73e1d6265 100644 --- a/src/state/openclaw-state-db.test.ts +++ b/src/state/openclaw-state-db.test.ts @@ -46,7 +46,7 @@ import { FIRST_USE_STATE_TABLES, OPENCLAW_STATE_SCHEMA_VERSION, } from "./openclaw-state-db-contract.js"; -import { hasDanglingSkillWorkshopCollectionReviewIndex } from "./openclaw-state-db-dangling-workshop-index.js"; +import { hasDanglingSkillWorkshopCollectionReviewIndex } from "./openclaw-state-db-doctor-schema.js"; import { prepareStateDatabaseSchemaRepair } from "./openclaw-state-db-maintenance.js"; import { ensureGitHubPublicationSchema } from "./openclaw-state-db-schema-additive.js"; import { OpenClawStateDatabaseSchemaMigrationRequiredError } from "./openclaw-state-db-schema-migration-required.js"; @@ -1758,56 +1758,80 @@ describe("openclaw state database", () => { ).toEqual({ review_id: "review-v15", backup_id: "backup-v15" }); }); - it("requires Doctor to repair the dangling v15 Workshop review index", () => { - const stateDir = createTempStateDir(); - const options = { env: { OPENCLAW_STATE_DIR: stateDir } }; - const databasePath = materializeCurrentStateDatabase(stateDir); - const { DatabaseSync } = requireNodeSqlite(); - const database = new DatabaseSync(databasePath); - database - .prepare( - `INSERT INTO skill_workshop_collection_reviews ( + it.each([16, OPENCLAW_STATE_SCHEMA_VERSION])( + "requires Doctor to repair the dangling Workshop review index in schema v%i", + (version) => { + const stateDir = createTempStateDir(); + const options = { env: { OPENCLAW_STATE_DIR: stateDir } }; + const databasePath = materializeCurrentStateDatabase(stateDir); + const { DatabaseSync } = requireNodeSqlite(); + const database = new DatabaseSync(databasePath); + database + .prepare( + `INSERT INTO skill_workshop_collection_reviews ( review_id, owner_agent_id, backup_id, create_time, kept_names_json, written_names_json, dropped_json ) VALUES ('review-preserved', 'main', 'backup-preserved', 1, '[]', '[]', '[]')`, - ) - .run(); - database.close(); - const rootpage = createDanglingSkillWorkshopReviewIndex(databasePath); + ) + .run(); + if (version === 16) { + removePreparedWorkerOwnershipColumns(database); + database.exec(` + PRAGMA user_version = 16; + UPDATE schema_meta SET schema_version = 16 WHERE meta_key = 'primary'; + `); + } + database.close(); + const rootpage = createDanglingSkillWorkshopReviewIndex(databasePath); - const defensiveProbe = new DatabaseSync(databasePath); - expect(hasDanglingSkillWorkshopCollectionReviewIndex(defensiveProbe)).toBe(true); - const schemaVersion = readSqliteNumberPragma(defensiveProbe, "schema_version"); - defensiveProbe.exec(`PRAGMA schema_version = ${schemaVersion + 1};`); - expect(readSqliteNumberPragma(defensiveProbe, "schema_version")).toBe(schemaVersion); - defensiveProbe.close(); + const defensiveProbe = new DatabaseSync(databasePath); + expect(hasDanglingSkillWorkshopCollectionReviewIndex(defensiveProbe)).toBe(true); + const schemaVersion = readSqliteNumberPragma(defensiveProbe, "schema_version"); + defensiveProbe.exec(`PRAGMA schema_version = ${schemaVersion + 1};`); + expect(readSqliteNumberPragma(defensiveProbe, "schema_version")).toBe(schemaVersion); + defensiveProbe.close(); - expect(() => openOpenClawStateDatabase(options)).toThrow( - /legacy-workshop-review-index.*openclaw doctor --fix/u, - ); - expect(readDanglingSkillWorkshopReviewIndex(databasePath)).toMatchObject({ rootpage }); + expect(() => openOpenClawStateDatabase(options)).toThrow( + /legacy-workshop-review-index.*openclaw doctor --fix/u, + ); + expect(() => repairOpenClawStateDatabaseSchemaIfNeeded(options)).toThrow( + /legacy-workshop-review-index.*openclaw doctor --fix/u, + ); + expect(readDanglingSkillWorkshopReviewIndex(databasePath)).toMatchObject({ rootpage }); - expect(repairOpenClawStateDatabaseSchema(options)).toEqual({ - changes: ["Removed dangling legacy Skill Workshop review index"], - warnings: [], - }); + expect(repairOpenClawStateDatabaseSchema(options)).toEqual({ + changes: + version === 16 + ? expect.arrayContaining([ + "Removed dangling legacy Skill Workshop review index", + "Recorded prepared worker ownership and one-use lifecycle (v17)", + ]) + : ["Removed dangling legacy Skill Workshop review index"], + warnings: [], + }); - const repaired = new DatabaseSync(databasePath, { readOnly: true }); - try { - expect( - repaired - .prepare( - "SELECT review_id, backup_id FROM skill_workshop_collection_reviews WHERE review_id = 'review-preserved'", - ) - .get(), - ).toEqual({ review_id: "review-preserved", backup_id: "backup-preserved" }); - expect(repaired.prepare("PRAGMA integrity_check").all()).toEqual([{ integrity_check: "ok" }]); - expect(readDanglingSkillWorkshopReviewIndex(databasePath)).toBeUndefined(); - } finally { - repaired.close(); - } - expect(() => openOpenClawStateDatabase(options)).not.toThrow(); - }); + const repaired = new DatabaseSync(databasePath, { readOnly: true }); + try { + expect( + repaired + .prepare( + "SELECT review_id, backup_id FROM skill_workshop_collection_reviews WHERE review_id = 'review-preserved'", + ) + .get(), + ).toEqual({ review_id: "review-preserved", backup_id: "backup-preserved" }); + expect(repaired.prepare("PRAGMA integrity_check").all()).toEqual([ + { integrity_check: "ok" }, + ]); + expect(readSqliteNumberPragma(repaired, "user_version")).toBe( + OPENCLAW_STATE_SCHEMA_VERSION, + ); + expect(readDanglingSkillWorkshopReviewIndex(databasePath)).toBeUndefined(); + } finally { + repaired.close(); + } + expect(() => openOpenClawStateDatabase(options)).not.toThrow(); + }, + ); it("does not repair a dangling Workshop index in a newer unsupported schema", () => { const stateDir = createTempStateDir(); diff --git a/src/state/openclaw-state-db.ts b/src/state/openclaw-state-db.ts index ce7cb0da3e65..69e342aa07ee 100644 --- a/src/state/openclaw-state-db.ts +++ b/src/state/openclaw-state-db.ts @@ -1,8 +1,5 @@ -// OpenClaw state database manages shared persisted state and migrations. import { existsSync } from "node:fs"; import type { DatabaseSync } from "node:sqlite"; -import { clearNodeSqliteKyselyCacheForDatabase } from "../infra/kysely-sync.js"; -import { openNodeSqliteDatabase } from "../infra/node-sqlite.js"; import { normalizeSqliteNonNegativeInteger, readSqliteBusyTimeout, @@ -16,7 +13,6 @@ import { verifyAndRepairCanonicalSqliteIndexes, } from "../infra/sqlite-index-schema.js"; import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js"; -import { assertSqliteSchemaTablesPresent } from "../infra/sqlite-schema-contract.js"; import { prepareSqliteReadOnlyLocation } from "../infra/sqlite-snapshot-source.js"; import { migrateSqliteSchemaToStrictInTransaction } from "../infra/sqlite-strict.js"; import type { SqliteTransactionOptions } from "../infra/sqlite-transaction.js"; @@ -27,26 +23,23 @@ import { } from "../infra/state-database-coordinator.js"; import { migrateLegacyCronRunLogsToTaskRuns } from "../infra/state-migrations.cron-run-logs.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; -import { clearOpenClawDatabaseQuarantine } from "./openclaw-quarantine-store.js"; import { getOpenClawDatabaseMaintenanceScope, observeOpenClawDatabaseMaintenanceResource, } from "./openclaw-state-db-async-lifecycle.js"; -import { repairAuditEventsSchema } from "./openclaw-state-db-audit-migration.js"; import { openClawStateDatabaseCache as stateDbCache, recordOpenClawStateDatabaseOpenFailure, - clearOpenClawStateDatabaseOpenFailure, } from "./openclaw-state-db-cache.js"; import { OPENCLAW_DATABASE_SCHEMA_DOCS_URL, - LAZY_ADDITIVE_STATE_TABLES, OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, OPENCLAW_STATE_SCHEMA_VERSION, OPENCLAW_STATE_STRICT_SCHEMA_VERSION, type OpenClawStateDatabase, type OpenClawStateDatabaseOptions, } from "./openclaw-state-db-contract.js"; +import { openDoctorStateSchemaReadAdmission } from "./openclaw-state-db-doctor-schema.js"; import { assertCurrentStateRuntimeSchema, isOpenClawStateSchemaFastPathEligible, @@ -54,7 +47,6 @@ import { } from "./openclaw-state-db-fast-path.js"; import { assertOpenClawStateDatabaseForMaintenance, - assertOpenClawStateDatabaseOwner, markCurrentStateSchemaVersion, openClawStateMigrationAssertions, resolveDatabasePath, @@ -62,25 +54,22 @@ import { runStateSchemaMigrationTransaction, writeCurrentStateSchemaMetadata, executeCanonicalStateSchema, - prepareStateDatabaseSchemaRepair, } from "./openclaw-state-db-maintenance.js"; import { openUnpublishedStateDatabase } from "./openclaw-state-db-open.js"; -import * as operatorApprovalMigration from "./openclaw-state-db-operator-approval-migration.js"; import { ensureOpenClawStatePermissions } from "./openclaw-state-db-permissions.js"; import { openOpenClawStateReadConnection } from "./openclaw-state-db-read-connection.js"; import { withExistingOpenClawStateDatabaseReadOnly } from "./openclaw-state-db-readonly.js"; +import { repairStateSchema } from "./openclaw-state-db-repair.js"; import { ensureAdditiveStateColumns, ensureFirstUseAdditiveStateColumnsForStrictMigration, } from "./openclaw-state-db-schema-additive.js"; -import { tableExists } from "./openclaw-state-db-schema-helpers.js"; import { type AgentDatabasePathMigrationSummary as AgentPathSummary, assertCanonicalStateSchemaShape, dropLegacyStateTables, migrateAgentDatabaseRelativePaths as migrateAgentPaths, migrateWorkerPlacementExecutionModeSchema, - repairAgentDatabasesCompositePrimaryKey, repairLegacyGatewayRestartHandoffsForStrictMigration, } from "./openclaw-state-db-schema-repair.js"; import { migrateSingletonStateFoldInV12 } from "./openclaw-state-db-schema-v12-foldin.js"; @@ -96,16 +85,14 @@ import { withOpenClawStateStartupCheckpointConnection, } from "./openclaw-state-db-startup-checkpoint.js"; import * as retirements from "./openclaw-state-db-table-retirements.js"; -import { recoverOrphanTaskDeliveryRows } from "./openclaw-state-db-task-delivery-recovery.js"; import { runCoordinatedStateTransaction, withSharedStateWriteCoordinator, } from "./openclaw-state-db-write-coordination.js"; -import { describeAgentPathMigration, warnAgentPathMigration } from "./openclaw-state-db.paths.js"; +import { warnAgentPathMigration } from "./openclaw-state-db.paths.js"; import { assertOpenClawStateWriteAllowed, isOpenClawStateWriteContentionError, - OpenClawStateOwnershipError, runWithOpenClawStateWriteAccess, } from "./openclaw-state-ownership.js"; import { getOpenClawStateRuntimeSchema } from "./openclaw-state-schema-compatibility.js"; @@ -114,7 +101,7 @@ import { type StateSchemaPublicationBlocker, } from "./openclaw-state-schema-publication.js"; import { OPENCLAW_STATE_SCHEMA_SQL } from "./openclaw-state-schema.js"; -import { UpdateSchemaRefusalError } from "./openclaw-update-schema-refusal.js"; + export { registerOpenClawStateDatabaseLifecycleListener } from "./openclaw-state-db-cache.js"; export { OPENCLAW_DATABASE_SCHEMA_DOCS_URL, OPENCLAW_SQLITE_BUSY_TIMEOUT_MS }; @@ -138,167 +125,6 @@ function assertOpenClawStateDatabaseFreshOpenAllowed( const stateDbLog = createSubsystemLogger("state/db"); const deferredStateDatabases = new WeakSet(); -function repairStateSchema(pathname: string, env: NodeJS.ProcessEnv) { - ensureOpenClawStatePermissions(pathname, env); - const db = openNodeSqliteDatabase(pathname); - const rebuiltIndexNames = new Set(); - let ownershipRefused = false; - try { - db.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`); - const repairAdmittedSchema = prepareStateDatabaseSchemaRepair(db, pathname, env); - db.exec("PRAGMA foreign_keys = OFF;"); - const changes = runStateSchemaMigrationTransaction( - db, - pathname, - () => { - const applied = repairAdmittedSchema(); - applied.push(...recoverOrphanTaskDeliveryRows(db, pathname)); - const previousVersion = readStateSchemaMigrationVersion(db); - const preAuditSchema = previousVersion === 1 && !tableExists(db, "audit_events"); - if (preAuditSchema) { - assertOpenClawStateDatabaseOwner(db, { pathname }); - } - if (previousVersion === OPENCLAW_STATE_SCHEMA_VERSION) { - for (const name of verifyAndRepairCanonicalSqliteIndexes( - db, - pathname, - OPENCLAW_STATE_SCHEMA_SQL, - { allowMissingColumns: true }, - )) { - rebuiltIndexNames.add(name); - } - // Current-schema doctor repair may normalize recognized columns or - // table options, but it must never recreate a missing table empty. - assertSqliteSchemaTablesPresent(db, pathname, OPENCLAW_STATE_SCHEMA_SQL, { - allowedMissingTables: LAZY_ADDITIVE_STATE_TABLES, - }); - } else { - openClawStateMigrationAssertions.get(previousVersion)?.(db, { pathname }); - assertSqliteIntegrity(db, pathname); - } - dropLegacyStateTables(db); - applied.push(...retirements.runRetiredStateTableMigrations(db, previousVersion)); - if (migrateSingletonStateFoldInV12(db, previousVersion)) { - applied.push("Folded singleton state tables into config_machine_state (v12)"); - } - if (migrateWorkerPlacementExecutionModeSchema(db, previousVersion)) { - applied.push("Migrated cloud worker placements to execution modes"); - } - applied.push( - ...describeAgentPathMigration(migrateAgentPaths(db, previousVersion, pathname)), - ); - if (repairAgentDatabasesCompositePrimaryKey(db)) { - applied.push(`Migrated shared state agent database registry primary key → agent_id,path`); - } - if (repairAuditEventsSchema(db)) { - applied.push( - `Migrated shared state audit event ledger → versioned message lifecycle schema`, - ); - } - applied.push(...operatorApprovalMigration.repairOperatorApprovalSchema(db)); - const needsSessionWatchMigration = - sessionWatchMigration.needsSessionWatchCursorProvenanceMigration(db, previousVersion); - const sessionWatchResult = sessionWatchMigration.migrateSessionWatchCursorProvenance(db); - if (needsSessionWatchMigration) { - applied.push( - `Migrated shared state session watch cursors → provenance column (${sessionWatchResult.migratedAmbientWatches} ambient, ${sessionWatchResult.removedLegacySentinels} sentinels removed)`, - ); - } - assertCanonicalStateSchemaShape(db, pathname); - // Recognized schema-1 stores predate audit; Doctor must finish their schema - // before its later read-only workspace and agent readers can consume it. - if (preAuditSchema || tableExists(db, "audit_events")) { - ensureAdditiveStateColumns(db); - for (const migration of versionedStateMigrations) { - if (migration.migrate(db, previousVersion)) { - applied.push(migration.applied); - } - } - executeCanonicalStateSchema(db, { - includeVersionLazyAdditiveTables: previousVersion !== OPENCLAW_STATE_SCHEMA_VERSION, - }); - if (previousVersion < OPENCLAW_STATE_STRICT_SCHEMA_VERSION) { - repairLegacyGatewayRestartHandoffsForStrictMigration(db); - ensureFirstUseAdditiveStateColumnsForStrictMigration(db); - } - const strictMigration = migrateSqliteSchemaToStrictInTransaction( - db, - getOpenClawStateRuntimeSchema({ - includeVersionLazyAdditiveTables: previousVersion !== OPENCLAW_STATE_SCHEMA_VERSION, - }), - { databaseLabel: pathname }, - ); - if (strictMigration.migratedTables.length > 0) { - applied.push( - `Migrated shared state tables to SQLite STRICT typing (${strictMigration.migratedTables.length})`, - ); - } - for (const name of repairCanonicalSqliteIndexes(db, pathname, OPENCLAW_STATE_SCHEMA_SQL, { - verifyPhysicalIntegrity: false, - })) { - rebuiltIndexNames.add(name); - } - } - markCurrentStateSchemaVersion(db, { - createMetadataIfMissing: previousVersion < OPENCLAW_STATE_SCHEMA_VERSION, - }); - if (readStateSchemaContentVersion(db) === OPENCLAW_STATE_SCHEMA_VERSION) { - assertCurrentStateRuntimeSchema(db, pathname); - } - if (rebuiltIndexNames.size > 0) { - applied.push(`Rebuilt canonical shared-state SQLite indexes (${rebuiltIndexNames.size})`); - } - return applied; - }, - { - busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, - databaseLabel: pathname, - operationLabel: "state.schema.repair", - }, - ); - const quarantineCleared = clearOpenClawDatabaseQuarantine(pathname, { env }); - clearOpenClawStateDatabaseOpenFailure(pathname); - return { - changes, - warnings: quarantineCleared - ? [] - : [ - `Persisted quarantine record for ${pathname} could not be cleared; rerun openclaw doctor --fix so the repaired database is not refused again.`, - ], - }; - } catch (err) { - if (err instanceof UpdateSchemaRefusalError) { - throw err; - } - if (err instanceof OpenClawStateOwnershipError) { - ownershipRefused = true; - throw err; - } - // Reaching this catch inside doctor means repair itself refused or failed, - // so the runtime asserts' "run openclaw doctor --fix" advice is circular here. - const reason = String(err).replace( - /has a legacy ([a-z ]+) schema; run openclaw doctor --fix to migrate it\./u, - "has a legacy $1 schema; automatic repair refused the unrecognized schema shape.", - ); - return { - changes: [], - warnings: [`Failed migrating shared state database schema at ${pathname}: ${reason}`], - }; - } finally { - if (db.isOpen) { - db.exec("PRAGMA foreign_keys = ON;"); - clearNodeSqliteKyselyCacheForDatabase(db); - // Rollback cleanup may have closed the handle after an unrecoverable - // transaction failure; double-close throws ERR_INVALID_STATE and would - // discard the diagnostic warnings returned by the catch above. - db.close(); - } - if (!ownershipRefused) { - ensureOpenClawStatePermissions(pathname, env); - } - } -} - export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabaseOptions = {}): { changes: string[]; warnings: string[]; @@ -309,9 +135,39 @@ export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabase return { changes: [], warnings: [] }; } return runWithOpenClawStateWriteAccess( - { databasePath: pathname, env }, + { + databasePath: pathname, + env, + openStateSchemaReadAdmission: openDoctorStateSchemaReadAdmission, + }, "state schema repair", - () => withStateSchemaFence({ databasePath: pathname }, () => repairStateSchema(pathname, env)), + () => + withStateSchemaFence({ databasePath: pathname }, () => + repairStateSchema(pathname, env, "doctor"), + ), + ); +} + +/** Make exact legacy catalog damage readable before Doctor loads config-dependent state. */ +export function repairOpenClawStateDatabaseReadabilityForDoctor( + options: OpenClawStateDatabaseOptions = {}, +): { changes: string[]; warnings: string[] } { + const env = options.env ?? process.env; + const pathname = resolveDatabasePath(options); + if (!existsSync(pathname)) { + return { changes: [], warnings: [] }; + } + return runWithOpenClawStateWriteAccess( + { + databasePath: pathname, + env, + openStateSchemaReadAdmission: openDoctorStateSchemaReadAdmission, + }, + "Doctor state readability repair", + () => + withStateSchemaFence({ databasePath: pathname }, () => + repairStateSchema(pathname, env, "readability"), + ), ); } @@ -333,7 +189,9 @@ export function repairOpenClawStateDatabaseSchemaIfNeeded( "state schema repair preflight/repair", () => needsOpenClawStateDatabaseSchemaRepair(pathname) - ? withStateSchemaFence({ databasePath: pathname }, () => repairStateSchema(pathname, env)) + ? withStateSchemaFence({ databasePath: pathname }, () => + repairStateSchema(pathname, env, "automatic"), + ) : { changes: [], warnings: [] }, ); } diff --git a/src/state/openclaw-state-ownership.ts b/src/state/openclaw-state-ownership.ts index 425f1e2787fb..13f0c3779b69 100644 --- a/src/state/openclaw-state-ownership.ts +++ b/src/state/openclaw-state-ownership.ts @@ -19,9 +19,12 @@ import { acquireStateDatabaseCoordinator, StateDatabaseCoordinatorContentionError, } from "../infra/state-database-coordinator.js"; -import { OPENCLAW_SQLITE_BUSY_TIMEOUT_MS } from "./openclaw-state-db-contract.js"; -import { openDanglingWorkshopIndexReadAdmission } from "./openclaw-state-db-dangling-workshop-index.js"; +import { + OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + type OpenClawStateSchemaReadAdmission, +} from "./openclaw-state-db-contract.js"; import { tableExists } from "./openclaw-state-db-schema-helpers.js"; +import { normalizeOpenClawStateSchemaReadError } from "./openclaw-state-db-schema-migration-required.js"; export const STATE_SUPERVISION_KEY = "gateway.supervision"; const MAX_OWNERSHIP_TIMESTAMP_MS = 8_640_000_000_000_000; @@ -120,8 +123,6 @@ export function inspectOpenClawStateOwnershipFromDatabase( databasePath: string, configMachineStateTableReady = false, ): OpenClawExternalStateOwnership | null { - database.enableDefensive?.(false); - database.exec("PRAGMA writable_schema = ON;"); try { if (!configMachineStateTableReady && !tableExists(database, "config_machine_state")) { return null; @@ -136,30 +137,27 @@ export function inspectOpenClawStateOwnershipFromDatabase( throw new OpenClawStateOwnershipMetadataError(databasePath, "reserved value is not text"); } return parseExternalOwnership(row.value_json, databasePath); - } finally { - try { - database.exec("PRAGMA writable_schema = OFF;"); - } finally { - database.enableDefensive?.(true); - } + } catch (error) { + throw normalizeOpenClawStateSchemaReadError(error, databasePath); } } function inspectOwnershipThroughConnection( location: string, databasePath: string, + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission, ): OpenClawExternalStateOwnership | null { const database = openNodeSqliteDatabase(location, { readOnly: true }); - let closeSchemaReadAdmission: (() => void) | undefined; + let closeAdmission: (() => void) | undefined; try { - closeSchemaReadAdmission = openDanglingWorkshopIndexReadAdmission(database); + closeAdmission = openStateSchemaReadAdmission?.(database); database.exec( `PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS}; PRAGMA query_only = ON; PRAGMA trusted_schema = OFF;`, ); return inspectOpenClawStateOwnershipFromDatabase(database, databasePath); } finally { try { - closeSchemaReadAdmission?.(); + closeAdmission?.(); } finally { database.close(); } @@ -177,7 +175,11 @@ function inspectJournalAwarePublicOwnership( } } -function inspectOwnershipWhileCoordinatorHeld(databasePath: string, busyTimeoutMs: number) { +function inspectOwnershipWhileCoordinatorHeld( + databasePath: string, + busyTimeoutMs: number, + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission, +) { const resolvedPath = path.resolve(databasePath); if (!existsSync(resolvedPath)) { return null; @@ -185,11 +187,17 @@ function inspectOwnershipWhileCoordinatorHeld(databasePath: string, busyTimeoutM // Write admission owns locking and recovery while the coordinator is held. // Inspect the live committed view without cloning a potentially busy family. const database = openNodeSqliteDatabase(resolveExistingSqliteFileUri(resolvedPath)); + let closeAdmission: (() => void) | undefined; try { + closeAdmission = openStateSchemaReadAdmission?.(database); database.exec(`PRAGMA busy_timeout = ${busyTimeoutMs}; PRAGMA trusted_schema = OFF;`); return inspectOpenClawStateOwnershipFromDatabase(database, resolvedPath); } finally { - database.close(); + try { + closeAdmission?.(); + } finally { + database.close(); + } } } @@ -238,6 +246,7 @@ function acquireOpenClawStateWriteAccess(options: { databasePath: string; busyTimeoutMs?: number; env?: NodeJS.ProcessEnv; + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission; }): { release: () => void } { const resolvedPath = path.resolve(options.databasePath); const busyTimeoutMs = normalizeSqliteNonNegativeInteger( @@ -248,7 +257,11 @@ function acquireOpenClawStateWriteAccess(options: { try { quarantineOrphanedSqliteSidecars(resolvedPath); assertOwnershipAllowsWrite( - inspectOwnershipWhileCoordinatorHeld(resolvedPath, busyTimeoutMs), + inspectOwnershipWhileCoordinatorHeld( + resolvedPath, + busyTimeoutMs, + options.openStateSchemaReadAdmission, + ), resolvedPath, options.env ?? process.env, ); @@ -274,7 +287,12 @@ function acquireOpenClawStateWriteAccess(options: { } export function runWithOpenClawStateWriteAccess( - options: { databasePath: string; busyTimeoutMs?: number; env?: NodeJS.ProcessEnv }, + options: { + databasePath: string; + busyTimeoutMs?: number; + env?: NodeJS.ProcessEnv; + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission; + }, operationLabel: string, operation: () => T, ): T { @@ -291,6 +309,7 @@ export async function assertOpenClawStateWriteAllowedAtPath(options: { env?: NodeJS.ProcessEnv; recoverOrphanedSidecars?: boolean; signal?: AbortSignal; + openStateSchemaReadAdmission?: OpenClawStateSchemaReadAdmission; }): Promise { options.signal?.throwIfAborted(); const databasePath = path.resolve(options.databasePath); @@ -317,7 +336,11 @@ export async function assertOpenClawStateWriteAllowedAtPath(options: { try { options.signal?.throwIfAborted(); assertOwnershipAllowsWrite( - inspectOwnershipThroughConnection(prepared.location, databasePath), + inspectOwnershipThroughConnection( + prepared.location, + databasePath, + options.openStateSchemaReadAdmission, + ), databasePath, env, ); diff --git a/test/scripts/docker-e2e-plan.test.ts b/test/scripts/docker-e2e-plan.test.ts index eccbe6d5e57c..8af1ccef977f 100644 --- a/test/scripts/docker-e2e-plan.test.ts +++ b/test/scripts/docker-e2e-plan.test.ts @@ -1412,6 +1412,29 @@ await import('./scripts/check-docker-e2e-boundaries.mts');`, } }); + it("pins opt-in Workshop Doctor recovery to published 9.4 without credentials", () => { + const plan = planFor({ + selectedLaneNames: ["published-upgrade-survivor"], + upgradeSurvivorBaselines: "2026.9.3 2026.9.4 2026.9.5", + upgradeSurvivorScenarios: "workshop-doctor-recovery", + }); + const name = "published-upgrade-survivor-2026.9.4-workshop-doctor-recovery"; + expect(plan.lanes.map(summarizeLane)).toEqual([ + publishedUpgradeSurvivorLane(name, "openclaw@2026.9.4", "workshop-doctor-recovery"), + ]); + expect(plan.requiredPrepublishPluginPackages).toEqual([]); + expect(plan.credentials).toEqual([]); + for (const alias of ["reported-issues", "far-reaching"]) { + expect( + planFor({ + selectedLaneNames: ["published-upgrade-survivor"], + upgradeSurvivorBaselines: "2026.9.4", + upgradeSurvivorScenarios: alias, + }).lanes.map((lane) => lane.name), + ).not.toContain(name); + } + }); + it("runs sibling-source canaries from published 9.4 without provider or registry fixtures", () => { const plan = planFor({ selectedLaneNames: ["published-upgrade-survivor"], diff --git a/test/scripts/upgrade-survivor-assertions.test.ts b/test/scripts/upgrade-survivor-assertions.test.ts index 19d81685b422..0b3b04c4543c 100644 --- a/test/scripts/upgrade-survivor-assertions.test.ts +++ b/test/scripts/upgrade-survivor-assertions.test.ts @@ -394,27 +394,32 @@ describe("upgrade recovery result assertions", () => { ); }); - it("accepts clean updates for baselines that already have consent", () => { - const result = { - status: "ok", - after: { version: "2026.8.1" }, - steps: [{ name: "global update", exitCode: 0 }], - }; - expect(runJsonAssertion("assert-successful-update-json", result, "2026.8.1").status).toBe(0); - expect( - runJsonAssertion( - "assert-successful-update-json", - { - ...result, - steps: [{ name: "global update", exitCode: 1 }], - }, - "2026.8.1", - ).status, - ).not.toBe(0); - expect( - runPrefixedJsonAssertion("assert-successful-update-json", result, "2026.8.1").status, - ).toBe(0); - }); + it.each(["base", "workshop-doctor-recovery"])( + "accepts clean updates for baselines that already have consent (%s)", + (scenario) => + withEnv({ OPENCLAW_UPGRADE_SURVIVOR_SCENARIO: scenario }, () => { + const result = { + status: "ok", + after: { version: "2026.8.1" }, + steps: [{ name: "global update", exitCode: 0 }], + }; + const update = runJsonAssertion("assert-successful-update-json", result, "2026.8.1"); + expect(update.status, update.stderr).toBe(0); + expect( + runJsonAssertion( + "assert-successful-update-json", + { + ...result, + steps: [{ name: "global update", exitCode: 1 }], + }, + "2026.8.1", + ).status, + ).not.toBe(0); + expect( + runPrefixedJsonAssertion("assert-successful-update-json", result, "2026.8.1").status, + ).toBe(0); + }), + ); describe("missing Codex migration update result", () => { const scenarioEnv = { diff --git a/test/scripts/upgrade-survivor-migration-order.test.ts b/test/scripts/upgrade-survivor-migration-order.test.ts index 8cb1c02c1804..04c740f17cc4 100644 --- a/test/scripts/upgrade-survivor-migration-order.test.ts +++ b/test/scripts/upgrade-survivor-migration-order.test.ts @@ -1,7 +1,17 @@ -import { spawnSync } from "node:child_process"; -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { execFileSync, spawnSync } from "node:child_process"; +import { cpSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; +import { DatabaseSync } from "node:sqlite"; import { afterEach, describe, expect, it } from "vitest"; +import { + assertWorkshopDoctorRepair, + assertWorkshopRecoveredUpgrade, + assertWorkshopUpdateRefusal, + captureWorkshopBaseline, + captureWorkshopCandidate, + completeWorkshopRecovery, + seedWorkshopIndex, +} from "../../scripts/e2e/lib/upgrade-survivor/workshop-doctor-recovery.mjs"; import { resolveTestNodeExecPath } from "../../src/test-utils/node-process.js"; import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; @@ -24,10 +34,22 @@ function runFirstHop(scenario: string, automatic: boolean) { CURRENT_PHASE="$1" shift case "$CURRENT_PHASE" in - update-candidate) + install-baseline) baseline_spec="$OPENCLAW_UPGRADE_SURVIVOR_BASELINE" ;; + prepare-workshop-baseline) printf 'baseline-doctor\\n' >>"$HOME/events" ;; + capture-workshop-candidate) printf 'candidate-identity\\n' >>"$HOME/events" ;; + seed-workshop-baseline-index|seed-workshop-candidate-index) printf 'seed\\n' >>"$HOME/events" ;; + update-candidate|update-workshop-recovered-state) printf 'update\\n' >>"$HOME/events" if [ "$FIXTURE_AUTOMATIC" = 1 ]; then touch "$HOME/migrated"; fi ;; + repair-workshop-baseline|repair-workshop-candidate) printf 'explicit-doctor\\n' >>"$HOME/events" ;; + assert-workshop-published-refusal) + printf 'refusal\\n' >>"$HOME/events" + if [ "$FIXTURE_AUTOMATIC" = 1 ]; then return 42; fi + ;; + assert-workshop-baseline-repair|assert-workshop-candidate-repair|assert-workshop-recovered-upgrade) + printf 'verify\\n' >>"$HOME/events" + ;; doctor) printf 'doctor\\n' >>"$HOME/events" touch "$HOME/migrated" @@ -59,7 +81,8 @@ trap 'case "$BASH_COMMAND" in "phase "*) install_fixture_phases ;; esac' DEBUG OPENCLAW_CONFIG_PATH: join(state, "openclaw.json"), OPENCLAW_UPGRADE_SURVIVOR_RUNTIME_ROOT: join(home, "runtime"), OPENCLAW_UPGRADE_SURVIVOR_SUMMARY_JSON: summary, - OPENCLAW_UPGRADE_SURVIVOR_BASELINE: "openclaw@2026.7.1-2", + OPENCLAW_UPGRADE_SURVIVOR_BASELINE: + scenario === "workshop-doctor-recovery" ? "openclaw@2026.9.4" : "openclaw@2026.7.1-2", OPENCLAW_UPGRADE_SURVIVOR_SCENARIO: scenario, BASH_ENV: prelude, FIXTURE_AUTOMATIC: automatic ? "1" : "0", @@ -94,4 +117,251 @@ describe.skipIf(process.platform === "win32")("survivor first-hop observation", expect(events).toEqual(["update", "observe", "doctor", "observe", "consent"]); expect(summary.status).toBe("passed"); }); + + it.each([false, true])( + "keeps published refusal before explicit Workshop recovery (unexpected success=%s)", + (unexpectedSuccess) => { + const { result, events, summary } = runFirstHop( + "workshop-doctor-recovery", + unexpectedSuccess, + ); + expect(result.status, result.stderr).toBe(unexpectedSuccess ? 42 : 0); + expect(events).toEqual([ + "baseline-doctor", + "candidate-identity", + "seed", + "refusal", + ...(unexpectedSuccess + ? [] + : ["explicit-doctor", "verify", "update", "verify", "seed", "explicit-doctor", "verify"]), + ]); + expect(summary.status).toBe(unexpectedSuccess ? "failed" : "passed"); + expect(summary.updateRecovery).toBeNull(); + expect(summary.firstHopPostCore.availability).toBe("unavailable"); + if (unexpectedSuccess) { + expect(summary.failure.phase).toBe("assert-workshop-published-refusal"); + } + }, + ); +}); + +const workshopIndex = "idx_skill_workshop_collection_reviews_workspace_time"; + +function workshopFixture() { + const root = tempDirs.make("survivor-workshop-evidence-"); + const state = join(root, "state"); + const artifacts = join(root, "artifacts"); + const packageRoot = join(root, "installed"); + const candidateRoot = join(root, "package"); + for (const directory of [ + join(state, "state"), + artifacts, + join(packageRoot, "dist"), + join(candidateRoot, "dist"), + ]) { + mkdirSync(directory, { recursive: true }); + } + for (const directory of [packageRoot, candidateRoot]) { + writeFileSync( + join(directory, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.9.4" }), + ); + writeFileSync( + join(directory, "dist", "build-info.json"), + JSON.stringify({ buildId: directory === packageRoot ? "baseline" : "candidate" }), + ); + } + const baseline = captureWorkshopBaseline(packageRoot, artifacts); + const tarball = join(root, "candidate.tgz"); + execFileSync("tar", ["-czf", tarball, "package"], { cwd: root }); + const candidate = captureWorkshopCandidate(tarball, artifacts, "2026.9.4"); + const filename = join(state, "state", "openclaw.sqlite"); + const initial = new DatabaseSync(filename); + initial.exec(`CREATE TABLE skill_workshop_collection_reviews ( + review_id TEXT PRIMARY KEY, owner_agent_id TEXT, backup_id TEXT, create_time INTEGER, + kept_names_json TEXT, written_names_json TEXT, dropped_json TEXT + ); CREATE TABLE unrelated_records (value TEXT); INSERT INTO unrelated_records VALUES ('preserved');`); + initial.close(); + const seeded = seedWorkshopIndex(state, artifacts, "baseline"); + writeFileSync( + join(artifacts, "update.json"), + JSON.stringify({ + ok: false, + error: { + type: "cli_error", + message: `SQLite integrity_check failed: Page ${seeded.rootpage}: never used`, + }, + }), + ); + return { state, artifacts, packageRoot, candidateRoot, baseline, candidate, filename }; +} + +type WorkshopIdentity = { version: string; buildInfoSha256: string }; +function recordProcess( + observations: string, + pid: number, + role: "update" | "doctor", + identity: WorkshopIdentity, + malformedAtStart: boolean, + updateInProgress: boolean, + exitCode: number, + nativeReceipt = true, +) { + mkdirSync(join(observations, "diagnostics"), { recursive: true }); + const witness = { + role, + identity, + malformedAtStart, + updateInProgress, + exitCode, + pid, + parentPid: 100, + }; + writeFileSync(join(observations, `workshop-process-${pid}.json`), JSON.stringify(witness)); + if (nativeReceipt) { + writeFileSync( + join(observations, "diagnostics", `process-${pid}-exited.json`), + JSON.stringify({ + role, + packageVersion: identity.version, + pid, + parentPid: witness.parentPid, + exitCode, + }), + ); + } +} + +// Simulate command outcomes to test evidence rejection; real repair belongs to the Docker proof. +function simulateWorkshopRepair(filename: string) { + const database = new DatabaseSync(filename); + database.enableDefensive?.(false); + database.exec("PRAGMA writable_schema = ON;"); + database + .prepare("UPDATE sqlite_schema SET sql = ? WHERE name = ?") + .run( + `CREATE INDEX ${workshopIndex} ON skill_workshop_collection_reviews(review_id, create_time DESC)`, + workshopIndex, + ); + const version = database.prepare("PRAGMA schema_version").get()?.schema_version; + database.exec( + `PRAGMA writable_schema = OFF; PRAGMA schema_version = ${Number(version) + 1}; DROP INDEX ${workshopIndex};`, + ); + database.close(); +} + +describe("Workshop Doctor recovery evidence", () => { + it.each(["refused", "unexpected-success", "unrelated-data-changed", "installed-build-changed"])( + "validates the published updater's %s outcome without accepting repair", + (outcome) => { + const fixture = workshopFixture(); + const observations = join(fixture.artifacts, "first-update"); + recordProcess( + observations, + 101, + "update", + fixture.baseline, + true, + false, + outcome === "unexpected-success" ? 0 : 1, + ); + if (outcome === "unrelated-data-changed") { + const database = new DatabaseSync(fixture.filename); + database.enableDefensive?.(false); + database.exec( + "PRAGMA writable_schema = ON; UPDATE unrelated_records SET value = 'changed';", + ); + database.close(); + } + if (outcome === "installed-build-changed") { + cpSync(fixture.candidateRoot, fixture.packageRoot, { recursive: true }); + } + const verify = () => + assertWorkshopUpdateRefusal( + fixture.state, + fixture.artifacts, + observations, + fixture.packageRoot, + outcome === "unexpected-success" ? 0 : 1, + ); + if (outcome === "refused") { + expect(verify()).toMatchObject({ + status: "refused-before-candidate", + automaticRepair: false, + }); + } else { + expect(verify).toThrow( + outcome === "unexpected-success" + ? /must refuse/ + : outcome === "unrelated-data-changed" + ? /changed state/ + : /changed installed build/, + ); + } + }, + ); + + it.each([ + "intact", + "review-changed", + "already-repaired", + "missing-receipt", + "same-version-wrong-build", + "update-marker", + ])("requires explicit candidate Doctor evidence: %s", (outcome) => { + const fixture = workshopFixture(); + simulateWorkshopRepair(fixture.filename); + if (outcome === "review-changed") { + const database = new DatabaseSync(fixture.filename); + database.exec("UPDATE skill_workshop_collection_reviews SET backup_id = 'wrong-backup';"); + database.close(); + } + const observations = join(fixture.artifacts, "candidate-doctor"); + recordProcess( + observations, + 102, + "doctor", + outcome === "same-version-wrong-build" ? fixture.baseline : fixture.candidate, + outcome !== "already-repaired", + outcome === "update-marker", + 0, + outcome !== "missing-receipt", + ); + const verify = () => + assertWorkshopDoctorRepair(fixture.state, fixture.artifacts, observations, "candidate"); + if (outcome === "intact") { + expect(verify()).toMatchObject({ status: "explicit-doctor-repaired" }); + } else { + expect(verify).toThrow( + outcome === "review-changed" ? /retained Workshop review/ : /Missing matching doctor/, + ); + } + }); + + it("keeps the refused first attempt distinct from both repairs and the recovered upgrade", () => { + const fixture = workshopFixture(); + const first = join(fixture.artifacts, "first-update"); + recordProcess(first, 101, "update", fixture.baseline, true, false, 1); + assertWorkshopUpdateRefusal(fixture.state, fixture.artifacts, first, fixture.packageRoot, 1); + simulateWorkshopRepair(fixture.filename); + const baselineDoctor = join(fixture.artifacts, "baseline-doctor"); + recordProcess(baselineDoctor, 102, "doctor", fixture.baseline, true, false, 0); + assertWorkshopDoctorRepair(fixture.state, fixture.artifacts, baselineDoctor, "baseline"); + cpSync(fixture.candidateRoot, fixture.packageRoot, { recursive: true }); + const upgraded = join(fixture.artifacts, "recovered-upgrade"); + recordProcess(upgraded, 103, "update", fixture.baseline, false, false, 0); + recordProcess(upgraded, 104, "doctor", fixture.candidate, false, true, 0); + assertWorkshopRecoveredUpgrade(fixture.state, fixture.artifacts, upgraded, fixture.packageRoot); + seedWorkshopIndex(fixture.state, fixture.artifacts, "candidate"); + const candidateDoctor = join(fixture.artifacts, "candidate-doctor"); + simulateWorkshopRepair(fixture.filename); + recordProcess(candidateDoctor, 105, "doctor", fixture.candidate, true, false, 0); + assertWorkshopDoctorRepair(fixture.state, fixture.artifacts, candidateDoctor, "candidate"); + expect(completeWorkshopRecovery(fixture.state, fixture.artifacts)).toMatchObject({ + firstAttempt: { status: "refused-before-candidate", automaticRepair: false }, + baselineDoctor: { status: "explicit-doctor-repaired" }, + upgrade: { status: "upgraded-after-explicit-repair" }, + candidateDoctor: { status: "explicit-doctor-repaired" }, + }); + }); }); diff --git a/test/scripts/upgrade-survivor-plugin-registry.test.ts b/test/scripts/upgrade-survivor-plugin-registry.test.ts index 1d694e8f1aa8..64384f9d281b 100644 --- a/test/scripts/upgrade-survivor-plugin-registry.test.ts +++ b/test/scripts/upgrade-survivor-plugin-registry.test.ts @@ -241,6 +241,24 @@ describe("standalone upgrade survivor plugin registry", () => { ); }); + it.each([ + ["custom-plugin-siblings", "openclaw@2026.9.4"], + ["abandoned-update", "openclaw@2026.9.2"], + ["workshop-doctor-recovery", "openclaw@2026.9.4"], + ])("follows the planner's no-registry decision for %s", (scenario, baseline) => { + const { captureDir, result } = runSurvivor({ + OPENCLAW_UPGRADE_SURVIVOR_SCENARIO: scenario, + OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC: baseline, + }); + + expect(result.status, result.stderr).toBe(0); + expect(existsSync(join(captureDir, "node-args"))).toBe(false); + expect(existsSync(join(captureDir, "docker-run-args"))).toBe(true); + expect(readFileSync(join(captureDir, "docker-args"), "utf8")).not.toContain( + "/tmp/openclaw-prepublish-plugin-registry", + ); + }); + it("does not prepare a registry for a published candidate", () => { const { captureDir, packageTarball, result } = runSurvivor({ OPENCLAW_CURRENT_PACKAGE_TGZ: undefined, diff --git a/test/vitest/vitest.database-worker-core-paths.mjs b/test/vitest/vitest.database-worker-core-paths.mjs index 642ec882068f..df24943cfa23 100644 --- a/test/vitest/vitest.database-worker-core-paths.mjs +++ b/test/vitest/vitest.database-worker-core-paths.mjs @@ -59,6 +59,7 @@ export const databaseWorkerCoreTestFiles = [ "src/agents/sessions/sdk.auth-migration.test.ts", "src/agents/subagents/completion/subagent-completion-admission.store.test.ts", "src/commands/doctor-maintenance.worker.test.ts", + "src/flows/doctor-health.dangling-workshop-index.test.ts", "src/entry.memory-json.test.ts", "src/gateway/server-methods/memory-search.test.ts", "src/logging/diagnostic-session-context.test.ts",