mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 09:39:25 +00:00
fix(e2e): verify worktree repair stays in Doctor (#150365)
* fix(e2e): prove Doctor-owned worktree repair Keep the published project-worktree upgrade scenario aligned with Doctor-only repair after #150110. Verify updater Doctor on one specimen and startup preservation, explicit Doctor repair, and stable restart on an independent published-owner specimen. * fix(e2e): prepare fixture schemas through Doctor Upgrade the independent published specimen through the existing Doctor migration owner before Gateway startup, preserving legacy workspace metadata and exact session and transcript bytes for the subsequent workspace repair proof. * fix(e2e): resolve defining Doctor package owners Distinguish defining function chunks from generated forwarding entries with the installed package parser while retaining the existing general export resolver behavior and package hash checks.
This commit is contained in:
Peter Steinberger
GitHub
parent
6955a376f4
commit
f4e9d06bd3
5 changed files with 532 additions and 82 deletions
|
|
@ -196,14 +196,19 @@ strings are equal. Select one with `OPENCLAW_UPGRADE_SURVIVOR_SCENARIO` and set
|
|||
`projects-doctor` preserves one registered project and one configured workspace,
|
||||
then runs the real `doctor --lint --only core/doctor/project-clone-shape --json`
|
||||
twice. It checks stored rows, schema, sentinels, and read-only snapshot cleanup.
|
||||
`projects-startup-migration` creates a project and managed worktree through the
|
||||
published owners using local Git, then imports synthetic legacy session JSON/JSONL
|
||||
through published Doctor. It requires the updater's candidate Doctor repair to fill
|
||||
in the registered project's canonical workspace before the first Gateway startup.
|
||||
Both normal Gateway starts must leave the repaired session and transcript unchanged,
|
||||
perform no workspace backfill, become ready, and report clean shutdown before
|
||||
persisted readback. The fixture is a supported legacy-format import, not a
|
||||
historical runtime-generated session. No additional Doctor recovery pass runs. Set
|
||||
`projects-startup-migration` prepares two independent project/worktree specimens
|
||||
through the published owners using local Git. Each has a verified backup and
|
||||
synthetic legacy session JSON/JSONL imported through published Doctor. The update
|
||||
must repair the first specimen's canonical workspace through candidate Doctor.
|
||||
The second state stays outside that update's discovery. Before startup, the
|
||||
candidate's Doctor schema owner runs under its maintenance lock to upgrade that
|
||||
database while preserving the legacy workspace fields and exact session/transcript
|
||||
bytes. Its first normal Gateway startup must preserve that state. After
|
||||
clean shutdown, an explicit `doctor --fix --non-interactive` repairs its canonical
|
||||
workspace; a second startup must leave the repaired state unchanged. These are
|
||||
supported legacy-format imports, not historical runtime-generated sessions.
|
||||
Both Gateway runs must become ready and report clean shutdown before persisted
|
||||
readback. Set
|
||||
`OPENCLAW_UPGRADE_SURVIVOR_STARTUP_BINDINGS` to a reviewed JSON file containing the
|
||||
candidate `commit`, `agentSchema`, and `operations.prepare`/`operations.open`
|
||||
triples of compiled basename, exact export symbol, and SHA-256. The snapshot
|
||||
|
|
|
|||
|
|
@ -3,19 +3,27 @@ import assert from "node:assert/strict";
|
|||
import { execFileSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { resolveWorkerCellExport } from "./worker-cell-package.mjs";
|
||||
import {
|
||||
resolveWorkerCellExport,
|
||||
resolveWorkerCellFunctionBinding,
|
||||
} from "./worker-cell-package.mjs";
|
||||
|
||||
const BASELINE = "3a9d69db306cd7f081e06254cb89c4bcc14a7107";
|
||||
const BASELINE_AGENT_SCHEMA = 19;
|
||||
const KEY = "agent:main:dashboard:legacy-project-worktree";
|
||||
const OTHER_KEY = "agent:main:dashboard:legacy-project-sentinel";
|
||||
const SESSION = "00000000-0000-4000-8000-000000000001";
|
||||
const OTHER_SESSION = "00000000-0000-4000-8000-000000000002";
|
||||
const STAGES = new Set([
|
||||
"published-import",
|
||||
"after-update",
|
||||
"before-schema",
|
||||
"before-startup",
|
||||
"after-first-stop",
|
||||
"after-doctor",
|
||||
"after-second-stop",
|
||||
]);
|
||||
const BASELINE_BINDINGS = {
|
||||
|
|
@ -138,6 +146,7 @@ async function owners(ctx, packageRoot, baseline, bindingFile) {
|
|||
assert.equal(identity.buildInfo.commit, baseline ? BASELINE : candidateCommit);
|
||||
assert.equal(fs.realpathSync(path.join(packageRoot, "openclaw.mjs")), identity.cli);
|
||||
let bindings = BASELINE_BINDINGS;
|
||||
let agentSchema = BASELINE_AGENT_SCHEMA;
|
||||
if (!baseline) {
|
||||
const approved = readJson(bindingFile);
|
||||
assert.equal(approved.commit, candidateCommit);
|
||||
|
|
@ -152,7 +161,17 @@ async function owners(ctx, packageRoot, baseline, bindingFile) {
|
|||
approved.agentSchema,
|
||||
);
|
||||
bindings = approved.operations;
|
||||
agentSchema = approved.agentSchema;
|
||||
}
|
||||
return {
|
||||
...(await loadBindings(identity, packageRoot, bindings)),
|
||||
identity,
|
||||
baseline,
|
||||
agentSchema,
|
||||
};
|
||||
}
|
||||
|
||||
async function loadBindings(identity, packageRoot, bindings) {
|
||||
const api = {};
|
||||
const evidence = [];
|
||||
for (const [role, [name, symbol, expectedHash]] of Object.entries(bindings)) {
|
||||
|
|
@ -172,7 +191,87 @@ async function owners(ctx, packageRoot, baseline, bindingFile) {
|
|||
api[role] = module[alias];
|
||||
evidence.push({ role, relative, symbol, alias, sha256: expectedHash });
|
||||
}
|
||||
return { api, evidence, baseline };
|
||||
return { api, evidence };
|
||||
}
|
||||
|
||||
async function prepareSchema(ctx, packageRoot, bindings) {
|
||||
const owner = await owners(ctx, packageRoot, false, bindings);
|
||||
const before = readJson(path.join(ctx.artifacts, "worktree-before-schema.json"));
|
||||
assert.equal(before.agent.schema.userVersion, BASELINE_AGENT_SCHEMA);
|
||||
assert(
|
||||
owner.agentSchema > BASELINE_AGENT_SCHEMA,
|
||||
"Expected a published-to-candidate schema upgrade",
|
||||
);
|
||||
const require = createRequire(path.join(packageRoot, "package.json"));
|
||||
const parserPath = fs.realpathSync(require.resolve("typescript"));
|
||||
assert(childOf(fs.realpathSync(packageRoot), parserPath), "Use the installed package's parser");
|
||||
const ts = require(parserPath);
|
||||
assert.equal(
|
||||
ts.version,
|
||||
readJson(path.join(packageRoot, "package.json")).dependencies.typescript,
|
||||
);
|
||||
const doctorBindings = {};
|
||||
for (const [role, prefix, symbol] of [
|
||||
["lock", "doctor-sqlite-maintenance-lock", "withDoctorSqliteMaintenanceLock"],
|
||||
["migrate", "state-migrations.media-persistence", "migrateLegacyMediaPersistence"],
|
||||
["drain", "global-singleton", "drainGlobalSingletonLifecycleState"],
|
||||
["close", "openclaw-state-db-cache", "closeOpenClawStateDatabaseByPathAsync"],
|
||||
]) {
|
||||
doctorBindings[role] = resolveWorkerCellFunctionBinding(
|
||||
owner.identity,
|
||||
packageRoot,
|
||||
prefix,
|
||||
symbol,
|
||||
ts,
|
||||
);
|
||||
}
|
||||
const doctor = await loadBindings(owner.identity, packageRoot, doctorBindings);
|
||||
const { agentDb } = readJson(ctx.importReceipt);
|
||||
const errors = [];
|
||||
let result;
|
||||
try {
|
||||
result = await doctor.api.lock({
|
||||
env: process.env,
|
||||
operation: "project worktree fixture schema preparation",
|
||||
run: () =>
|
||||
doctor.api.migrate({
|
||||
env: process.env,
|
||||
configuredAgentDatabaseTargets: [{ agentId: "main", path: agentDb }],
|
||||
}),
|
||||
});
|
||||
} catch (error) {
|
||||
errors.push(error);
|
||||
}
|
||||
for (const [operation, argument] of [
|
||||
[doctor.api.drain, "close"],
|
||||
[doctor.api.close, ctx.stateDb],
|
||||
]) {
|
||||
try {
|
||||
await operation(argument);
|
||||
} catch (error) {
|
||||
errors.push(error);
|
||||
}
|
||||
}
|
||||
if (errors.length) {
|
||||
throw new AggregateError(errors, "Doctor schema preparation did not settle");
|
||||
}
|
||||
writeJson(path.join(ctx.artifacts, "worktree-schema-doctor.json"), {
|
||||
ownerBindings: doctor.evidence,
|
||||
parser: { version: ts.version, sha256: digest(parserPath) },
|
||||
fromSchema: before.agent.schema,
|
||||
targetSchema: owner.agentSchema,
|
||||
result,
|
||||
});
|
||||
assert.deepEqual(
|
||||
result,
|
||||
{
|
||||
changes: [
|
||||
`Upgraded agent database schema in ${agentDb}: v${BASELINE_AGENT_SCHEMA} -> v${owner.agentSchema}.`,
|
||||
],
|
||||
warnings: [],
|
||||
},
|
||||
"Doctor schema preparation warned, refused, or changed more than the schema",
|
||||
);
|
||||
}
|
||||
|
||||
async function inspectDatabase(owner, file, read) {
|
||||
|
|
@ -215,8 +314,9 @@ async function inspectDatabase(owner, file, read) {
|
|||
|
||||
async function seed(ctx, packageRoot) {
|
||||
assert(!fs.existsSync(ctx.fixture));
|
||||
const repo = path.join(ctx.root, "project-repo");
|
||||
const workspace = path.join(ctx.root, "agent-default");
|
||||
const fixtureRoot = path.dirname(ctx.stateDir);
|
||||
const repo = path.join(fixtureRoot, "project-repo");
|
||||
const workspace = path.join(fixtureRoot, "agent-default");
|
||||
fs.mkdirSync(path.join(repo, "packages/app"), { recursive: true });
|
||||
fs.mkdirSync(workspace);
|
||||
fs.writeFileSync(path.join(repo, "README.md"), "Published-owner project fixture\n", {
|
||||
|
|
@ -277,7 +377,7 @@ async function seed(ctx, packageRoot) {
|
|||
);
|
||||
const service = new owner.api.worktrees({
|
||||
env: gitEnv,
|
||||
getConfig: () => ({ worktreeRoot: path.join(ctx.root, "managed") }),
|
||||
getConfig: () => ({ worktreeRoot: path.join(fixtureRoot, "managed") }),
|
||||
});
|
||||
worktree = await service.create({
|
||||
repoRoot: project.repoRoot,
|
||||
|
|
@ -455,7 +555,7 @@ export function assertProjectWorktreeStartupLog(log, start) {
|
|||
/session: recorded canonical workspaces for (\d+) managed-worktree session\(s\)/g,
|
||||
),
|
||||
].map((match) => Number(match[1]));
|
||||
assert.deepEqual(backfills, [], "Gateway startup performed an unexpected workspace repair");
|
||||
assert.deepEqual(backfills, [], "Gateway startup performed a Doctor-owned workspace repair");
|
||||
assert.match(log, /(?:\[shutdown\]|shutdown) completed cleanly in \d+ms/);
|
||||
assert(
|
||||
!/(?:\[shutdown\]|shutdown) (?:completed in \d+ms with warnings:|failed in \d+ms)/.test(log),
|
||||
|
|
@ -463,29 +563,19 @@ export function assertProjectWorktreeStartupLog(log, start) {
|
|||
return { backfills, cleanShutdown: true };
|
||||
}
|
||||
|
||||
export function assertProjectWorktreeStartupPreservation(actual, original, projectRoot) {
|
||||
assert(STAGES.has(actual.stage));
|
||||
const expectedWorkspace = actual.stage === "published-import" ? undefined : projectRoot;
|
||||
const target = actual.agent.sessions.find((row) => row.session_key === KEY);
|
||||
assert.equal(
|
||||
JSON.parse(target.entry_json).worktree.canonicalWorkspaceDir,
|
||||
expectedWorkspace,
|
||||
"Update Doctor must repair the imported workspace before Gateway startup",
|
||||
);
|
||||
export function assertProjectWorktreeStartupPreservation(actual, original, expectedWorkspace) {
|
||||
assert.deepEqual(actual.shared, original.shared);
|
||||
assert.deepEqual(actual.agent.transcript, original.agent.transcript);
|
||||
assert.equal(actual.agent.sessions.length, original.agent.sessions.length);
|
||||
for (const row of actual.agent.sessions) {
|
||||
const before = original.agent.sessions.find((s) => s.session_key === row.session_key);
|
||||
assert(before, `Unexpected session row: ${row.session_key}`);
|
||||
if (row.session_key !== KEY) {
|
||||
if (row.session_key !== KEY || expectedWorkspace === undefined) {
|
||||
assert.deepEqual(row, before);
|
||||
continue;
|
||||
}
|
||||
const expected = JSON.parse(before.entry_json);
|
||||
if (expectedWorkspace) {
|
||||
expected.worktree.canonicalWorkspaceDir = expectedWorkspace;
|
||||
}
|
||||
expected.worktree.canonicalWorkspaceDir = expectedWorkspace;
|
||||
assert.deepEqual(JSON.parse(row.entry_json), expected);
|
||||
assert.equal(row.updated_at, before.updated_at);
|
||||
assert.equal(row.current_session_id, before.current_session_id);
|
||||
|
|
@ -503,6 +593,14 @@ async function snapshot(ctx, stage, packageRoot, bindings) {
|
|||
worktrees: rows(db.prepare("SELECT * FROM worktrees ORDER BY id")),
|
||||
}));
|
||||
const agent = await inspectDatabase(owner, imported.agentDb, (db) => ({
|
||||
schema: {
|
||||
userVersion: db.prepare("PRAGMA user_version").get().user_version,
|
||||
metadataVersion: db
|
||||
.prepare("SELECT schema_version FROM schema_meta WHERE meta_key = 'primary'")
|
||||
.get().schema_version,
|
||||
agentId: db.prepare("SELECT agent_id FROM schema_meta WHERE meta_key = 'primary'").get()
|
||||
.agent_id,
|
||||
},
|
||||
sessions: rows(
|
||||
db.prepare(
|
||||
"SELECT session_key,current_session_id,entry_json,updated_at FROM session_nodes WHERE session_key IN ('agent:main:dashboard:legacy-project-worktree','agent:main:dashboard:legacy-project-sentinel') ORDER BY session_key",
|
||||
|
|
@ -514,6 +612,14 @@ async function snapshot(ctx, stage, packageRoot, bindings) {
|
|||
),
|
||||
),
|
||||
}));
|
||||
const expectedSchema = ["published-import", "before-schema"].includes(stage)
|
||||
? BASELINE_AGENT_SCHEMA
|
||||
: owner.agentSchema;
|
||||
assert.deepEqual(agent.schema, {
|
||||
userVersion: expectedSchema,
|
||||
metadataVersion: expectedSchema,
|
||||
agentId: "main",
|
||||
});
|
||||
assert.equal(agent.sessions.length, 2);
|
||||
assert.equal(agent.transcript.length, 4);
|
||||
const row = agent.sessions.find((s) => s.session_key === KEY);
|
||||
|
|
@ -526,6 +632,14 @@ async function snapshot(ctx, stage, packageRoot, bindings) {
|
|||
assert.equal(entry.worktree.repoRoot, f.project.repoRoot);
|
||||
assert.equal(entry.updatedAt, 10);
|
||||
assert.equal(entry.lastActivityAt, 10);
|
||||
const expectedWorkspace = ["after-update", "after-doctor", "after-second-stop"].includes(stage)
|
||||
? f.project.repoRoot
|
||||
: undefined;
|
||||
assert.equal(
|
||||
entry.worktree.canonicalWorkspaceDir,
|
||||
expectedWorkspace,
|
||||
"Unexpected migration stage; do not delete metadata to recreate a legacy specimen",
|
||||
);
|
||||
for (const [file, expected] of Object.entries(f.sentinelHashes)) {
|
||||
assert.equal(digest(file), expected);
|
||||
}
|
||||
|
|
@ -537,20 +651,18 @@ async function snapshot(ctx, stage, packageRoot, bindings) {
|
|||
ownerBindings: owner.evidence,
|
||||
sentinelHashes: f.sentinelHashes,
|
||||
};
|
||||
const original =
|
||||
stage === "published-import"
|
||||
? result
|
||||
: readJson(path.join(ctx.artifacts, "worktree-published-import.json"));
|
||||
assertProjectWorktreeStartupPreservation(result, original, f.project.repoRoot);
|
||||
if (stage.startsWith("after-")) {
|
||||
const before = readJson(path.join(ctx.artifacts, "worktree-before-startup.json"));
|
||||
assert.deepEqual(agent, before.agent, "Startup changed persisted session/history bytes");
|
||||
assert.deepEqual(shared, before.shared);
|
||||
if (stage !== "published-import") {
|
||||
const original = readJson(path.join(ctx.artifacts, "worktree-published-import.json"));
|
||||
assertProjectWorktreeStartupPreservation(result, original, expectedWorkspace);
|
||||
}
|
||||
if (stage === "after-second-stop") {
|
||||
const first = readJson(path.join(ctx.artifacts, "worktree-after-first-stop.json"));
|
||||
assert.deepEqual(agent, first.agent, "Second startup changed persisted session/history bytes");
|
||||
assert.deepEqual(shared, first.shared);
|
||||
const repaired = readJson(path.join(ctx.artifacts, "worktree-after-doctor.json"));
|
||||
assert.deepEqual(
|
||||
agent,
|
||||
repaired.agent,
|
||||
"Second startup changed repaired session/history bytes",
|
||||
);
|
||||
assert.deepEqual(shared, repaired.shared);
|
||||
}
|
||||
writeJson(path.join(ctx.artifacts, `worktree-${stage}.json`), result);
|
||||
}
|
||||
|
|
@ -567,6 +679,9 @@ async function main() {
|
|||
} else if (mode === "snapshot") {
|
||||
assert.equal(args.length, 3);
|
||||
await snapshot(ctx, ...args);
|
||||
} else if (mode === "prepare-schema") {
|
||||
assert.equal(args.length, 2);
|
||||
await prepareSchema(ctx, ...args);
|
||||
} else if (mode === "assert-logs") {
|
||||
assert.equal(args.length, 2);
|
||||
const [start, file] = args;
|
||||
|
|
@ -578,7 +693,7 @@ async function main() {
|
|||
});
|
||||
} else {
|
||||
throw new Error(
|
||||
"Expected seed, assert-import, snapshot, or assert-logs; see reviewed recipe for arguments",
|
||||
"Expected seed, assert-import, snapshot, prepare-schema, or assert-logs; see reviewed recipe for arguments",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2021,6 +2021,39 @@ backup_project_worktree_fixture() {
|
|||
>"$ARTIFACT_ROOT/worktree-backup.json" 2>"$ARTIFACT_ROOT/worktree-backup.err"
|
||||
}
|
||||
|
||||
prepare_project_worktree_startup_fixture() (
|
||||
# The parent phase owns failure and cleanup; this child only prepares published state.
|
||||
trap - ERR EXIT HUP INT TERM
|
||||
local published_identity="$ARTIFACT_ROOT/baseline-package-identity.json"
|
||||
ARTIFACT_ROOT="$ARTIFACT_ROOT/worktree-startup"
|
||||
export OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_ROOT="$ARTIFACT_ROOT"
|
||||
mkdir "$ARTIFACT_ROOT"
|
||||
cp "$published_identity" "$ARTIFACT_ROOT/baseline-package-identity.json"
|
||||
openclaw_test_state_create "$RUNTIME_ROOT/worktree-startup-state" minimal
|
||||
node scripts/e2e/lib/upgrade-survivor/project-worktree-startup.mjs seed "$(package_root)"
|
||||
backup_project_worktree_fixture
|
||||
run_project_worktree_import dry-run
|
||||
run_project_worktree_import import
|
||||
node scripts/e2e/lib/upgrade-survivor/project-worktree-startup.mjs assert-import "$ARTIFACT_ROOT/worktree-import.json"
|
||||
node scripts/e2e/lib/upgrade-survivor/project-worktree-startup.mjs snapshot published-import "$(package_root)" -
|
||||
openclaw_e2e_write_state_env "$ARTIFACT_ROOT/state-env"
|
||||
)
|
||||
|
||||
run_project_worktree_startup_fixture() {
|
||||
OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_ROOT="$ARTIFACT_ROOT/worktree-startup" \
|
||||
node scripts/e2e/lib/upgrade-survivor/project-worktree-startup.mjs "$@"
|
||||
}
|
||||
|
||||
run_project_worktree_doctor() {
|
||||
openclaw_e2e_maybe_timeout "$COMMAND_TIMEOUT" env \
|
||||
-u OPENCLAW_UPDATE_IN_PROGRESS \
|
||||
-u OPENCLAW_UPDATE_POST_CORE_CONVERGENCE \
|
||||
-u OPENCLAW_UPDATE_PARENT_SUPPORTS_DOCTOR_CONFIG_WRITE \
|
||||
-u OPENCLAW_UPDATE_DEFER_CONFIGURED_PLUGIN_INSTALL_REPAIR \
|
||||
openclaw doctor --fix --non-interactive \
|
||||
>"$ARTIFACT_ROOT/worktree-doctor.log" 2>&1
|
||||
}
|
||||
|
||||
validate_worker_cell() {
|
||||
if [ "$WORKER_CELL" != "1" ]; then
|
||||
return 0
|
||||
|
|
@ -2049,6 +2082,7 @@ if [ "$WORKER_CELL" = "1" ]; then
|
|||
phase import-project-worktree run_project_worktree_import import
|
||||
phase assert-project-worktree-import node scripts/e2e/lib/upgrade-survivor/project-worktree-startup.mjs assert-import "$ARTIFACT_ROOT/worktree-import.json"
|
||||
phase snapshot-published-worktree node scripts/e2e/lib/upgrade-survivor/project-worktree-startup.mjs snapshot published-import "$(package_root)" -
|
||||
phase prepare-independent-worktree-startup prepare_project_worktree_startup_fixture
|
||||
else
|
||||
phase seed-taskflow node scripts/e2e/lib/upgrade-survivor/taskflow-restoration.mjs seed --package-root "$(package_root)"
|
||||
fi
|
||||
|
|
@ -2071,7 +2105,16 @@ if [ "$WORKER_CELL" = "1" ]; then
|
|||
assert-doctor "$ARTIFACT_ROOT/projects-doctor-repeat.json" before-repeat after-repeat
|
||||
phase assert-projects-preservation node scripts/e2e/lib/upgrade-survivor/projects-doctor.mjs assert-final
|
||||
elif [ "$SCENARIO" = "projects-startup-migration" ]; then
|
||||
phase snapshot-before-worktree-startup node scripts/e2e/lib/upgrade-survivor/project-worktree-startup.mjs \
|
||||
phase assert-update-doctor-worktree-repair node scripts/e2e/lib/upgrade-survivor/project-worktree-startup.mjs \
|
||||
snapshot after-update "$(package_root)" "$OPENCLAW_UPGRADE_SURVIVOR_STARTUP_BINDINGS"
|
||||
cp "$ARTIFACT_ROOT/installed-package-identity.json" "$ARTIFACT_ROOT/worktree-startup/installed-package-identity.json"
|
||||
source "$ARTIFACT_ROOT/worktree-startup/state-env"
|
||||
export USERPROFILE="$HOME"
|
||||
phase snapshot-before-worktree-schema run_project_worktree_startup_fixture \
|
||||
snapshot before-schema "$(package_root)" "$OPENCLAW_UPGRADE_SURVIVOR_STARTUP_BINDINGS"
|
||||
phase prepare-worktree-schema run_project_worktree_startup_fixture \
|
||||
prepare-schema "$(package_root)" "$OPENCLAW_UPGRADE_SURVIVOR_STARTUP_BINDINGS"
|
||||
phase snapshot-before-worktree-startup run_project_worktree_startup_fixture \
|
||||
snapshot before-startup "$(package_root)" "$OPENCLAW_UPGRADE_SURVIVOR_STARTUP_BINDINGS"
|
||||
for startup in first second; do
|
||||
GATEWAY_LOG="$ARTIFACT_ROOT/worktree-$startup-gateway.log"
|
||||
|
|
@ -2080,10 +2123,15 @@ if [ "$WORKER_CELL" = "1" ]; then
|
|||
phase "$startup-worktree-gateway-start" start_gateway
|
||||
phase "$startup-worktree-gateway-probes" check_gateway_probes
|
||||
phase "$startup-worktree-gateway-stop" stop_gateway
|
||||
phase "assert-$startup-worktree-startup-log" node scripts/e2e/lib/upgrade-survivor/project-worktree-startup.mjs \
|
||||
phase "assert-$startup-worktree-startup-log" run_project_worktree_startup_fixture \
|
||||
assert-logs "$startup" "$GATEWAY_LOG"
|
||||
phase "snapshot-$startup-worktree-stop" node scripts/e2e/lib/upgrade-survivor/project-worktree-startup.mjs \
|
||||
phase "snapshot-$startup-worktree-stop" run_project_worktree_startup_fixture \
|
||||
snapshot "after-$startup-stop" "$(package_root)" "$OPENCLAW_UPGRADE_SURVIVOR_STARTUP_BINDINGS"
|
||||
if [ "$startup" = first ]; then
|
||||
phase repair-project-worktree run_project_worktree_doctor
|
||||
phase snapshot-after-worktree-doctor run_project_worktree_startup_fixture \
|
||||
snapshot after-doctor "$(package_root)" "$OPENCLAW_UPGRADE_SURVIVOR_STARTUP_BINDINGS"
|
||||
fi
|
||||
done
|
||||
else
|
||||
phase gateway-start start_gateway
|
||||
|
|
|
|||
|
|
@ -74,6 +74,49 @@ export function resolveWorkerCellExport(source, name) {
|
|||
return matches[0];
|
||||
}
|
||||
|
||||
/** Generated forwarding entries can share the defining owner's filename prefix. */
|
||||
export function resolveWorkerCellFunctionBinding(identity, packageRoot, prefix, symbol, ts) {
|
||||
const root = fs.realpathSync(packageRoot);
|
||||
const matches = [];
|
||||
for (const relative of Object.keys(identity.files)) {
|
||||
const name = path.posix.basename(relative);
|
||||
if (
|
||||
path.posix.dirname(relative) !== "dist" ||
|
||||
!name.startsWith(`${prefix}-`) ||
|
||||
!name.endsWith(".mjs")
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const file = path.join(root, relative);
|
||||
assert.equal(fs.realpathSync(file), file, `Owner must be a regular package path: ${relative}`);
|
||||
assert(fs.lstatSync(file).isFile());
|
||||
const bytes = fs.readFileSync(file);
|
||||
const expectedHash = identity.files[relative].sha256;
|
||||
assert.equal(hash(bytes), expectedHash, `Package owner changed: ${relative}`);
|
||||
const source = bytes.toString("utf8");
|
||||
const ast = ts.createSourceFile(
|
||||
relative,
|
||||
source,
|
||||
ts.ScriptTarget.Latest,
|
||||
true,
|
||||
ts.ScriptKind.JS,
|
||||
);
|
||||
assert.equal(ast.parseDiagnostics.length, 0, `Cannot parse package owner: ${relative}`);
|
||||
const definitions = ast.statements.filter(
|
||||
(entry) => ts.isFunctionDeclaration(entry) && entry.name?.text === symbol && entry.body,
|
||||
);
|
||||
if (definitions.length === 0) {
|
||||
continue;
|
||||
}
|
||||
assert.equal(definitions.length, 1, `Ambiguous local definition: ${symbol}`);
|
||||
if (resolveWorkerCellExport(source, symbol)) {
|
||||
matches.push([name, symbol, expectedHash]);
|
||||
}
|
||||
}
|
||||
assert.equal(matches.length, 1, `Expected one installed defining ${prefix} owner`);
|
||||
return matches[0];
|
||||
}
|
||||
|
||||
function inspectTarball(tarball, runtimeRoot) {
|
||||
const bytes = fs.readFileSync(tarball);
|
||||
const sha256 = hash(bytes);
|
||||
|
|
|
|||
|
|
@ -1,13 +1,25 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import ts from "typescript";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
assertProjectWorktreeImportReport,
|
||||
assertProjectWorktreeStartupLog,
|
||||
assertProjectWorktreeStartupPreservation,
|
||||
} from "../../scripts/e2e/lib/upgrade-survivor/project-worktree-startup.mjs";
|
||||
import {
|
||||
readWorkerCellPackageIdentity,
|
||||
resolveWorkerCellExport,
|
||||
resolveWorkerCellFunctionBinding,
|
||||
} from "../../scripts/e2e/lib/upgrade-survivor/worker-cell-package.mjs";
|
||||
import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
const sessionKey = "agent:main:dashboard:legacy-project-worktree";
|
||||
const original = {
|
||||
stage: "published-import",
|
||||
shared: { project: [{ id: "project" }], worktrees: [{ id: "worktree" }] },
|
||||
agent: {
|
||||
sessions: [
|
||||
|
|
@ -28,8 +40,8 @@ const original = {
|
|||
transcript: [{ session_id: "target", seq: 1, event_json: "original bytes", created_at: 10 }],
|
||||
},
|
||||
};
|
||||
function migrated(stage = "before-startup") {
|
||||
const result = { ...structuredClone(original), stage };
|
||||
function migrated() {
|
||||
const result = structuredClone(original);
|
||||
const row = result.agent.sessions[0]!;
|
||||
const entry = JSON.parse(row.entry_json);
|
||||
entry.worktree.canonicalWorkspaceDir = "/fixture/project";
|
||||
|
|
@ -82,29 +94,253 @@ function publishedImportEvidence() {
|
|||
};
|
||||
}
|
||||
|
||||
describe("published project-worktree startup evidence", () => {
|
||||
it("requires update Doctor to repair the workspace before either Gateway startup", () => {
|
||||
const schemaSymbol = "migrateLegacyMediaPersistence";
|
||||
const schemaDefinition = `throw new Error("Fixture modules must not execute");
|
||||
async function ${schemaSymbol}() {}
|
||||
export { ${schemaSymbol} as t };
|
||||
`;
|
||||
const schemaForwarder = `import { t as ${schemaSymbol} } from "./doctor-owner-real.mjs";
|
||||
export { ${schemaSymbol} };
|
||||
`;
|
||||
function doctorOwnerFixture(files: Record<string, string>) {
|
||||
const root = tempDirs.make("openclaw-doctor-owner-binding-");
|
||||
mkdirSync(path.join(root, "dist"));
|
||||
writeFileSync(
|
||||
path.join(root, "package.json"),
|
||||
JSON.stringify({ name: "openclaw", version: "0.0.0-test" }),
|
||||
);
|
||||
writeFileSync(path.join(root, "openclaw.mjs"), "// synthetic package identity\n");
|
||||
writeFileSync(
|
||||
path.join(root, "dist/build-info.json"),
|
||||
JSON.stringify({ version: "0.0.0-test", commit: "1".repeat(40) }),
|
||||
);
|
||||
for (const [name, source] of Object.entries(files)) {
|
||||
writeFileSync(path.join(root, "dist", name), source);
|
||||
}
|
||||
return { root, identity: readWorkerCellPackageIdentity(root) };
|
||||
}
|
||||
|
||||
describe("published project-worktree Doctor ownership evidence", () => {
|
||||
it("selects the defining Doctor chunk while retaining valid forwarding exports", () => {
|
||||
const { root, identity } = doctorOwnerFixture({
|
||||
"doctor-owner-entry.mjs": schemaForwarder,
|
||||
"doctor-owner-real.mjs": schemaDefinition,
|
||||
});
|
||||
expect(resolveWorkerCellExport(schemaForwarder, schemaSymbol)).toBe(schemaSymbol);
|
||||
expect(
|
||||
resolveWorkerCellFunctionBinding(identity, root, "doctor-owner", schemaSymbol, ts),
|
||||
).toEqual([
|
||||
"doctor-owner-real.mjs",
|
||||
schemaSymbol,
|
||||
createHash("sha256").update(schemaDefinition).digest("hex"),
|
||||
]);
|
||||
});
|
||||
|
||||
it.each<{ name: string; files: Record<string, string>; error: RegExp }>([
|
||||
{
|
||||
name: "two defining owners",
|
||||
files: {
|
||||
"doctor-owner-real.mjs": schemaDefinition,
|
||||
"doctor-owner-other.mjs": schemaDefinition,
|
||||
},
|
||||
error: /one installed defining/,
|
||||
},
|
||||
{
|
||||
name: "forwarder without a definition",
|
||||
files: { "doctor-owner-entry.mjs": schemaForwarder },
|
||||
error: /one installed defining/,
|
||||
},
|
||||
{
|
||||
name: "malformed definition",
|
||||
files: { "doctor-owner-real.mjs": `function ${schemaSymbol}( {` },
|
||||
error: /Cannot parse package owner/,
|
||||
},
|
||||
])("rejects $name before importing Doctor code", ({ files, error }) => {
|
||||
const { root, identity } = doctorOwnerFixture(files);
|
||||
expect(() =>
|
||||
assertProjectWorktreeStartupPreservation(original, original, "/fixture/project"),
|
||||
resolveWorkerCellFunctionBinding(identity, root, "doctor-owner", schemaSymbol, ts),
|
||||
).toThrow(error);
|
||||
});
|
||||
|
||||
it("rejects changed candidate owner bytes", () => {
|
||||
const { root, identity } = doctorOwnerFixture({ "doctor-owner-real.mjs": schemaDefinition });
|
||||
writeFileSync(
|
||||
path.join(root, "dist/doctor-owner-real.mjs"),
|
||||
`${schemaDefinition}\n// changed\n`,
|
||||
);
|
||||
expect(() =>
|
||||
resolveWorkerCellFunctionBinding(identity, root, "doctor-owner", schemaSymbol, ts),
|
||||
).toThrow(/Package owner changed/);
|
||||
});
|
||||
|
||||
it("prepares the independent schema before startup and repairs workspace metadata between runs", () => {
|
||||
const root = tempDirs.make("openclaw-project-worktree-doctor-order-");
|
||||
const bin = path.join(root, "bin");
|
||||
const artifacts = path.join(root, "artifacts");
|
||||
const runtime = path.join(root, "runtime");
|
||||
const events = path.join(root, "events");
|
||||
mkdirSync(bin);
|
||||
mkdirSync(artifacts);
|
||||
mkdirSync(runtime);
|
||||
// This orchestration unit test replaces package operations, not the runner's phase sequence.
|
||||
writeFileSync(
|
||||
path.join(bin, "openclaw"),
|
||||
`#!/bin/bash
|
||||
set -eu
|
||||
if [ "$*" = 'doctor --fix --non-interactive' ]; then
|
||||
[ -z "\${OPENCLAW_UPDATE_IN_PROGRESS+x}" ]
|
||||
[ -z "\${OPENCLAW_UPDATE_POST_CORE_CONVERGENCE+x}" ]
|
||||
[ -z "\${OPENCLAW_UPDATE_PARENT_SUPPORTS_DOCTOR_CONFIG_WRITE+x}" ]
|
||||
[ -z "\${OPENCLAW_UPDATE_DEFER_CONFIGURED_PLUGIN_INSTALL_REPAIR+x}" ]
|
||||
printf repaired > "$OPENCLAW_STATE_DIR/workspace-state"
|
||||
printf 'doctor %s\\n' "$OPENCLAW_STATE_DIR" >> "$UNIT_EVENTS"
|
||||
else
|
||||
printf '{}\\n'
|
||||
fi
|
||||
`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
const source = readFileSync("scripts/e2e/lib/upgrade-survivor/run.sh", "utf8");
|
||||
const helpers = source.slice(
|
||||
source.indexOf("run_project_worktree_import()"),
|
||||
source.indexOf("validate_worker_cell()"),
|
||||
);
|
||||
const scenario = source.slice(
|
||||
source.indexOf('if [ "$WORKER_CELL" = "1" ]; then\n phase worker-baseline-identity'),
|
||||
source.indexOf('\nif [ "$SCENARIO" = "workshop-doctor-recovery" ]; then\n phase '),
|
||||
);
|
||||
const result = spawnSync(
|
||||
"/bin/bash",
|
||||
[
|
||||
"-c",
|
||||
`set -eu
|
||||
source scripts/lib/openclaw-e2e-instance.sh
|
||||
SCENARIO=projects-startup-migration
|
||||
WORKER_CELL=1
|
||||
COMMAND_TIMEOUT=5s
|
||||
ARTIFACT_ROOT="$1/artifacts"
|
||||
RUNTIME_ROOT="$1/runtime"
|
||||
export UNIT_EVENTS="$1/events"
|
||||
export PATH="$1/bin:$PATH"
|
||||
export OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_ROOT="$ARTIFACT_ROOT"
|
||||
export OPENCLAW_UPGRADE_SURVIVOR_STARTUP_BINDINGS=unit-bindings
|
||||
export OPENCLAW_UPDATE_IN_PROGRESS=unit-marker
|
||||
export OPENCLAW_UPDATE_POST_CORE_CONVERGENCE=unit-marker
|
||||
export OPENCLAW_UPDATE_PARENT_SUPPORTS_DOCTOR_CONFIG_WRITE=unit-marker
|
||||
export OPENCLAW_UPDATE_DEFER_CONFIGURED_PLUGIN_INSTALL_REPAIR=unit-marker
|
||||
baseline_spec=unit-baseline
|
||||
candidate_version=unit-candidate
|
||||
package_root() { printf '%s\\n' "$RUNTIME_ROOT/unit-package"; }
|
||||
openclaw_test_state_create() {
|
||||
export HOME="$1" OPENCLAW_HOME="$1" OPENCLAW_STATE_DIR="$1/.openclaw"
|
||||
export OPENCLAW_CONFIG_PATH="$OPENCLAW_STATE_DIR/openclaw.json"
|
||||
unset OPENCLAW_AGENT_DIR
|
||||
mkdir -p "$OPENCLAW_STATE_DIR"
|
||||
}
|
||||
openclaw_test_state_create "$RUNTIME_ROOT/state-home"
|
||||
node() {
|
||||
if [ "$1" = -e ]; then
|
||||
printf '%s\\n' "$OPENCLAW_STATE_DIR/sessions.json"
|
||||
elif [ "$1" = scripts/e2e/lib/upgrade-survivor/worker-cell-package.mjs ]; then
|
||||
printf '{}\\n' > "$ARTIFACT_ROOT/baseline-package-identity.json"
|
||||
elif [ "$1" = scripts/e2e/lib/upgrade-survivor/project-worktree-startup.mjs ]; then
|
||||
case "$2" in
|
||||
seed)
|
||||
printf legacy > "$OPENCLAW_STATE_DIR/workspace-state"
|
||||
printf 19 > "$OPENCLAW_STATE_DIR/schema-version"
|
||||
printf 'seed %s\\n' "$OPENCLAW_STATE_DIR" >> "$UNIT_EVENTS" ;;
|
||||
prepare-schema)
|
||||
[ "$(cat "$OPENCLAW_STATE_DIR/schema-version")" = 19 ]
|
||||
printf 21 > "$OPENCLAW_STATE_DIR/schema-version"
|
||||
printf 'schema-doctor %s\\n' "$OPENCLAW_STATE_DIR" >> "$UNIT_EVENTS" ;;
|
||||
snapshot)
|
||||
case "$3" in
|
||||
published-import|before-schema) [ "$(cat "$OPENCLAW_STATE_DIR/schema-version")" = 19 ] ;;
|
||||
*)
|
||||
if [ "$(cat "$OPENCLAW_STATE_DIR/schema-version")" != 21 ]; then
|
||||
printf 'Candidate schema migration required before Gateway startup\\n' >&2
|
||||
return 96
|
||||
fi ;;
|
||||
esac
|
||||
printf 'snapshot %s %s %s\\n' "$3" "$OPENCLAW_STATE_DIR" "$(cat "$OPENCLAW_STATE_DIR/workspace-state")" >> "$UNIT_EVENTS" ;;
|
||||
assert-import|assert-logs) : ;;
|
||||
*) return 97 ;;
|
||||
esac
|
||||
else
|
||||
return 97
|
||||
fi
|
||||
}
|
||||
update_candidate() {
|
||||
printf repaired > "$OPENCLAW_STATE_DIR/workspace-state"
|
||||
printf 21 > "$OPENCLAW_STATE_DIR/schema-version"
|
||||
printf '{}\\n' > "$ARTIFACT_ROOT/installed-package-identity.json"
|
||||
printf 'update %s\\n' "$OPENCLAW_STATE_DIR" >> "$UNIT_EVENTS"
|
||||
}
|
||||
start_gateway() {
|
||||
printf 'start %s %s\\n' "$OPENCLAW_STATE_DIR" "$(cat "$OPENCLAW_STATE_DIR/workspace-state")" >> "$UNIT_EVENTS"
|
||||
}
|
||||
stop_gateway() { printf 'stop %s\\n' "$OPENCLAW_STATE_DIR" >> "$UNIT_EVENTS"; }
|
||||
check_gateway_probes() { :; }
|
||||
phase() {
|
||||
shift
|
||||
case "$1" in
|
||||
node|prepare_project_worktree_startup_fixture|run_project_worktree_startup_fixture|run_project_worktree_doctor|backup_project_worktree_fixture|run_project_worktree_import|update_candidate|start_gateway|stop_gateway|check_gateway_probes) "$@" ;;
|
||||
*) : ;;
|
||||
esac
|
||||
}
|
||||
${helpers}
|
||||
${scenario}
|
||||
`,
|
||||
"project-worktree-doctor-order",
|
||||
root,
|
||||
],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
expect(result.status, result.stdout + result.stderr).toBe(0);
|
||||
const first = path.join(runtime, "state-home/.openclaw");
|
||||
const second = path.join(runtime, "worktree-startup-state/.openclaw");
|
||||
expect(readFileSync(events, "utf8").trim().split("\n")).toEqual([
|
||||
`seed ${first}`,
|
||||
`snapshot published-import ${first} legacy`,
|
||||
`seed ${second}`,
|
||||
`snapshot published-import ${second} legacy`,
|
||||
`update ${first}`,
|
||||
`snapshot after-update ${first} repaired`,
|
||||
`snapshot before-schema ${second} legacy`,
|
||||
`schema-doctor ${second}`,
|
||||
`snapshot before-startup ${second} legacy`,
|
||||
`start ${second} legacy`,
|
||||
`stop ${second}`,
|
||||
`snapshot after-first-stop ${second} legacy`,
|
||||
`doctor ${second}`,
|
||||
`snapshot after-doctor ${second} repaired`,
|
||||
`start ${second} repaired`,
|
||||
`stop ${second}`,
|
||||
`snapshot after-second-stop ${second} repaired`,
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves the imported shape until Doctor adds only the canonical workspace", () => {
|
||||
expect(() =>
|
||||
assertProjectWorktreeStartupPreservation(original, original, undefined),
|
||||
).not.toThrow();
|
||||
for (const stage of ["before-startup", "after-first-stop", "after-second-stop"]) {
|
||||
expect(() =>
|
||||
assertProjectWorktreeStartupPreservation(migrated(stage), original, "/fixture/project"),
|
||||
).not.toThrow();
|
||||
expect(() =>
|
||||
assertProjectWorktreeStartupPreservation(
|
||||
{ ...original, stage },
|
||||
original,
|
||||
"/fixture/project",
|
||||
),
|
||||
).toThrow();
|
||||
}
|
||||
const premature = migrated("published-import");
|
||||
expect(() =>
|
||||
assertProjectWorktreeStartupPreservation(premature, premature, "/fixture/project"),
|
||||
assertProjectWorktreeStartupPreservation(migrated(), original, "/fixture/project"),
|
||||
).not.toThrow();
|
||||
expect(() =>
|
||||
assertProjectWorktreeStartupPreservation(migrated(), original, undefined),
|
||||
).toThrow();
|
||||
expect(() =>
|
||||
assertProjectWorktreeStartupPreservation(migrated("unknown"), original, "/fixture/project"),
|
||||
assertProjectWorktreeStartupPreservation(original, original, "/fixture/project"),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("rejects startup rewriting the imported session JSON without changing its fields", () => {
|
||||
const rewritten = structuredClone(original);
|
||||
const row = rewritten.agent.sessions[0]!;
|
||||
row.entry_json = JSON.stringify(JSON.parse(row.entry_json), null, 2);
|
||||
expect(() =>
|
||||
assertProjectWorktreeStartupPreservation(rewritten, original, undefined),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
|
|
@ -159,28 +395,31 @@ describe("published project-worktree startup evidence", () => {
|
|||
shutdownPrefix: "2026-09-16T15:10:35.584+00:00 [shutdown]",
|
||||
},
|
||||
])(
|
||||
"requires clean shutdown without startup repair from $format logs",
|
||||
"requires clean shutdown without a runtime repair in $format logs",
|
||||
({ migration, shutdownPrefix }) => {
|
||||
const closed = `${shutdownPrefix} completed cleanly in 19ms`;
|
||||
const warned = `${shutdownPrefix} completed in 19ms with warnings: database drain`;
|
||||
const failed = `${shutdownPrefix} failed in 19ms`;
|
||||
for (const start of ["first", "second"]) {
|
||||
expect(assertProjectWorktreeStartupLog(closed, start)).toEqual({
|
||||
backfills: [],
|
||||
cleanShutdown: true,
|
||||
});
|
||||
for (const log of [
|
||||
"gateway ready",
|
||||
migration,
|
||||
`${migration}\n${closed}`,
|
||||
warned,
|
||||
failed,
|
||||
`${closed}\n${warned}`,
|
||||
`${closed}\n${failed}`,
|
||||
]) {
|
||||
expect(() => assertProjectWorktreeStartupLog(log, start)).toThrow();
|
||||
}
|
||||
expect(assertProjectWorktreeStartupLog(closed, "first")).toEqual({
|
||||
backfills: [],
|
||||
cleanShutdown: true,
|
||||
});
|
||||
expect(assertProjectWorktreeStartupLog(closed, "second")).toEqual({
|
||||
backfills: [],
|
||||
cleanShutdown: true,
|
||||
});
|
||||
for (const log of [
|
||||
"gateway ready",
|
||||
migration,
|
||||
`${migration}\n${closed}`,
|
||||
`${migration}\n${warned}`,
|
||||
`${migration}\n${failed}`,
|
||||
`${migration}\n${closed}\n${warned}`,
|
||||
`${migration}\n${closed}\n${failed}`,
|
||||
]) {
|
||||
expect(() => assertProjectWorktreeStartupLog(log, "first")).toThrow();
|
||||
}
|
||||
expect(() => assertProjectWorktreeStartupLog(`${migration}\n${closed}`, "second")).toThrow();
|
||||
},
|
||||
);
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue