mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 17:53:39 +00:00
chore: cover worker state in published upgrade tests (#149487)
* test: add worker state upgrade survivor cells Add opt-in Projects Doctor and terminal task/flow restoration cells using the unchanged published updater, verified package bytes, and the canonical survivor lifecycle. Preserve failed synthetic state until the outer Docker owner has joined. Validation: 161 focused tests, selected changed checks, and P2 review. Actual package upgrade cells remain separately qualified against frozen candidate artifacts. * test: activate taskflow survivor fixture on Gateway startup * test: clean worker runtime with container ownership * fix(test): preserve Docker status through exit traps
This commit is contained in:
Peter Steinberger
GitHub
parent
e75cc4dd91
commit
ec66133ba0
20 changed files with 1869 additions and 18 deletions
|
|
@ -116,10 +116,13 @@ const repositoryScriptEntries = [
|
|||
"scripts/e2e/lib/upgrade-survivor/missing-configured-plugin-migration.mjs!",
|
||||
"scripts/e2e/lib/upgrade-survivor/probe-gateway.mjs!",
|
||||
"scripts/e2e/lib/upgrade-survivor/probe-volume-gateway.mjs!",
|
||||
"scripts/e2e/lib/upgrade-survivor/projects-doctor.mjs!",
|
||||
"scripts/e2e/lib/upgrade-survivor/recovery-cleanup.mjs!",
|
||||
"scripts/e2e/lib/upgrade-survivor/schema-expectation.mjs!",
|
||||
// update-restart-auth.sh installs this manager/launch adapter into the fixture bin directory.
|
||||
"scripts/e2e/lib/upgrade-survivor/systemd-fixture.mjs!",
|
||||
"scripts/e2e/lib/upgrade-survivor/taskflow-restoration.mjs!",
|
||||
"scripts/e2e/lib/upgrade-survivor/worker-cell-package.mjs!",
|
||||
"scripts/e2e/lib/upgrade-survivor/mobile-pairing-client.mts!",
|
||||
"scripts/e2e/lib/upgrade-survivor/watchos-direct-node.mjs!",
|
||||
"scripts/embedded-run-abort-leak.ts!",
|
||||
|
|
|
|||
|
|
@ -185,6 +185,23 @@ Those default release runs pin this scenario to the published 2026.9.4 driver,
|
|||
including when the source candidate still reports version 2026.9.4; other
|
||||
scenarios retain their existing baseline selection.
|
||||
|
||||
The opt-in `projects-doctor` and `taskflow-restoration` scenarios require the exact
|
||||
published `openclaw@2026.9.4` baseline and a frozen candidate tarball. They use
|
||||
isolated state, manual restart, and no live providers or registry companion fixtures;
|
||||
neither runs through `reported-issues` or `far-reaching`. Both verify the original
|
||||
published driver and installed candidate payload bytes, including when their version
|
||||
strings are equal. Select one with `OPENCLAW_UPGRADE_SURVIVOR_SCENARIO` and set
|
||||
`OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC=openclaw@2026.9.4`.
|
||||
|
||||
`projects-doctor` preserves one registered project and one configured workspace,
|
||||
then runs the real `doctor --lint --only core/doctor/project-clone-shape --json`
|
||||
twice. It checks stored rows, schema, sentinels, and read-only snapshot cleanup.
|
||||
`taskflow-restoration` preserves three terminal tasks and two flows, starts a fresh
|
||||
candidate Gateway, exercises awaited task SDK reads through a synthetic local plugin,
|
||||
and reads two task pages on the same Gateway connection. Complete task, delivery,
|
||||
and flow records are checked again after Gateway shutdown. These cells cover
|
||||
terminal persisted state; they do not exercise active task recovery or provider work.
|
||||
|
||||
The `legacy-operator-state` scenario uses the published baseline's own CLI to
|
||||
create a second agent, allowlist exec approvals, and two command cron jobs: one
|
||||
without an explicit agent and one owned by `ops`. It leaves `systemAgent`
|
||||
|
|
|
|||
|
|
@ -42,6 +42,8 @@ const SCENARIOS = new Set([
|
|||
"configured-plugin-installs",
|
||||
"missing-configured-plugin-migration",
|
||||
"custom-plugin-siblings",
|
||||
"projects-doctor",
|
||||
"taskflow-restoration",
|
||||
"stale-source-plugin-shadow",
|
||||
"prerelease-plugin-registry",
|
||||
"tilde-log-path",
|
||||
|
|
@ -1664,6 +1666,13 @@ function assertSuccessfulUpdateJson([file, expectedVersion, observationRoot]) {
|
|||
const result = readUpdateJson(file, observationRoot);
|
||||
const plugins = result?.postUpdate?.plugins;
|
||||
assert(result?.status === "ok", `update did not report ok: ${String(result?.status)}`);
|
||||
if (["projects-doctor", "taskflow-restoration"].includes(getScenario())) {
|
||||
assertStrict.equal(
|
||||
result.before?.version,
|
||||
"2026.9.4",
|
||||
"Worker cell used the wrong published driver",
|
||||
);
|
||||
}
|
||||
const expectedMissingPluginFailure =
|
||||
getScenario() === "missing-configured-plugin-migration"
|
||||
? assertExpectedMissingCodexOutcome(result, expectedVersion)
|
||||
|
|
|
|||
405
scripts/e2e/lib/upgrade-survivor/projects-doctor.mjs
Normal file
405
scripts/e2e/lib/upgrade-survivor/projects-doctor.mjs
Normal file
|
|
@ -0,0 +1,405 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { resolveWorkerCellExport } from "./worker-cell-package.mjs";
|
||||
|
||||
// These chunks belong to the integrity-pinned published 2026.9.4 package.
|
||||
const BASELINE_CHUNKS = {
|
||||
"project-registry-D_ymI9b-.mjs":
|
||||
"89ba1a94f0238ff7e5c9ca7d4b42bc2dcb7c666c042c67a3fb79a44a832713f5",
|
||||
"kysely-sync-CrjZjQJR.mjs": "529a18c25ace3d1e8d40a630f5ffb30857aa0684d54ae39f4f2518424ceae13b",
|
||||
"openclaw-state-db-DoQEJuhr.mjs":
|
||||
"d3a52e65d0bca8993cbb6d48ae5d51e475e3aced8c7b9e15da77733f4fc8abcf",
|
||||
"openclaw-state-db-m8z7pMTn.mjs":
|
||||
"0efee81689942591023d3d68bc67cdab21a86c38bafc7bb275f03d4d62d3ce10",
|
||||
"openclaw-state-db-cache-BXmHZWzx.mjs":
|
||||
"6e8c7fe9bb935220a23b2193b0132f2e3686b64941aebd233493464fb06a72a1",
|
||||
};
|
||||
const STAGES = new Set([
|
||||
"baseline",
|
||||
"after-update",
|
||||
"before-doctor",
|
||||
"after-doctor",
|
||||
"before-repeat",
|
||||
"after-repeat",
|
||||
]);
|
||||
|
||||
function digest(file) {
|
||||
assert(fs.lstatSync(file).isFile(), `Expected a regular fixture file: ${file}`);
|
||||
return createHash("sha256").update(fs.readFileSync(file)).digest("hex");
|
||||
}
|
||||
|
||||
function readJson(file) {
|
||||
return JSON.parse(fs.readFileSync(file, "utf8"));
|
||||
}
|
||||
|
||||
function writeJson(file, value) {
|
||||
fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`, { flag: "wx" });
|
||||
}
|
||||
|
||||
function within(root, file) {
|
||||
const relative = path.relative(root, file);
|
||||
return (
|
||||
relative !== "" &&
|
||||
relative !== ".." &&
|
||||
!relative.startsWith(`..${path.sep}`) &&
|
||||
!path.isAbsolute(relative)
|
||||
);
|
||||
}
|
||||
|
||||
function context() {
|
||||
const required = (key) => {
|
||||
const value = process.env[key];
|
||||
assert(value && path.isAbsolute(value), `${key} must be an isolated absolute path`);
|
||||
return fs.realpathSync(value);
|
||||
};
|
||||
const root = required("OPENCLAW_UPGRADE_SURVIVOR_RUNTIME_ROOT");
|
||||
const artifacts = required("OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_ROOT");
|
||||
const stateDir = required("OPENCLAW_STATE_DIR");
|
||||
const configPath = process.env.OPENCLAW_CONFIG_PATH;
|
||||
assert(
|
||||
configPath && within(stateDir, path.resolve(configPath)),
|
||||
"Config must belong to the scenario state",
|
||||
);
|
||||
assert(within(root, stateDir), "State must belong to the scenario runtime");
|
||||
const tempRoots = [required("TMPDIR"), required("XDG_CACHE_HOME")];
|
||||
assert(
|
||||
tempRoots.every((dir) => within(root, dir)),
|
||||
"Snapshot roots must belong to the scenario runtime",
|
||||
);
|
||||
return {
|
||||
root,
|
||||
artifacts,
|
||||
stateDir,
|
||||
configPath,
|
||||
tempRoots,
|
||||
databasePath: path.join(stateDir, "state", "openclaw.sqlite"),
|
||||
manifestPath: path.join(artifacts, "projects-inventory.json"),
|
||||
};
|
||||
}
|
||||
|
||||
function snapshotPath(ctx, stage) {
|
||||
assert(STAGES.has(stage), `Unknown Projects snapshot stage: ${stage}`);
|
||||
return path.join(ctx.artifacts, `projects-${stage}.json`);
|
||||
}
|
||||
|
||||
function fixture(ctx) {
|
||||
const expected = readJson(ctx.manifestPath);
|
||||
assert.equal(expected.root, ctx.root, "Projects fixture belongs to another runtime");
|
||||
assert.equal(
|
||||
expected.stateDir,
|
||||
ctx.stateDir,
|
||||
"Projects fixture belongs to another state directory",
|
||||
);
|
||||
assert.equal(expected.configPath, ctx.configPath, "Projects fixture belongs to another config");
|
||||
return expected;
|
||||
}
|
||||
|
||||
function retainedSnapshots(roots) {
|
||||
const retained = [];
|
||||
const visit = (dir) => {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const file = path.join(dir, entry.name);
|
||||
if (/^openclaw-(?:sqlite-readonly-|doctor-lint-state-)/.test(entry.name)) {
|
||||
retained.push(file);
|
||||
}
|
||||
if (entry.isDirectory()) {
|
||||
visit(file);
|
||||
}
|
||||
}
|
||||
};
|
||||
for (const root of new Set(roots)) {
|
||||
visit(root);
|
||||
}
|
||||
return retained.toSorted((a, b) => a.localeCompare(b));
|
||||
}
|
||||
|
||||
function sqliteFamily(databasePath) {
|
||||
return Object.fromEntries(
|
||||
["", "-wal", "-shm", "-journal"].flatMap((suffix) => {
|
||||
const file = `${databasePath}${suffix}`;
|
||||
return fs.existsSync(file) ? [[suffix || "main", digest(file)]] : [];
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export function assertProjectsInventory(inventory, baseline) {
|
||||
assert.deepEqual(inventory.rows, baseline.rows, "Persisted project inventory changed");
|
||||
assert.deepEqual(inventory.schema, baseline.schema, "Projects table schema changed");
|
||||
assert.equal(inventory.workspace, baseline.workspace, "Configured workspace changed");
|
||||
assert.deepEqual(inventory.sentinels, baseline.sentinels, "Project or workspace files changed");
|
||||
}
|
||||
|
||||
export function assertProjectsDoctorResult(report, before, after) {
|
||||
assert.equal(report.ok, true, "Projects Doctor reported an unhealthy result");
|
||||
assert.equal(report.checksRun, 1, "Projects Doctor must run exactly the selected check");
|
||||
assert.deepEqual(report.findings, [], "Projects Doctor skipped or failed inventory inspection");
|
||||
assertProjectsInventory(after, before);
|
||||
assert.deepEqual(
|
||||
after.sqliteFamily,
|
||||
before.sqliteFamily,
|
||||
"Read-only Doctor changed SQLite files",
|
||||
);
|
||||
assert.equal(after.configHash, before.configHash, "Read-only Doctor rewrote the config");
|
||||
assert.deepEqual(
|
||||
after.retainedSnapshots,
|
||||
before.retainedSnapshots,
|
||||
"Doctor retained a private snapshot",
|
||||
);
|
||||
}
|
||||
|
||||
async function artifactPreservingReader(ctx, stage, packageRoot) {
|
||||
assert(
|
||||
packageRoot && path.isAbsolute(packageRoot),
|
||||
"Snapshot requires the installed package root",
|
||||
);
|
||||
const identity = readJson(
|
||||
path.join(
|
||||
ctx.artifacts,
|
||||
stage === "baseline" ? "baseline-package-identity.json" : "installed-package-identity.json",
|
||||
),
|
||||
);
|
||||
assert.equal(
|
||||
fs.realpathSync(path.join(packageRoot, "openclaw.mjs")),
|
||||
identity.cli,
|
||||
"Snapshot package differs from the verified installed CLI",
|
||||
);
|
||||
const symbol = "withExistingOpenClawStateDatabaseArtifactPreservingReadOnly";
|
||||
const matches = [];
|
||||
for (const [relative, expected] of Object.entries(identity.files)) {
|
||||
if (!/^dist\/openclaw-state-db-readonly-[\w-]+\.mjs$/.test(relative)) {
|
||||
continue;
|
||||
}
|
||||
const file = path.join(packageRoot, relative);
|
||||
assert(fs.lstatSync(file).isFile(), `Expected a regular snapshot owner: ${file}`);
|
||||
const bytes = fs.readFileSync(file);
|
||||
const sha256 = createHash("sha256").update(bytes).digest("hex");
|
||||
assert.equal(sha256, expected.sha256, `Installed snapshot owner changed: ${relative}`);
|
||||
const alias = resolveWorkerCellExport(bytes.toString("utf8"), symbol);
|
||||
if (alias) {
|
||||
matches.push({ file, relative, sha256, alias });
|
||||
}
|
||||
}
|
||||
assert.equal(
|
||||
matches.length,
|
||||
1,
|
||||
"Expected exactly one installed artifact-preserving state reader",
|
||||
);
|
||||
const binding = matches[0];
|
||||
const module = await import(pathToFileURL(binding.file).href);
|
||||
const read = module[binding.alias];
|
||||
assert.equal(typeof read, "function", "Installed artifact-preserving state reader is missing");
|
||||
return {
|
||||
read,
|
||||
evidence: { file: binding.relative, sha256: binding.sha256, export: binding.alias },
|
||||
};
|
||||
}
|
||||
|
||||
async function snapshot(ctx, stage, packageRoot) {
|
||||
const expected = fixture(ctx);
|
||||
const familyBefore = sqliteFamily(ctx.databasePath);
|
||||
const reader = await artifactPreservingReader(ctx, stage, packageRoot);
|
||||
const observed = reader.read(
|
||||
({ db }) => ({
|
||||
rows: db
|
||||
.prepare(
|
||||
"SELECT id, display_name, repo_root, origin_url, source, created_at_ms, updated_at_ms FROM projects ORDER BY id",
|
||||
)
|
||||
.all()
|
||||
.map((row) => Object.assign({}, row)),
|
||||
schema: db
|
||||
.prepare(
|
||||
"SELECT type, name, sql FROM sqlite_schema WHERE tbl_name = 'projects' ORDER BY type, name",
|
||||
)
|
||||
.all()
|
||||
.map((row) => Object.assign({}, row)),
|
||||
}),
|
||||
{ path: ctx.databasePath, env: process.env },
|
||||
);
|
||||
assert(observed, "Projects state database is missing");
|
||||
const familyAfter = sqliteFamily(ctx.databasePath);
|
||||
assert.deepEqual(
|
||||
familyAfter,
|
||||
familyBefore,
|
||||
"Inventory observer changed SQLite files; Doctor mutation evidence is inconclusive",
|
||||
);
|
||||
const config = readJson(ctx.configPath);
|
||||
const result = {
|
||||
...observed,
|
||||
observer: reader.evidence,
|
||||
workspace: config.agents?.defaults?.workspace,
|
||||
sentinels: Object.fromEntries(
|
||||
Object.keys(expected.sentinels).map((file) => [file, digest(file)]),
|
||||
),
|
||||
configHash: digest(ctx.configPath),
|
||||
sqliteFamily: familyAfter,
|
||||
retainedSnapshots: retainedSnapshots(ctx.tempRoots),
|
||||
};
|
||||
assert.deepEqual(result.rows, [expected.row], "Expected exactly the seeded registered project");
|
||||
assert.equal(result.workspace, expected.workspace, "Configured workspace changed");
|
||||
assert.deepEqual(result.sentinels, expected.sentinels, "Synthetic project files changed");
|
||||
assert.deepEqual(result.retainedSnapshots, [], "A private SQLite/Doctor snapshot remains");
|
||||
if (stage !== "baseline") {
|
||||
assertProjectsInventory(result, readJson(snapshotPath(ctx, "baseline")));
|
||||
}
|
||||
writeJson(snapshotPath(ctx, stage), result);
|
||||
}
|
||||
|
||||
async function seed(ctx, packageRoot) {
|
||||
assert(
|
||||
packageRoot && path.isAbsolute(packageRoot),
|
||||
"Seed requires the installed published package root",
|
||||
);
|
||||
const manifest = readJson(path.join(packageRoot, "package.json"));
|
||||
assert.equal(manifest.name, "openclaw");
|
||||
assert.equal(manifest.version, "2026.9.4", "Fixture must be created by the published baseline");
|
||||
assert(!fs.existsSync(ctx.manifestPath), "Projects fixture was already seeded");
|
||||
for (const [file, expectedHash] of Object.entries(BASELINE_CHUNKS)) {
|
||||
assert.equal(
|
||||
digest(path.join(packageRoot, "dist", file)),
|
||||
expectedHash,
|
||||
`Published module changed: ${file}`,
|
||||
);
|
||||
}
|
||||
const workspace = path.join(ctx.root, "workspace");
|
||||
const registered = path.join(ctx.root, "registered");
|
||||
const sentinels = {};
|
||||
for (const [dir, text] of [
|
||||
[workspace, "workspace survives update — 東京\n"],
|
||||
[registered, "registered project survives update — λ\n"],
|
||||
]) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
assert(
|
||||
!fs.existsSync(path.join(dir, ".git")),
|
||||
"Projects fixture must not contain a Git checkout",
|
||||
);
|
||||
const file = path.join(dir, "PROJECTS-PROOF.txt");
|
||||
fs.writeFileSync(file, text, { flag: "wx" });
|
||||
sentinels[file] = digest(file);
|
||||
}
|
||||
const cfg = {
|
||||
gateway: {
|
||||
mode: "local",
|
||||
bind: "loopback",
|
||||
auth: { mode: "token", token: "projects-doctor-survivor-token" },
|
||||
controlUi: { enabled: false },
|
||||
},
|
||||
agents: { defaults: { workspace, heartbeat: { every: "0m" } } },
|
||||
plugins: { enabled: false },
|
||||
};
|
||||
fs.writeFileSync(ctx.configPath, `${JSON.stringify(cfg, null, 2)}\n`);
|
||||
const row = {
|
||||
id: "doctor-upgrade-registered",
|
||||
display_name: "Doctor upgrade inventory — 東京",
|
||||
repo_root: registered,
|
||||
origin_url: null,
|
||||
source: "registered",
|
||||
created_at_ms: 1789430400000,
|
||||
updated_at_ms: 1789430400000,
|
||||
};
|
||||
const load = (file) => import(pathToFileURL(path.join(packageRoot, "dist", file)).href);
|
||||
const state = await load("openclaw-state-db-m8z7pMTn.mjs");
|
||||
const { n: listProjectRegistry } = await load("project-registry-D_ymI9b-.mjs");
|
||||
const { i: getNodeSqliteKysely, n: executeSqliteQuerySync } = await load(
|
||||
"kysely-sync-CrjZjQJR.mjs",
|
||||
);
|
||||
const options = { path: ctx.databasePath, env: process.env };
|
||||
try {
|
||||
// The published list owner bootstraps its own schema without probing Git.
|
||||
const initial = listProjectRegistry(cfg, options);
|
||||
assert.equal(
|
||||
initial.length,
|
||||
1,
|
||||
"Expected only the configured workspace before fixture insertion",
|
||||
);
|
||||
assert.equal(initial[0].source, "workspace");
|
||||
state.runOpenClawStateWriteTransaction(
|
||||
({ db }) => {
|
||||
executeSqliteQuerySync(db, getNodeSqliteKysely(db).insertInto("projects").values(row));
|
||||
},
|
||||
options,
|
||||
{ operationLabel: "projects.upgrade.fixture" },
|
||||
);
|
||||
const inventory = listProjectRegistry(cfg, options);
|
||||
assert.equal(inventory.length, 2);
|
||||
assert.deepEqual(
|
||||
inventory.find((project) => project.id === row.id),
|
||||
{
|
||||
id: row.id,
|
||||
displayName: row.display_name,
|
||||
repoRoot: row.repo_root,
|
||||
source: row.source,
|
||||
},
|
||||
);
|
||||
assert.equal(inventory.find((project) => project.source === "workspace")?.repoRoot, workspace);
|
||||
} finally {
|
||||
state.closeOpenClawStateDatabaseByPath(ctx.databasePath);
|
||||
}
|
||||
assert.equal(
|
||||
state.isOpenClawStateDatabaseOpen(ctx.databasePath),
|
||||
false,
|
||||
"Published fixture writer did not close",
|
||||
);
|
||||
writeJson(ctx.manifestPath, {
|
||||
root: ctx.root,
|
||||
stateDir: ctx.stateDir,
|
||||
configPath: ctx.configPath,
|
||||
baselineVersion: manifest.version,
|
||||
baselineChunks: BASELINE_CHUNKS,
|
||||
row,
|
||||
workspace,
|
||||
sentinels,
|
||||
});
|
||||
await snapshot(ctx, "baseline", packageRoot);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const [mode, ...args] = process.argv.slice(2);
|
||||
const ctx = context();
|
||||
if (mode === "seed") {
|
||||
assert.equal(args.length, 1);
|
||||
await seed(ctx, args[0]);
|
||||
} else if (mode === "snapshot") {
|
||||
assert.equal(args.length, 2);
|
||||
await snapshot(ctx, args[0], args[1]);
|
||||
} else if (mode === "assert-doctor") {
|
||||
assert.equal(args.length, 3);
|
||||
const [reportPath, before, after] = args;
|
||||
assertProjectsDoctorResult(
|
||||
readJson(reportPath),
|
||||
readJson(snapshotPath(ctx, before)),
|
||||
readJson(snapshotPath(ctx, after)),
|
||||
);
|
||||
writeJson(path.join(ctx.artifacts, `projects-${after}-assertion.json`), {
|
||||
ok: true,
|
||||
before,
|
||||
after,
|
||||
});
|
||||
} else if (mode === "assert-final") {
|
||||
assert.equal(args.length, 0);
|
||||
for (const stage of STAGES) {
|
||||
assert(fs.existsSync(snapshotPath(ctx, stage)), `Missing Projects phase: ${stage}`);
|
||||
}
|
||||
for (const stage of ["after-doctor", "after-repeat"]) {
|
||||
assert.equal(readJson(path.join(ctx.artifacts, `projects-${stage}-assertion.json`)).ok, true);
|
||||
}
|
||||
const baseline = readJson(snapshotPath(ctx, "baseline"));
|
||||
const final = readJson(snapshotPath(ctx, "after-repeat"));
|
||||
assertProjectsInventory(final, baseline);
|
||||
writeJson(path.join(ctx.artifacts, "projects-doctor-result.json"), {
|
||||
ok: true,
|
||||
baselineVersion: "2026.9.4",
|
||||
storedProjects: final.rows.length,
|
||||
workspace: final.workspace,
|
||||
doctorInvocations: 2,
|
||||
});
|
||||
} else {
|
||||
throw new Error(`Unknown Projects Doctor fixture mode: ${mode}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
|
||||
await main();
|
||||
}
|
||||
|
|
@ -1,4 +1,8 @@
|
|||
#!/usr/bin/env bash
|
||||
# Bash 5.3+ can deadlock writing heredoc pipes on macOS before the reader starts.
|
||||
if [[ ${OSTYPE:-} == darwin* && $BASH != /bin/bash ]] && ((BASH_VERSINFO[0] > 5 || (BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] >= 3))); then
|
||||
exec /bin/bash "$0" "$@"
|
||||
fi
|
||||
set -Eeuo pipefail
|
||||
# Signal traps inherit the foreground command's redirections. Keep harness stdout separate so the
|
||||
# final summary location cannot corrupt a command artifact when the run is interrupted.
|
||||
|
|
@ -10,6 +14,10 @@ source scripts/e2e/lib/upgrade-survivor/plugin-dependency-fixtures.sh
|
|||
source scripts/e2e/lib/upgrade-survivor/backup-rollback.sh
|
||||
|
||||
SCENARIO="${OPENCLAW_UPGRADE_SURVIVOR_SCENARIO:-base}"
|
||||
WORKER_CELL=0
|
||||
if [ "$SCENARIO" = "projects-doctor" ] || [ "$SCENARIO" = "taskflow-restoration" ]; then
|
||||
WORKER_CELL=1
|
||||
fi
|
||||
|
||||
export npm_config_loglevel=error
|
||||
export npm_config_fund=false
|
||||
|
|
@ -43,7 +51,7 @@ if [ "$SCENARIO" = "mobile-pairing-reconnect" ]; then
|
|||
node -e 'process.stdout.write(require("node:crypto").randomBytes(32).toString("hex"))'
|
||||
)"
|
||||
fi
|
||||
if [ "$SCENARIO" = "watchos-direct-node" ] || [ "$SCENARIO" = "mobile-pairing-reconnect" ]; then
|
||||
if [ "$SCENARIO" = "watchos-direct-node" ] || [ "$SCENARIO" = "mobile-pairing-reconnect" ] || [ "$WORKER_CELL" = "1" ]; then
|
||||
unset OPENAI_API_KEY DISCORD_BOT_TOKEN TELEGRAM_BOT_TOKEN
|
||||
else
|
||||
export OPENAI_API_KEY="sk-openclaw-upgrade-survivor"
|
||||
|
|
@ -69,6 +77,12 @@ chmod 700 "$RUNTIME_ROOT"
|
|||
export TMPDIR="${OPENCLAW_UPGRADE_SURVIVOR_TMPDIR:-$RUNTIME_ROOT/tmp}"
|
||||
export OPENCLAW_TEST_STATE_TMPDIR="${OPENCLAW_UPGRADE_SURVIVOR_TEST_STATE_TMPDIR:-$RUNTIME_ROOT/state-tmp}"
|
||||
mkdir -p "$TMPDIR" "$OPENCLAW_TEST_STATE_TMPDIR"
|
||||
if [ "$WORKER_CELL" = "1" ]; then
|
||||
export XDG_CACHE_HOME="$RUNTIME_ROOT/xdg-cache"
|
||||
export OPENCLAW_SKIP_CRON=1
|
||||
export OPENCLAW_SKIP_STARTUP_MODEL_PREWARM=1
|
||||
mkdir -p "$XDG_CACHE_HOME"
|
||||
fi
|
||||
if [ "$SCENARIO" = "legacy-operator-state" ]; then
|
||||
export npm_config_prefix="$RUNTIME_ROOT/npm-prefix"
|
||||
else
|
||||
|
|
@ -1942,11 +1956,89 @@ assertAgentReplyContainsMarker(process.argv[2], process.argv[3]);
|
|||
NODE
|
||||
}
|
||||
|
||||
prepare_worker_cell_package() {
|
||||
node scripts/e2e/lib/upgrade-survivor/worker-cell-package.mjs candidate "$(package_root)" "$CANDIDATE_SPEC"
|
||||
OPENCLAW_UPGRADE_SURVIVOR_CANDIDATE_COMMIT="$(node -e \
|
||||
'process.stdout.write(JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")).buildInfo.commit)' \
|
||||
"$ARTIFACT_ROOT/candidate-package-identity.json")"
|
||||
export OPENCLAW_UPGRADE_SURVIVOR_CANDIDATE_COMMIT
|
||||
}
|
||||
|
||||
assert_worker_cell_update() {
|
||||
if [ "$update_outcome" != "success" ] || [ "$update_repair_required" != "0" ]; then
|
||||
echo "$SCENARIO requires successful original-driver update without follow-up repair" >&2
|
||||
return 1
|
||||
fi
|
||||
node scripts/e2e/lib/upgrade-survivor/worker-cell-package.mjs installed "$(package_root)" "$CANDIDATE_SPEC"
|
||||
}
|
||||
|
||||
run_projects_doctor() {
|
||||
local stage="$1"
|
||||
openclaw_e2e_maybe_timeout "$COMMAND_TIMEOUT" env \
|
||||
-u OPENCLAW_UPDATE_IN_PROGRESS \
|
||||
-u OPENCLAW_UPDATE_POST_CORE_CONVERGENCE \
|
||||
-u OPENCLAW_UPDATE_PARENT_SUPPORTS_DOCTOR_CONFIG_WRITE \
|
||||
-u OPENCLAW_UPDATE_DEFER_CONFIGURED_PLUGIN_INSTALL_REPAIR \
|
||||
openclaw doctor --lint --only core/doctor/project-clone-shape --json \
|
||||
>"$ARTIFACT_ROOT/projects-doctor-$stage.json" 2>"$ARTIFACT_ROOT/projects-doctor-$stage.err"
|
||||
}
|
||||
|
||||
validate_worker_cell() {
|
||||
if [ "$WORKER_CELL" != "1" ]; then
|
||||
return 0
|
||||
fi
|
||||
if [ "$BASELINE_RAW" != "openclaw@2026.9.4" ] || [ "$CANDIDATE_KIND" != "tarball" ] ||
|
||||
[ "$UPDATE_RESTART_MODE" != "manual" ] || [ "$ROOT_MANAGED_VPS" != "0" ] || [ "$LIVE_OPENAI" != "0" ]; then
|
||||
echo "$SCENARIO requires published openclaw@2026.9.4, a candidate tarball, isolated manual restart, and no live provider" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
phase storage-preflight storage_preflight
|
||||
phase validate-update-restart-mode validate_update_restart_mode
|
||||
phase validate-worker-cell validate_worker_cell
|
||||
phase reset-run-state reset_run_state
|
||||
phase install-baseline install_baseline
|
||||
phase initialize-state initialize_state
|
||||
if [ "$WORKER_CELL" = "1" ]; then
|
||||
phase worker-baseline-identity node scripts/e2e/lib/upgrade-survivor/worker-cell-package.mjs baseline "$(package_root)"
|
||||
if [ "$SCENARIO" = "projects-doctor" ]; then
|
||||
phase seed-projects-inventory node scripts/e2e/lib/upgrade-survivor/projects-doctor.mjs seed "$(package_root)"
|
||||
else
|
||||
phase seed-taskflow node scripts/e2e/lib/upgrade-survivor/taskflow-restoration.mjs seed --package-root "$(package_root)"
|
||||
fi
|
||||
phase validate-baseline-config validate_baseline_config
|
||||
phase resolve-worker-candidate resolve_candidate_version
|
||||
phase worker-candidate-identity prepare_worker_cell_package
|
||||
phase update-worker-candidate update_candidate
|
||||
phase assert-worker-installed-identity assert_worker_cell_update
|
||||
if [ "$SCENARIO" = "projects-doctor" ]; then
|
||||
phase projects-after-update node scripts/e2e/lib/upgrade-survivor/projects-doctor.mjs snapshot after-update "$(package_root)"
|
||||
phase projects-before-doctor node scripts/e2e/lib/upgrade-survivor/projects-doctor.mjs snapshot before-doctor "$(package_root)"
|
||||
phase projects-doctor run_projects_doctor first
|
||||
phase projects-after-doctor node scripts/e2e/lib/upgrade-survivor/projects-doctor.mjs snapshot after-doctor "$(package_root)"
|
||||
phase assert-projects-doctor node scripts/e2e/lib/upgrade-survivor/projects-doctor.mjs \
|
||||
assert-doctor "$ARTIFACT_ROOT/projects-doctor-first.json" before-doctor after-doctor
|
||||
phase projects-before-repeat node scripts/e2e/lib/upgrade-survivor/projects-doctor.mjs snapshot before-repeat "$(package_root)"
|
||||
phase projects-doctor-repeat run_projects_doctor repeat
|
||||
phase projects-after-repeat node scripts/e2e/lib/upgrade-survivor/projects-doctor.mjs snapshot after-repeat "$(package_root)"
|
||||
phase assert-projects-doctor-repeat node scripts/e2e/lib/upgrade-survivor/projects-doctor.mjs \
|
||||
assert-doctor "$ARTIFACT_ROOT/projects-doctor-repeat.json" before-repeat after-repeat
|
||||
phase assert-projects-preservation node scripts/e2e/lib/upgrade-survivor/projects-doctor.mjs assert-final
|
||||
else
|
||||
phase gateway-start start_gateway
|
||||
phase gateway-probes check_gateway_probes
|
||||
phase taskflow-sdk-and-pages node scripts/e2e/lib/upgrade-survivor/taskflow-restoration.mjs probe \
|
||||
--package-root "$(package_root)" --url ws://127.0.0.1:18789 \
|
||||
--expected-commit "$OPENCLAW_UPGRADE_SURVIVOR_CANDIDATE_COMMIT"
|
||||
phase gateway-stop stop_gateway
|
||||
phase assert-taskflow-persistence node scripts/e2e/lib/upgrade-survivor/taskflow-restoration.mjs assert-state \
|
||||
--package-root "$(package_root)" --expected-commit "$OPENCLAW_UPGRADE_SURVIVOR_CANDIDATE_COMMIT"
|
||||
fi
|
||||
run_completed="1"
|
||||
echo "Upgrade survivor Docker E2E passed baseline=${baseline_spec} scenario=${SCENARIO} candidate=${candidate_version}."
|
||||
exit 0
|
||||
fi
|
||||
if [ "$SCENARIO" = "workshop-doctor-recovery" ]; then
|
||||
if [ "$baseline_spec" != "openclaw@2026.9.4" ]; then
|
||||
echo "workshop-doctor-recovery requires the exact published openclaw@2026.9.4 baseline" >&2
|
||||
|
|
|
|||
|
|
@ -0,0 +1,234 @@
|
|||
import assert from "node:assert/strict";
|
||||
|
||||
export const TASKFLOW_PLUGIN_ID = "taskflow-survivor";
|
||||
export const TASKFLOW_PLUGIN_MANIFEST = {
|
||||
id: TASKFLOW_PLUGIN_ID,
|
||||
activation: { onStartup: true },
|
||||
configSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
};
|
||||
export const TASKFLOW_METHOD = "taskflow-survivor.read";
|
||||
export const TASKFLOW_OWNER = "agent:main:taskflow-update-cell";
|
||||
export const TASKFLOW_TASK_IDS = [1, 2, 3].map((index) => `update-cell-task-0${index}`);
|
||||
export const TASKFLOW_FLOW_IDS = ["update-cell-flow-managed", "update-cell-flow-mirrored"];
|
||||
|
||||
export function createTaskflowFixture(now) {
|
||||
assert(Number.isSafeInteger(now) && now > 3003, "Invalid fixture timestamp");
|
||||
const tasks = TASKFLOW_TASK_IDS.map((taskId, offset) => {
|
||||
const index = offset + 1;
|
||||
return {
|
||||
taskId,
|
||||
runtime: "cli",
|
||||
sourceId: `update-cell-source-${index}`,
|
||||
requesterSessionKey: TASKFLOW_OWNER,
|
||||
ownerKey: TASKFLOW_OWNER,
|
||||
scopeKind: "session",
|
||||
parentFlowId: TASKFLOW_FLOW_IDS[index === 3 ? 1 : 0],
|
||||
agentId: "main",
|
||||
requesterAgentId: "main",
|
||||
runId: `update-cell-run-0${index}`,
|
||||
label: `Task/flow survivor ${index} — 東京`,
|
||||
task: `Task/flow survivor ${index} — 東京`,
|
||||
status: "succeeded",
|
||||
deliveryStatus: "not_applicable",
|
||||
notifyPolicy: "silent",
|
||||
createdAt: now - 3000 - index,
|
||||
startedAt: now - 2000 - index,
|
||||
endedAt: now - 1000 - index,
|
||||
lastEventAt: now - 1000 - index,
|
||||
cleanupAfter: now + 86_400_000,
|
||||
toolUseCount: index,
|
||||
lastToolName: "synthetic-fixture",
|
||||
progressSummary: `Progress ${index} — 東京`,
|
||||
terminalSummary: `Completed ${index} — 東京`,
|
||||
terminalOutcome: "succeeded",
|
||||
detail: {
|
||||
fixture: "taskflow-update-cell",
|
||||
index,
|
||||
payload: { enabled: true, optional: null },
|
||||
tags: ["persisted", "東京"],
|
||||
},
|
||||
};
|
||||
});
|
||||
const flows = TASKFLOW_FLOW_IDS.map((flowId, index) => {
|
||||
const flow = {
|
||||
flowId,
|
||||
syncMode: index === 0 ? "managed" : "task_mirrored",
|
||||
ownerKey: TASKFLOW_OWNER,
|
||||
revision: index === 0 ? 7 : 3,
|
||||
status: "succeeded",
|
||||
notifyPolicy: "silent",
|
||||
goal: `Flow survivor ${index} — 東京`,
|
||||
currentStep: "Complete",
|
||||
stateJson: { fixture: "taskflow-update-cell", index, payload: [true, null, "東京"] },
|
||||
createdAt: now - 4000 - index,
|
||||
updatedAt: tasks[index === 0 ? 0 : 2].endedAt,
|
||||
endedAt: tasks[index === 0 ? 0 : 2].endedAt,
|
||||
};
|
||||
if (index === 0) {
|
||||
flow.controllerId = "update-cell-controller";
|
||||
}
|
||||
return flow;
|
||||
});
|
||||
const deliveryStates = tasks.map((task) => ({
|
||||
taskId: task.taskId,
|
||||
lastNotifiedEventAt: task.endedAt,
|
||||
}));
|
||||
return { tasks, flows, deliveryStates };
|
||||
}
|
||||
|
||||
// Compare JSON-visible owner records; optional absent fields are not durable values.
|
||||
export function normalizeTaskflowSnapshot(snapshot) {
|
||||
return Object.fromEntries(
|
||||
["tasks", "flows", "deliveryStates"].map((key) => {
|
||||
const id = key === "flows" ? "flowId" : "taskId";
|
||||
const records = snapshot[key] instanceof Map ? [...snapshot[key].values()] : snapshot[key];
|
||||
assert(Array.isArray(records), `Missing ${key} snapshot`);
|
||||
const serializedRecords = JSON.stringify(records);
|
||||
return [key, JSON.parse(serializedRecords).toSorted((a, b) => a[id].localeCompare(b[id]))];
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export function assertTaskflowSnapshot(actual, expected) {
|
||||
assert.deepEqual(normalizeTaskflowSnapshot(actual), normalizeTaskflowSnapshot(expected));
|
||||
}
|
||||
|
||||
function runView(task) {
|
||||
const { taskId, requesterSessionKey, scopeKind, parentFlowId, task: title } = task;
|
||||
const fields = [
|
||||
"runtime",
|
||||
"sourceId",
|
||||
"ownerKey",
|
||||
"agentId",
|
||||
"runId",
|
||||
"label",
|
||||
"status",
|
||||
"deliveryStatus",
|
||||
"notifyPolicy",
|
||||
"createdAt",
|
||||
"startedAt",
|
||||
"endedAt",
|
||||
"lastEventAt",
|
||||
"cleanupAfter",
|
||||
"progressSummary",
|
||||
"terminalSummary",
|
||||
"terminalOutcome",
|
||||
];
|
||||
return {
|
||||
id: taskId,
|
||||
sessionKey: requesterSessionKey,
|
||||
scope: scopeKind,
|
||||
flowId: parentFlowId,
|
||||
title,
|
||||
...Object.fromEntries(fields.map((key) => [key, task[key]])),
|
||||
};
|
||||
}
|
||||
|
||||
function flowView(flow) {
|
||||
return {
|
||||
id: flow.flowId,
|
||||
...Object.fromEntries(
|
||||
[
|
||||
"ownerKey",
|
||||
"status",
|
||||
"notifyPolicy",
|
||||
"goal",
|
||||
"currentStep",
|
||||
"createdAt",
|
||||
"updatedAt",
|
||||
"endedAt",
|
||||
].map((key) => [key, flow[key]]),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function summary(total) {
|
||||
return {
|
||||
total,
|
||||
active: 0,
|
||||
terminal: total,
|
||||
failures: 0,
|
||||
byStatus: {
|
||||
queued: 0,
|
||||
running: 0,
|
||||
succeeded: total,
|
||||
failed: 0,
|
||||
timed_out: 0,
|
||||
cancelled: 0,
|
||||
lost: 0,
|
||||
},
|
||||
byRuntime: { subagent: 0, acp: 0, cron: 0, cli: total },
|
||||
};
|
||||
}
|
||||
|
||||
export function assertTaskflowSdkReads(actual, fixture) {
|
||||
assert.equal(actual.ownerKey, TASKFLOW_OWNER);
|
||||
const runs = fixture.tasks.map(runView);
|
||||
assert.deepEqual(actual.runs, runs);
|
||||
assert.deepEqual(actual.runDetails, runs);
|
||||
assert.deepEqual(actual.resolvedRuns, runs);
|
||||
assert.deepEqual(actual.flows, fixture.flows.map(flowView));
|
||||
assert.deepEqual(actual.managedFlow, fixture.flows[0]);
|
||||
for (const [index, flow] of fixture.flows.entries()) {
|
||||
const tasks = fixture.tasks.filter((task) => task.parentFlowId === flow.flowId).map(runView);
|
||||
assert.deepEqual(actual.flowDetails[index], {
|
||||
...flowView(flow),
|
||||
state: flow.stateJson,
|
||||
tasks,
|
||||
taskSummary: summary(tasks.length),
|
||||
});
|
||||
assert.deepEqual(actual.flowSummaries[index], summary(tasks.length));
|
||||
}
|
||||
}
|
||||
|
||||
export function assertTaskflowGatewayReads(pages, details, fixture) {
|
||||
assert.equal(pages.length, 2);
|
||||
assert.equal(pages[0].tasks.length, 2);
|
||||
assert.equal(pages[1].tasks.length, 1);
|
||||
assert.equal(typeof pages[0].nextCursor, "string");
|
||||
assert(pages[0].nextCursor.length > 0);
|
||||
assert.equal(pages[1].nextCursor, undefined);
|
||||
assert.deepEqual(
|
||||
pages.flatMap((page) => page.tasks.map((task) => task.id)),
|
||||
TASKFLOW_TASK_IDS,
|
||||
);
|
||||
assert.equal(details.length, fixture.tasks.length);
|
||||
for (const [index, task] of fixture.tasks.entries()) {
|
||||
const listed = pages.flatMap((page) => page.tasks)[index];
|
||||
const detailed = details[index].task;
|
||||
for (const result of [listed, detailed]) {
|
||||
assert.equal(result.id, task.taskId);
|
||||
assert.equal(result.taskId, task.taskId);
|
||||
assert.equal(result.title, task.label);
|
||||
assert.equal(result.kind, task.runtime);
|
||||
assert.equal(result.status, "completed");
|
||||
assert.equal(result.execution.state, "finished");
|
||||
for (const key of [
|
||||
"runtime",
|
||||
"ownerKey",
|
||||
"agentId",
|
||||
"runId",
|
||||
"sourceId",
|
||||
"createdAt",
|
||||
"startedAt",
|
||||
"endedAt",
|
||||
"toolUseCount",
|
||||
"lastToolName",
|
||||
"progressSummary",
|
||||
"terminalSummary",
|
||||
"deliveryStatus",
|
||||
"terminalOutcome",
|
||||
]) {
|
||||
assert.deepEqual(result[key], task[key], `Gateway ${task.taskId}.${key}`);
|
||||
}
|
||||
assert.equal(result.flowId, task.parentFlowId);
|
||||
assert.equal(result.sessionKey, task.requesterSessionKey);
|
||||
assert.equal(result.updatedAt, task.lastEventAt);
|
||||
// The protocol advertises a requester-session history route even without a transcript.
|
||||
assert.equal(result.hasTranscript, true);
|
||||
}
|
||||
assert.equal(detailed.prompt, task.task);
|
||||
assert.equal(detailed.result, task.terminalSummary);
|
||||
assert.equal(listed.prompt, undefined);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
import {
|
||||
TASKFLOW_FLOW_IDS,
|
||||
TASKFLOW_METHOD,
|
||||
TASKFLOW_OWNER,
|
||||
TASKFLOW_PLUGIN_ID,
|
||||
TASKFLOW_TASK_IDS,
|
||||
} from "./taskflow-restoration-fixture.mjs";
|
||||
|
||||
export default {
|
||||
id: TASKFLOW_PLUGIN_ID,
|
||||
register(api) {
|
||||
// Registration is inert on the published baseline; only the candidate is queried.
|
||||
api.registerGatewayMethod(
|
||||
TASKFLOW_METHOD,
|
||||
async ({ respond }) => {
|
||||
const tasks = api.runtime.tasks.async;
|
||||
const binding = { sessionKey: TASKFLOW_OWNER, agentId: "main" };
|
||||
const runs = tasks.runs.bindSession(binding);
|
||||
const flows = tasks.flows.bindSession(binding);
|
||||
const managed = tasks.managedFlows.bindSession(binding);
|
||||
const timings = [];
|
||||
const read = async (name, operation) => {
|
||||
const started = performance.now();
|
||||
try {
|
||||
return await operation();
|
||||
} finally {
|
||||
timings.push({ name, elapsedMs: performance.now() - started });
|
||||
}
|
||||
};
|
||||
const runDetails = [];
|
||||
const resolvedRuns = [];
|
||||
for (const [index, id] of TASKFLOW_TASK_IDS.entries()) {
|
||||
runDetails.push(await read(`runs.get:${id}`, () => runs.get(id)));
|
||||
resolvedRuns.push(
|
||||
await read(`runs.resolve:${id}`, () => runs.resolve(`update-cell-run-0${index + 1}`)),
|
||||
);
|
||||
}
|
||||
const flowDetails = [];
|
||||
const flowSummaries = [];
|
||||
for (const id of TASKFLOW_FLOW_IDS) {
|
||||
flowDetails.push(await read(`flows.get:${id}`, () => flows.get(id)));
|
||||
flowSummaries.push(
|
||||
await read(`flows.getTaskSummary:${id}`, () => flows.getTaskSummary(id)),
|
||||
);
|
||||
}
|
||||
respond(true, {
|
||||
ownerKey: TASKFLOW_OWNER,
|
||||
runtimeVersion: api.runtime.version,
|
||||
stateDir: process.env.OPENCLAW_STATE_DIR,
|
||||
runs: await read("runs.list", () => runs.list()),
|
||||
runDetails,
|
||||
resolvedRuns,
|
||||
flows: await read("flows.list", () => flows.list()),
|
||||
flowDetails,
|
||||
flowSummaries,
|
||||
managedFlow: await read("managedFlows.get", () => managed.get(TASKFLOW_FLOW_IDS[0])),
|
||||
timings,
|
||||
});
|
||||
},
|
||||
{ scope: "operator.admin" },
|
||||
);
|
||||
},
|
||||
};
|
||||
337
scripts/e2e/lib/upgrade-survivor/taskflow-restoration.mjs
Normal file
337
scripts/e2e/lib/upgrade-survivor/taskflow-restoration.mjs
Normal file
|
|
@ -0,0 +1,337 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import { createRequire } from "node:module";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { parseArgs } from "node:util";
|
||||
import {
|
||||
assertTaskflowGatewayReads,
|
||||
assertTaskflowSdkReads,
|
||||
assertTaskflowSnapshot,
|
||||
createTaskflowFixture,
|
||||
normalizeTaskflowSnapshot,
|
||||
TASKFLOW_METHOD,
|
||||
TASKFLOW_PLUGIN_MANIFEST,
|
||||
TASKFLOW_TASK_IDS,
|
||||
} from "./taskflow-restoration-fixture.mjs";
|
||||
import taskflowPlugin from "./taskflow-restoration-plugin.mjs";
|
||||
import { resolveWorkerCellExport } from "./worker-cell-package.mjs";
|
||||
|
||||
const BASELINE_COMMIT = "3a9d69db306cd7f081e06254cb89c4bcc14a7107";
|
||||
const pluginId = taskflowPlugin.id;
|
||||
const BASELINE_MODULES = {
|
||||
"task-registry.store.sqlite-CI1kWe-v.mjs":
|
||||
"d4979372e7232d63ab7a92531b2ebab94631a74b88ac902ccbe0bcf58b5253b2",
|
||||
"task-flow-registry.store.sqlite-B6eAniay.mjs":
|
||||
"e04935c9d03883dc4da22af6443279754fda4d616bc85e38e59f7c2e6d1c5b66",
|
||||
};
|
||||
const { positionals, values } = parseArgs({
|
||||
allowPositionals: true,
|
||||
options: {
|
||||
"package-root": { type: "string" },
|
||||
"expected-commit": { type: "string" },
|
||||
url: { type: "string" },
|
||||
},
|
||||
});
|
||||
const [mode] = positionals;
|
||||
assert(
|
||||
positionals.length === 1 && ["seed", "assert-state", "probe"].includes(mode),
|
||||
"Expected seed, assert-state, or probe",
|
||||
);
|
||||
assert(values["package-root"], "--package-root is required");
|
||||
const packageRoot = await fs.realpath(values["package-root"]);
|
||||
const artifacts = process.env.OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_ROOT;
|
||||
const runtimeRoot = process.env.OPENCLAW_UPGRADE_SURVIVOR_RUNTIME_ROOT;
|
||||
const stateDir = process.env.OPENCLAW_STATE_DIR;
|
||||
const configPath = process.env.OPENCLAW_CONFIG_PATH;
|
||||
assert(artifacts && runtimeRoot && stateDir && configPath, "Missing isolated survivor paths");
|
||||
assert(path.isAbsolute(stateDir) && path.isAbsolute(configPath), "State paths must be absolute");
|
||||
assert(
|
||||
path.resolve(stateDir).startsWith(`${path.resolve(runtimeRoot)}${path.sep}`),
|
||||
"Taskflow state must belong to its isolated runtime",
|
||||
);
|
||||
assert(
|
||||
path.resolve(configPath).startsWith(`${path.resolve(stateDir)}${path.sep}`),
|
||||
"Taskflow config must belong to its state root",
|
||||
);
|
||||
const expectedFile = path.join(artifacts, "taskflow-seed.json");
|
||||
const moduleEvidence = [];
|
||||
|
||||
function digest(value) {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
async function readJson(file) {
|
||||
return JSON.parse(await fs.readFile(file, "utf8"));
|
||||
}
|
||||
|
||||
async function writeJson(file, value) {
|
||||
await fs.mkdir(path.dirname(file), { recursive: true });
|
||||
await fs.writeFile(file, `${JSON.stringify(value, null, 2)}\n`);
|
||||
}
|
||||
|
||||
async function installedFile(relative) {
|
||||
const file = await fs.realpath(path.join(packageRoot, relative));
|
||||
assert(file.startsWith(`${packageRoot}${path.sep}`), "Installed module escaped package root");
|
||||
return file;
|
||||
}
|
||||
|
||||
const manifest = await readJson(await installedFile("package.json"));
|
||||
const build = await readJson(await installedFile("dist/build-info.json"));
|
||||
assert.equal(manifest.name, "openclaw");
|
||||
assert.equal(build.version, manifest.version);
|
||||
const expectedCommit = mode === "seed" ? BASELINE_COMMIT : values["expected-commit"];
|
||||
assert(/^[a-f0-9]{40}$/u.test(expectedCommit ?? ""), "Expected exact package commit");
|
||||
assert.equal(build.commit, expectedCommit, "Installed package is not the selected source");
|
||||
if (mode === "seed") {
|
||||
assert.equal(manifest.version, "2026.9.4");
|
||||
}
|
||||
|
||||
async function loadOwner(prefix, names) {
|
||||
const files = (await fs.readdir(path.join(packageRoot, "dist"))).filter(
|
||||
(name) => name.startsWith(`${prefix}-`) && name.endsWith(".mjs"),
|
||||
);
|
||||
const matches = [];
|
||||
for (const name of files) {
|
||||
const file = await installedFile(`dist/${name}`);
|
||||
const source = await fs.readFile(file, "utf8");
|
||||
const exports = names.map((symbol) => resolveWorkerCellExport(source, symbol));
|
||||
if (exports.every(Boolean)) {
|
||||
matches.push({ file, name, exports, sha256: digest(source) });
|
||||
}
|
||||
}
|
||||
assert.equal(matches.length, 1, `Expected one installed ${prefix} owner`);
|
||||
const match = matches[0];
|
||||
if (mode === "seed") {
|
||||
assert.equal(match.sha256, BASELINE_MODULES[match.name], "Published owner bytes changed");
|
||||
}
|
||||
moduleEvidence.push({ file: match.file, sha256: match.sha256, names, exports: match.exports });
|
||||
const module = await import(pathToFileURL(match.file).href);
|
||||
return Object.fromEntries(
|
||||
names.map((name, index) => {
|
||||
const operation = module[match.exports[index]];
|
||||
assert.equal(typeof operation, "function", `Missing installed owner operation ${name}`);
|
||||
return [name, operation];
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function withStores(operation) {
|
||||
const task = await loadOwner("task-registry.store.sqlite", [
|
||||
"upsertTaskWithDeliveryStateToSqlite",
|
||||
"loadTaskRegistryStateFromSqlite",
|
||||
"closeTaskRegistryDatabase",
|
||||
]);
|
||||
const flow = await loadOwner("task-flow-registry.store.sqlite", [
|
||||
"upsertTaskFlowRegistryRecordToSqlite",
|
||||
"loadTaskFlowRegistryStateFromSqlite",
|
||||
"closeTaskFlowRegistryDatabase",
|
||||
]);
|
||||
let failure;
|
||||
let result;
|
||||
try {
|
||||
result = operation(task, flow);
|
||||
} catch (error) {
|
||||
failure = error;
|
||||
}
|
||||
const errors = failure ? [failure] : [];
|
||||
for (const close of [flow.closeTaskFlowRegistryDatabase, task.closeTaskRegistryDatabase]) {
|
||||
try {
|
||||
close();
|
||||
} catch (error) {
|
||||
errors.push(error);
|
||||
}
|
||||
}
|
||||
if (errors.length) {
|
||||
throw new AggregateError(errors, "Task/flow owner operation or close failed");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function snapshot(task, flow) {
|
||||
return normalizeTaskflowSnapshot({
|
||||
...task.loadTaskRegistryStateFromSqlite(),
|
||||
...flow.loadTaskFlowRegistryStateFromSqlite(),
|
||||
});
|
||||
}
|
||||
|
||||
async function databaseMetrics() {
|
||||
const databasePath = path.join(stateDir, "state", "openclaw.sqlite");
|
||||
const sizes = {};
|
||||
for (const suffix of ["", "-wal", "-shm"]) {
|
||||
try {
|
||||
sizes[suffix || "database"] = (await fs.stat(`${databasePath}${suffix}`)).size;
|
||||
} catch (error) {
|
||||
if (error.code !== "ENOENT" || suffix === "") {
|
||||
throw error;
|
||||
}
|
||||
sizes[suffix] = 0;
|
||||
}
|
||||
}
|
||||
// Both store owners are closed. This observer never repairs or writes schema.
|
||||
const { DatabaseSync } = await import("node:sqlite");
|
||||
const db = new DatabaseSync(databasePath, { readOnly: true });
|
||||
try {
|
||||
return {
|
||||
databasePath,
|
||||
sizes,
|
||||
schemaVersion: db.prepare("PRAGMA user_version").get().user_version,
|
||||
};
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function seed() {
|
||||
const fixture = createTaskflowFixture(Date.now());
|
||||
const restored = await withStores((task, flow) => {
|
||||
for (const record of fixture.flows) {
|
||||
flow.upsertTaskFlowRegistryRecordToSqlite(record);
|
||||
}
|
||||
for (const [index, record] of fixture.tasks.entries()) {
|
||||
task.upsertTaskWithDeliveryStateToSqlite({
|
||||
task: record,
|
||||
deliveryState: fixture.deliveryStates[index],
|
||||
});
|
||||
}
|
||||
const actual = snapshot(task, flow);
|
||||
assertTaskflowSnapshot(actual, fixture);
|
||||
return actual;
|
||||
});
|
||||
const pluginRoot = path.join(runtimeRoot, "taskflow-plugin");
|
||||
await fs.mkdir(pluginRoot, { recursive: true });
|
||||
for (const file of ["taskflow-restoration-plugin.mjs", "taskflow-restoration-fixture.mjs"]) {
|
||||
await fs.copyFile(new URL(file, import.meta.url), path.join(pluginRoot, file));
|
||||
}
|
||||
await writeJson(path.join(pluginRoot, "package.json"), {
|
||||
name: "@openclaw-test/taskflow-survivor",
|
||||
version: "1.0.0",
|
||||
type: "module",
|
||||
openclaw: { extensions: ["./taskflow-restoration-plugin.mjs"] },
|
||||
});
|
||||
await writeJson(path.join(pluginRoot, "openclaw.plugin.json"), TASKFLOW_PLUGIN_MANIFEST);
|
||||
const token = process.env.GATEWAY_AUTH_TOKEN_REF;
|
||||
assert(token, "Missing synthetic Gateway token");
|
||||
await writeJson(configPath, {
|
||||
gateway: {
|
||||
mode: "local",
|
||||
bind: "loopback",
|
||||
auth: { mode: "token", token },
|
||||
controlUi: { enabled: false },
|
||||
},
|
||||
agents: {
|
||||
defaults: { workspace: path.join(runtimeRoot, "workspace"), heartbeat: { every: "0m" } },
|
||||
},
|
||||
plugins: {
|
||||
allow: [pluginId],
|
||||
load: { paths: [pluginRoot] },
|
||||
entries: { [pluginId]: { enabled: true } },
|
||||
},
|
||||
});
|
||||
await writeJson(expectedFile, {
|
||||
build,
|
||||
fixture,
|
||||
snapshot: restored,
|
||||
modules: moduleEvidence,
|
||||
sha256: digest(JSON.stringify(restored)),
|
||||
database: await databaseMetrics(),
|
||||
});
|
||||
}
|
||||
|
||||
async function assertState() {
|
||||
const expected = await readJson(expectedFile);
|
||||
const actual = await withStores(snapshot);
|
||||
await writeJson(path.join(artifacts, "taskflow-after.json"), {
|
||||
build,
|
||||
snapshot: actual,
|
||||
modules: moduleEvidence,
|
||||
sha256: digest(JSON.stringify(actual)),
|
||||
database: await databaseMetrics(),
|
||||
});
|
||||
assertTaskflowSnapshot(actual, expected.snapshot);
|
||||
}
|
||||
|
||||
async function probe() {
|
||||
assert(values.url, "--url is required");
|
||||
const url = new URL(values.url);
|
||||
assert(
|
||||
url.protocol === "ws:" && ["127.0.0.1", "localhost", "[::1]"].includes(url.hostname),
|
||||
"Expected isolated loopback Gateway",
|
||||
);
|
||||
const expected = await readJson(expectedFile);
|
||||
assert(process.env.GATEWAY_AUTH_TOKEN_REF, "Missing synthetic Gateway token");
|
||||
const require = createRequire(path.join(packageRoot, "package.json"));
|
||||
const sdkPath = await fs.realpath(require.resolve("openclaw/plugin-sdk/gateway-runtime"));
|
||||
assert(sdkPath.startsWith(`${packageRoot}${path.sep}`), "Gateway SDK escaped installed package");
|
||||
const { GatewayClient } = await import(pathToFileURL(sdkPath).href);
|
||||
let resolveHello;
|
||||
let rejectHello;
|
||||
const helloPromise = new Promise((resolve, reject) => {
|
||||
resolveHello = resolve;
|
||||
rejectHello = reject;
|
||||
});
|
||||
const client = new GatewayClient({
|
||||
url: values.url,
|
||||
token: process.env.GATEWAY_AUTH_TOKEN_REF,
|
||||
clientName: "cli",
|
||||
mode: "cli",
|
||||
role: "operator",
|
||||
scopes: ["operator.admin"],
|
||||
deviceIdentity: null,
|
||||
requestTimeoutMs: 60_000,
|
||||
onHelloOk: resolveHello,
|
||||
onConnectError: rejectHello,
|
||||
});
|
||||
const timer = setTimeout(
|
||||
() => rejectHello(new Error("Taskflow Gateway connection timed out")),
|
||||
60_000,
|
||||
);
|
||||
const started = performance.now();
|
||||
const evidence = { build, sdkPath, calls: [] };
|
||||
const output = path.join(artifacts, "taskflow-gateway.json");
|
||||
const request = async (method, params) => {
|
||||
const start = performance.now();
|
||||
try {
|
||||
const payload = await client.request(method, params);
|
||||
evidence.calls.push({ method, params, payload, elapsedMs: performance.now() - start });
|
||||
return payload;
|
||||
} finally {
|
||||
await writeJson(output, evidence);
|
||||
}
|
||||
};
|
||||
try {
|
||||
client.start();
|
||||
const hello = await helloPromise;
|
||||
// hello.auth may contain a device token. The server object carries connId/bootId/buildId.
|
||||
evidence.hello = { type: hello.type, protocol: hello.protocol, server: hello.server };
|
||||
assert.equal(typeof hello.server.connId, "string");
|
||||
clearTimeout(timer);
|
||||
const query = { limit: 2, sortBy: "updatedAt" };
|
||||
const first = await request("tasks.list", query);
|
||||
assert.equal(typeof first.nextCursor, "string", "First page lacks continuation");
|
||||
const second = await request("tasks.list", { ...query, cursor: first.nextCursor });
|
||||
const details = [];
|
||||
for (const taskId of TASKFLOW_TASK_IDS) {
|
||||
details.push(await request("tasks.get", { taskId }));
|
||||
}
|
||||
const sdk = await request(TASKFLOW_METHOD, {});
|
||||
await writeJson(output, {
|
||||
...evidence,
|
||||
elapsedMs: performance.now() - started,
|
||||
pages: [first, second],
|
||||
details,
|
||||
sdk,
|
||||
});
|
||||
assert.equal(sdk.stateDir, stateDir, "SDK reads used a different state root");
|
||||
assert.equal(sdk.runtimeVersion, manifest.version);
|
||||
assertTaskflowGatewayReads([first, second], details, expected.fixture);
|
||||
assertTaskflowSdkReads(sdk, expected.fixture);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
await client.stopAndWait();
|
||||
}
|
||||
}
|
||||
|
||||
await (mode === "seed" ? seed() : mode === "assert-state" ? assertState() : probe());
|
||||
console.log(`taskflow-restoration:${mode} passed commit=${build.commit}`);
|
||||
166
scripts/e2e/lib/upgrade-survivor/worker-cell-package.mjs
Normal file
166
scripts/e2e/lib/upgrade-survivor/worker-cell-package.mjs
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const baselineVersion = "2026.9.4";
|
||||
const baselineCommit = "3a9d69db306cd7f081e06254cb89c4bcc14a7107";
|
||||
const baselineUrl = "https://registry.npmjs.org/openclaw/-/openclaw-2026.9.4.tgz";
|
||||
const baselineIntegrity =
|
||||
"sha512-lTQpEEe1Xm3u2PCHaPEr+vP8paGk1vLdHuzdItsNToaLI6hAqRVvgJYg+GxukJhETJp4tPy/S1Gftl4KuB8n7A==";
|
||||
|
||||
function hash(bytes, algorithm = "sha256", encoding = "hex") {
|
||||
return createHash(algorithm).update(bytes).digest(encoding);
|
||||
}
|
||||
|
||||
function readJson(file) {
|
||||
return JSON.parse(fs.readFileSync(file, "utf8"));
|
||||
}
|
||||
|
||||
function writeJson(file, value) {
|
||||
fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`);
|
||||
}
|
||||
|
||||
// npm owns dependency reification. Compare the immutable application payload,
|
||||
// including its complete dist inventory, separately from installed node_modules.
|
||||
export function readWorkerCellPackageIdentity(packageRoot) {
|
||||
const files = {};
|
||||
const visit = (relative) => {
|
||||
const file = path.join(packageRoot, relative);
|
||||
const stat = fs.lstatSync(file);
|
||||
if (stat.isSymbolicLink()) {
|
||||
files[relative] = { symlink: fs.readlinkSync(file) };
|
||||
} else if (stat.isDirectory()) {
|
||||
for (const name of fs.readdirSync(file).toSorted((a, b) => a.localeCompare(b))) {
|
||||
visit(path.posix.join(relative, name));
|
||||
}
|
||||
} else {
|
||||
assert(stat.isFile(), `Unsupported package entry: ${relative}`);
|
||||
files[relative] = { sha256: hash(fs.readFileSync(file)), size: stat.size };
|
||||
}
|
||||
};
|
||||
for (const relative of ["package.json", "openclaw.mjs", "dist"]) {
|
||||
visit(relative);
|
||||
}
|
||||
const manifest = readJson(path.join(packageRoot, "package.json"));
|
||||
assert.equal(manifest.name, "openclaw");
|
||||
const buildInfo = readJson(path.join(packageRoot, "dist/build-info.json"));
|
||||
assert.equal(buildInfo.version, manifest.version);
|
||||
assert.match(buildInfo.commit, /^[a-f0-9]{40}$/u);
|
||||
return { version: manifest.version, buildInfo, files };
|
||||
}
|
||||
|
||||
export function assertWorkerCellPackageIdentity(actual, expected) {
|
||||
assert.deepEqual(
|
||||
actual,
|
||||
expected,
|
||||
"Installed application payload differs from the frozen tarball",
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveWorkerCellExport(source, name) {
|
||||
const matches = [];
|
||||
for (const block of source.matchAll(/export\s*\{([^}]+)\}\s*;/gu)) {
|
||||
for (const specifier of block[1].split(",")) {
|
||||
const parts = specifier.trim().split(/\s+as\s+/u);
|
||||
if (parts[0] === name && parts.length <= 2) {
|
||||
matches.push(parts[1] ?? parts[0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert(matches.length <= 1, `Ambiguous compiled export ${name}`);
|
||||
return matches[0];
|
||||
}
|
||||
|
||||
function inspectTarball(tarball, runtimeRoot) {
|
||||
const bytes = fs.readFileSync(tarball);
|
||||
const sha256 = hash(bytes);
|
||||
const integrity = `sha512-${hash(bytes, "sha512", "base64")}`;
|
||||
const scratch = fs.mkdtempSync(path.join(runtimeRoot, "package-identity-"));
|
||||
try {
|
||||
execFileSync(
|
||||
"tar",
|
||||
[
|
||||
"-xzf",
|
||||
tarball,
|
||||
"-C",
|
||||
scratch,
|
||||
"package/package.json",
|
||||
"package/openclaw.mjs",
|
||||
"package/dist",
|
||||
],
|
||||
{ stdio: ["ignore", "pipe", "pipe"] },
|
||||
);
|
||||
return { sha256, integrity, ...readWorkerCellPackageIdentity(path.join(scratch, "package")) };
|
||||
} finally {
|
||||
fs.rmSync(scratch, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const [mode, packageRoot, candidateTarball] = process.argv.slice(2);
|
||||
const artifacts = process.env.OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_ROOT;
|
||||
const runtimeRoot = process.env.OPENCLAW_UPGRADE_SURVIVOR_RUNTIME_ROOT;
|
||||
assert(artifacts && runtimeRoot && packageRoot, "Missing isolated worker-cell paths");
|
||||
if (mode === "baseline") {
|
||||
const response = await fetch(baselineUrl);
|
||||
assert(response.ok, `Published baseline download failed: ${response.status}`);
|
||||
const bytes = Buffer.from(await response.arrayBuffer());
|
||||
assert.equal(`sha512-${hash(bytes, "sha512", "base64")}`, baselineIntegrity);
|
||||
const tarball = path.join(runtimeRoot, "published-driver.tgz");
|
||||
fs.writeFileSync(tarball, bytes, { flag: "wx" });
|
||||
const expected = inspectTarball(tarball, runtimeRoot);
|
||||
assert.equal(expected.version, baselineVersion);
|
||||
assert.equal(expected.buildInfo.commit, baselineCommit);
|
||||
const actual = readWorkerCellPackageIdentity(packageRoot);
|
||||
assertWorkerCellPackageIdentity(actual, {
|
||||
version: expected.version,
|
||||
buildInfo: expected.buildInfo,
|
||||
files: expected.files,
|
||||
});
|
||||
writeJson(path.join(artifacts, "baseline-package-identity.json"), {
|
||||
url: baselineUrl,
|
||||
cli: fs.realpathSync(path.join(packageRoot, "openclaw.mjs")),
|
||||
...expected,
|
||||
});
|
||||
} else if (mode === "candidate") {
|
||||
assert(candidateTarball, "Missing frozen candidate tarball");
|
||||
const expected = inspectTarball(candidateTarball, runtimeRoot);
|
||||
assert.equal(
|
||||
expected.buildInfo.commit,
|
||||
process.env.OPENCLAW_DOCKER_E2E_SELECTED_SHA,
|
||||
"Candidate build commit must equal the selected source SHA",
|
||||
);
|
||||
assert.notEqual(
|
||||
expected.buildInfo.commit,
|
||||
baselineCommit,
|
||||
"Candidate still contains published bytes",
|
||||
);
|
||||
writeJson(path.join(artifacts, "candidate-package-identity.json"), expected);
|
||||
} else if (mode === "installed") {
|
||||
const expected = readJson(path.join(artifacts, "candidate-package-identity.json"));
|
||||
assert.equal(
|
||||
hash(fs.readFileSync(candidateTarball)),
|
||||
expected.sha256,
|
||||
"Candidate tarball changed",
|
||||
);
|
||||
const actual = readWorkerCellPackageIdentity(packageRoot);
|
||||
assertWorkerCellPackageIdentity(actual, {
|
||||
version: expected.version,
|
||||
buildInfo: expected.buildInfo,
|
||||
files: expected.files,
|
||||
});
|
||||
writeJson(path.join(artifacts, "installed-package-identity.json"), {
|
||||
cli: fs.realpathSync(path.join(packageRoot, "openclaw.mjs")),
|
||||
...actual,
|
||||
});
|
||||
} else {
|
||||
throw new Error("Expected baseline, candidate, or installed package-identity mode");
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
await main();
|
||||
}
|
||||
|
|
@ -11,6 +11,7 @@ set -euo pipefail
|
|||
PACKAGE_TGZ=""
|
||||
AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT=""
|
||||
UPGRADE_SCENARIO_STAGE=""
|
||||
WORKER_RUNTIME_HOST_ROOT=""
|
||||
run_completed="0"
|
||||
diagnostics_ready=0
|
||||
cleanup_outer() {
|
||||
|
|
@ -44,6 +45,21 @@ cleanup_outer() {
|
|||
if [ -n "$UPGRADE_SCENARIO_STAGE" ]; then
|
||||
rm -rf "$UPGRADE_SCENARIO_STAGE"
|
||||
fi
|
||||
if [ -n "$WORKER_RUNTIME_HOST_ROOT" ]; then
|
||||
if [ "$exit_status" -eq 0 ] && [ "$run_completed" = "1" ]; then
|
||||
# The image user owns the private child and can differ from the host user.
|
||||
if ! docker_e2e_docker_cmd run --rm --network none \
|
||||
--entrypoint rm \
|
||||
-v "$WORKER_RUNTIME_HOST_ROOT:/tmp/openclaw-worker-cleanup" \
|
||||
"$IMAGE_NAME" -rf -- /tmp/openclaw-worker-cleanup/runtime ||
|
||||
! rm -rf "$WORKER_RUNTIME_HOST_ROOT"; then
|
||||
echo "Worker-cell runtime cleanup failed: $WORKER_RUNTIME_HOST_ROOT" >&2
|
||||
exit_status=1
|
||||
fi
|
||||
else
|
||||
echo "Preserved failed synthetic worker-cell state: $WORKER_RUNTIME_HOST_ROOT" >&2
|
||||
fi
|
||||
fi
|
||||
if [ "$exit_status" -ne 0 ]; then
|
||||
printf '[upgrade-survivor] FAILED (exit %s)\n' "$exit_status" >&2
|
||||
fi
|
||||
|
|
@ -141,6 +157,7 @@ SKIP_BUILD="${OPENCLAW_UPGRADE_SURVIVOR_E2E_SKIP_BUILD:-0}"
|
|||
DOCKER_RUN_TIMEOUT="${OPENCLAW_UPGRADE_SURVIVOR_DOCKER_RUN_TIMEOUT:-1200s}"
|
||||
BASELINE_SPEC="${OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC:-}"
|
||||
SCENARIO="${OPENCLAW_UPGRADE_SURVIVOR_SCENARIO:-base}"
|
||||
SURVIVOR_RUNTIME_ROOT="${OPENCLAW_UPGRADE_SURVIVOR_RUNTIME_ROOT:-/tmp/openclaw-upgrade-survivor-runtime}"
|
||||
if [ "$OPENCLAW_FROZEN_UPGRADE_SURVIVOR_CLAWHUB_MODE" = legacy ]; then
|
||||
legacy_clawhub_package="@openclaw/whatsapp"
|
||||
[ "$SCENARIO" = configured-plugin-installs ] && legacy_clawhub_package="@openclaw/matrix"
|
||||
|
|
@ -218,6 +235,15 @@ if [ "$SCENARIO" = "abandoned-update" ] && {
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$SCENARIO" = "projects-doctor" ] || [ "$SCENARIO" = "taskflow-restoration" ]; then
|
||||
if [ "${OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE:-0}" != "1" ] ||
|
||||
[ "$BASELINE_SPEC" != "openclaw@2026.9.4" ] ||
|
||||
[ "$UPDATE_RESTART_MODE" != "manual" ] || [ "$ROOT_MANAGED_VPS" != "0" ] || [ "$LIVE_OPENAI" != "0" ]; then
|
||||
echo "$SCENARIO requires published openclaw@2026.9.4, manual restart, isolated state, and no live provider" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$SCENARIO" = "workshop-doctor-recovery" ] && {
|
||||
[ "${OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE:-0}" != "1" ] ||
|
||||
[ "$UPDATE_RESTART_MODE" != "manual" ] || [ "$ROOT_MANAGED_VPS" != "0" ] || [ "$LIVE_OPENAI" != "0" ];
|
||||
|
|
@ -303,6 +329,10 @@ if [ "${OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE:-0}" = "1" ]; then
|
|||
fi
|
||||
|
||||
mkdir -p "$ARTIFACT_DIR"
|
||||
if [ "$SCENARIO" = "projects-doctor" ] || [ "$SCENARIO" = "taskflow-restoration" ]; then
|
||||
ARTIFACT_DIR="$(mktemp -d "$ARTIFACT_DIR/worker-run.XXXXXX")"
|
||||
echo "Worker survivor artifacts: $ARTIFACT_DIR"
|
||||
fi
|
||||
chmod -R a+rwX "$ARTIFACT_DIR" || true
|
||||
prepare_diagnostics_capture
|
||||
|
||||
|
|
@ -335,6 +365,11 @@ if [ "${OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE:-0}" = "1" ]; then
|
|||
CANDIDATE_SPEC="$(normalize_npm_candidate "$CANDIDATE_RAW")"
|
||||
fi
|
||||
|
||||
if { [ "$SCENARIO" = "projects-doctor" ] || [ "$SCENARIO" = "taskflow-restoration" ]; } && [ "$CANDIDATE_KIND" != "tarball" ]; then
|
||||
echo "$SCENARIO requires a frozen candidate tarball" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$CANDIDATE_IS_CURRENT" = "1" ] && [ -z "${OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR:-}" ]; then
|
||||
registry_required="$(
|
||||
OPENCLAW_DOCKER_ALL_LANES=published-upgrade-survivor \
|
||||
|
|
@ -371,6 +406,16 @@ if [ "${OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE:-0}" = "1" ]; then
|
|||
|
||||
OPENCLAW_TEST_STATE_FUNCTION_B64="$(docker_e2e_test_state_function_b64)"
|
||||
|
||||
if [ "$SCENARIO" = "projects-doctor" ] || [ "$SCENARIO" = "taskflow-restoration" ]; then
|
||||
WORKER_RUNTIME_HOST_ROOT="$(mktemp -d "$ARTIFACT_DIR/worker-runtime.XXXXXX")"
|
||||
chmod a+rwx "$WORKER_RUNTIME_HOST_ROOT"
|
||||
UPGRADE_SCENARIO_ARGS+=(
|
||||
-v "$WORKER_RUNTIME_HOST_ROOT:$SURVIVOR_RUNTIME_ROOT"
|
||||
)
|
||||
# The container user creates/owns the private child, not the host mount point.
|
||||
SURVIVOR_RUNTIME_ROOT="$SURVIVOR_RUNTIME_ROOT/runtime"
|
||||
fi
|
||||
|
||||
docker_e2e_build_or_reuse "$IMAGE_NAME" upgrade-survivor "$ROOT_DIR/scripts/e2e/Dockerfile" "$ROOT_DIR" "bare" "$SKIP_BUILD"
|
||||
|
||||
echo "Running published upgrade survivor Docker E2E..."
|
||||
|
|
@ -383,7 +428,7 @@ if [ "${OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE:-0}" = "1" ]; then
|
|||
-e OPENCLAW_UPGRADE_SURVIVOR_CANDIDATE_SPEC="$CANDIDATE_SPEC" \
|
||||
-e OPENCLAW_DOCKER_E2E_SELECTED_SHA="${OPENCLAW_DOCKER_E2E_SELECTED_SHA:-}" \
|
||||
-e OPENCLAW_UPGRADE_SURVIVOR_SCENARIO="$SCENARIO" \
|
||||
-e OPENCLAW_UPGRADE_SURVIVOR_RUNTIME_ROOT="${OPENCLAW_UPGRADE_SURVIVOR_RUNTIME_ROOT:-/tmp/openclaw-upgrade-survivor-runtime}" \
|
||||
-e OPENCLAW_UPGRADE_SURVIVOR_RUNTIME_ROOT="$SURVIVOR_RUNTIME_ROOT" \
|
||||
-e OPENCLAW_UPGRADE_SURVIVOR_UPDATE_RESTART_MODE="$UPDATE_RESTART_MODE" \
|
||||
-e OPENCLAW_UPGRADE_SURVIVOR_COMMAND_TIMEOUT="$COMMAND_TIMEOUT" \
|
||||
-e OPENCLAW_UPGRADE_SURVIVOR_VOLUME_SESSIONS="${OPENCLAW_UPGRADE_SURVIVOR_VOLUME_SESSIONS:-}" \
|
||||
|
|
|
|||
|
|
@ -134,7 +134,7 @@ child.on("error", (error) => {
|
|||
process.exit(127);
|
||||
});
|
||||
' "$timeout_value" "$@"
|
||||
return
|
||||
return "$?"
|
||||
fi
|
||||
echo "timeout command not found; cannot bound Docker command after ${timeout_value}" >&2
|
||||
return 127
|
||||
|
|
@ -152,7 +152,8 @@ docker_e2e_docker_cmd() {
|
|||
shift
|
||||
docker_e2e_docker_run_resource_args "$@" || return $?
|
||||
docker_e2e_docker_run_with_resource_diagnostics "$timeout_value" "$@"
|
||||
return
|
||||
# A bare return in an EXIT trap can restore the trap's original status.
|
||||
return "$?"
|
||||
fi
|
||||
docker_e2e_timeout_cmd "$timeout_value" docker "$@"
|
||||
}
|
||||
|
|
@ -168,7 +169,7 @@ docker_e2e_docker_run_cmd() {
|
|||
shift
|
||||
docker_e2e_docker_run_resource_args "$@" || return $?
|
||||
docker_e2e_docker_run_with_resource_diagnostics "$timeout_value" "$@"
|
||||
return
|
||||
return "$?"
|
||||
fi
|
||||
docker_e2e_timeout_cmd "$timeout_value" docker "$@"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -133,6 +133,14 @@ const UPGRADE_SURVIVOR_RUNTIME_COMPANION_PACKAGES = ["@openclaw/codex"];
|
|||
// Pre-protocol catalogs are content-addressed. Unknown legacy blocks fail
|
||||
// closed instead of requiring a dependency or reimplementing a JavaScript parser.
|
||||
const LEGACY_UPGRADE_SURVIVOR_SCENARIO_CATALOGS = new Map([
|
||||
[
|
||||
"f2549a057028829ff5286db89d357e5b3d4ec1f5cdb3ca07672b7e34a739b60a",
|
||||
"base msteams-polls abandoned-update legacy-operator-state workshop-doctor-recovery mobile-pairing-reconnect acpx-openclaw-tools-bridge feishu-channel bootstrap-persona channel-post-core-restore codex-allowlist-survival plugin-deps-cleanup configured-plugin-installs missing-configured-plugin-migration custom-plugin-siblings projects-doctor taskflow-restoration stale-source-plugin-shadow prerelease-plugin-registry tilde-log-path meeting-transcripts-sqlite versioned-runtime-deps cron-scheduled-authority sqlite-volume recovery-cleanup auth-profile-v2026-7-2-beta-5 watchos-direct-node",
|
||||
],
|
||||
[
|
||||
"b0166f96bf3839d53ce94721b563fcf2b6604ddef04028cd8d9c7769275566c7",
|
||||
"base msteams-polls abandoned-update legacy-operator-state mobile-pairing-reconnect acpx-openclaw-tools-bridge feishu-channel bootstrap-persona channel-post-core-restore codex-allowlist-survival plugin-deps-cleanup configured-plugin-installs missing-configured-plugin-migration custom-plugin-siblings projects-doctor taskflow-restoration stale-source-plugin-shadow prerelease-plugin-registry tilde-log-path meeting-transcripts-sqlite versioned-runtime-deps cron-scheduled-authority sqlite-volume recovery-cleanup auth-profile-v2026-7-2-beta-5 watchos-direct-node",
|
||||
],
|
||||
[
|
||||
"7d9d7520c2c34d51fff78e542a7f539b77080bfd04648438e95aec4af3fe362e",
|
||||
"base msteams-polls abandoned-update legacy-operator-state workshop-doctor-recovery mobile-pairing-reconnect acpx-openclaw-tools-bridge feishu-channel bootstrap-persona channel-post-core-restore codex-allowlist-survival plugin-deps-cleanup configured-plugin-installs missing-configured-plugin-migration custom-plugin-siblings stale-source-plugin-shadow prerelease-plugin-registry tilde-log-path meeting-transcripts-sqlite versioned-runtime-deps cron-scheduled-authority sqlite-volume recovery-cleanup auth-profile-v2026-7-2-beta-5 watchos-direct-node",
|
||||
|
|
@ -690,6 +698,8 @@ export function requiredPrepublishPluginPackagesForLanes(poolLanes: DockerE2eLan
|
|||
!scenario ||
|
||||
scenario === "abandoned-update" ||
|
||||
scenario === "custom-plugin-siblings" ||
|
||||
scenario === "projects-doctor" ||
|
||||
scenario === "taskflow-restoration" ||
|
||||
scenario === "workshop-doctor-recovery"
|
||||
) {
|
||||
continue;
|
||||
|
|
|
|||
|
|
@ -27,11 +27,11 @@ docker_e2e_resource_limit_temp_dir() {
|
|||
local template="${TMPDIR:-/tmp}/openclaw-docker-resource-limits.XXXXXX"
|
||||
if command -v mktemp >/dev/null 2>&1; then
|
||||
mktemp -d "$template"
|
||||
return
|
||||
return "$?"
|
||||
fi
|
||||
if [ -x /usr/bin/mktemp ]; then
|
||||
/usr/bin/mktemp -d "$template"
|
||||
return
|
||||
return "$?"
|
||||
fi
|
||||
echo "mktemp command not found; cannot create Docker resource-limit diagnostics" >&2
|
||||
return 127
|
||||
|
|
@ -40,11 +40,11 @@ docker_e2e_resource_limit_temp_dir() {
|
|||
docker_e2e_diagnostic_bin() {
|
||||
if command -v "$1" >/dev/null 2>&1; then
|
||||
command -v "$1"
|
||||
return
|
||||
return "$?"
|
||||
fi
|
||||
if [ -x "/usr/bin/$1" ]; then
|
||||
printf '%s\n' "/usr/bin/$1"
|
||||
return
|
||||
return "$?"
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
|
@ -52,7 +52,7 @@ docker_e2e_diagnostic_bin() {
|
|||
docker_e2e_remove_diagnostic_dir() {
|
||||
if command -v rm >/dev/null 2>&1; then
|
||||
rm -rf "$1"
|
||||
return
|
||||
return "$?"
|
||||
fi
|
||||
/bin/rm -rf "$1"
|
||||
}
|
||||
|
|
@ -66,7 +66,7 @@ docker_e2e_docker_run_with_resource_diagnostics() {
|
|||
shift
|
||||
if [ "${#DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" -eq 0 ]; then
|
||||
docker_e2e_timeout_cmd "$timeout_value" docker run "$@"
|
||||
return
|
||||
return "$?"
|
||||
fi
|
||||
|
||||
local diagnostic_dir=""
|
||||
|
|
@ -74,7 +74,7 @@ docker_e2e_docker_run_with_resource_diagnostics() {
|
|||
docker_e2e_timeout_cmd \
|
||||
"$timeout_value" \
|
||||
docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
|
||||
return
|
||||
return "$?"
|
||||
fi
|
||||
local tee_bin=""
|
||||
if ! tee_bin="$(docker_e2e_diagnostic_bin tee)"; then
|
||||
|
|
@ -82,7 +82,7 @@ docker_e2e_docker_run_with_resource_diagnostics() {
|
|||
docker_e2e_timeout_cmd \
|
||||
"$timeout_value" \
|
||||
docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
|
||||
return
|
||||
return "$?"
|
||||
fi
|
||||
local tail_bin=""
|
||||
if ! tail_bin="$(docker_e2e_diagnostic_bin tail)"; then
|
||||
|
|
@ -90,7 +90,7 @@ docker_e2e_docker_run_with_resource_diagnostics() {
|
|||
docker_e2e_timeout_cmd \
|
||||
"$timeout_value" \
|
||||
docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
|
||||
return
|
||||
return "$?"
|
||||
fi
|
||||
local stderr_file="${diagnostic_dir}/stderr"
|
||||
local stderr_fifo="${diagnostic_dir}/stderr.pipe"
|
||||
|
|
@ -102,7 +102,7 @@ docker_e2e_docker_run_with_resource_diagnostics() {
|
|||
docker_e2e_timeout_cmd \
|
||||
"$timeout_value" \
|
||||
docker run "${DOCKER_E2E_RUN_RESOURCE_ARGS[@]}" "$@"
|
||||
return
|
||||
return "$?"
|
||||
fi
|
||||
|
||||
# Some tail implementations reopen named FIFOs passed through stdin and wait for a new writer.
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@ const UPGRADE_SURVIVOR_SCENARIOS = Object.freeze([
|
|||
"configured-plugin-installs",
|
||||
"missing-configured-plugin-migration",
|
||||
"custom-plugin-siblings",
|
||||
"projects-doctor",
|
||||
"taskflow-restoration",
|
||||
"stale-source-plugin-shadow",
|
||||
"prerelease-plugin-registry",
|
||||
"tilde-log-path",
|
||||
|
|
@ -43,6 +45,8 @@ const scenarioMinimumBaselines = new Map([
|
|||
const TRUSTED_HARNESS_OWNED_SCENARIOS = new Set([
|
||||
"mobile-pairing-reconnect",
|
||||
"abandoned-update",
|
||||
"projects-doctor",
|
||||
"taskflow-restoration",
|
||||
"workshop-doctor-recovery",
|
||||
]);
|
||||
|
||||
|
|
@ -60,6 +64,8 @@ const aggregateScenarios = UPGRADE_SURVIVOR_SCENARIOS.filter(
|
|||
scenario !== "msteams-polls" &&
|
||||
scenario !== "abandoned-update" &&
|
||||
scenario !== "missing-configured-plugin-migration" &&
|
||||
scenario !== "projects-doctor" &&
|
||||
scenario !== "taskflow-restoration" &&
|
||||
scenario !== "workshop-doctor-recovery" &&
|
||||
scenario !== "mobile-pairing-reconnect" &&
|
||||
scenario !== "watchos-direct-node" &&
|
||||
|
|
@ -156,6 +162,9 @@ function comparePublishedReleaseVersion(a, b) {
|
|||
|
||||
export function supportsUpgradeSurvivorScenarioAtBaseline(scenario, baselineSpec) {
|
||||
const version = parsePublishedReleaseVersion(baselineSpec);
|
||||
if (scenario === "projects-doctor" || scenario === "taskflow-restoration") {
|
||||
return baselineSpec === "openclaw@2026.9.4";
|
||||
}
|
||||
if (scenario === "abandoned-update" || scenario === "missing-configured-plugin-migration") {
|
||||
return baselineSpec === "openclaw@2026.9.2";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2526,6 +2526,62 @@ stderr="$(<"$TMPDIR/stderr")"
|
|||
execDockerSnippet(script);
|
||||
});
|
||||
|
||||
describe.each(process.platform === "darwin" ? ["/bin/bash", "bash"] : ["/bin/bash"])(
|
||||
"%s EXIT-trap Docker status",
|
||||
(shell) => {
|
||||
const cases = ["docker_e2e_docker_cmd", "docker_e2e_docker_run_cmd"].flatMap((helper) =>
|
||||
["normal", "no-diagnostics", "node-watchdog"].flatMap((mode) =>
|
||||
[
|
||||
{ entryStatus: 0, commandStatus: 43 },
|
||||
{ entryStatus: 42, commandStatus: 0 },
|
||||
].map(({ entryStatus, commandStatus }) => ({ helper, mode, entryStatus, commandStatus })),
|
||||
),
|
||||
);
|
||||
it.each(cases)(
|
||||
"preserves $helper status $commandStatus in $mode after exit $entryStatus",
|
||||
({ helper, mode, entryStatus, commandStatus }) => {
|
||||
const workDir = tempDirs.make("docker-exit-status-");
|
||||
writeExecutables(join(workDir, "bin"), {
|
||||
timeout: PASSTHROUGH_TIMEOUT_SCRIPT,
|
||||
node: `#!/bin/bash\nexec ${shellQuote(testNodeExecPath)} "$@"\n`,
|
||||
docker: `#!/bin/bash\nexit ${commandStatus}\n`,
|
||||
});
|
||||
const setup =
|
||||
mode === "no-diagnostics"
|
||||
? "docker_e2e_resource_limit_temp_dir() { return 1; }"
|
||||
: mode === "node-watchdog"
|
||||
? "docker_e2e_timeout_bin() { return 1; }"
|
||||
: "";
|
||||
const script = repoShell(workDir)`
|
||||
export PATH="$TMPDIR/bin:$PATH"
|
||||
export OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS=1
|
||||
source "$ROOT_DIR/scripts/lib/docker-e2e-container.sh"
|
||||
${setup}
|
||||
on_exit() {
|
||||
trap - EXIT
|
||||
set +e
|
||||
if ${helper} run demo; then
|
||||
observed=0
|
||||
else
|
||||
observed="$?"
|
||||
fi
|
||||
printf '%s\\n' "$observed"
|
||||
exit 0
|
||||
}
|
||||
trap on_exit EXIT
|
||||
exit ${entryStatus}
|
||||
`;
|
||||
const result = spawnSync(shell, ["--noprofile", "--norc", "-c", script], {
|
||||
encoding: "utf8",
|
||||
env: { ...process.env, BASH_ENV: "", ENV: "" },
|
||||
});
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
expect(result.stdout).toBe(`${commandStatus}\n`);
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("uses a Node watchdog for Docker commands when timeout is unavailable", () => {
|
||||
const workDir = tempDirs.make("openclaw-docker-node-timeout-");
|
||||
writeExecutables(join(workDir, "bin"), {
|
||||
|
|
@ -3041,7 +3097,7 @@ docker_e2e_docker_run_cmd run demo
|
|||
);
|
||||
expect(publishedRunner).toContain(
|
||||
[
|
||||
'if [ "$SCENARIO" = "watchos-direct-node" ] || [ "$SCENARIO" = "mobile-pairing-reconnect" ]; then',
|
||||
'if [ "$SCENARIO" = "watchos-direct-node" ] || [ "$SCENARIO" = "mobile-pairing-reconnect" ] || [ "$WORKER_CELL" = "1" ]; then',
|
||||
" unset OPENAI_API_KEY DISCORD_BOT_TOKEN TELEGRAM_BOT_TOKEN",
|
||||
"else",
|
||||
' export OPENAI_API_KEY="sk-openclaw-upgrade-survivor"',
|
||||
|
|
|
|||
|
|
@ -1456,6 +1456,32 @@ await import('./scripts/check-docker-e2e-boundaries.mts');`,
|
|||
).toContain(name);
|
||||
});
|
||||
|
||||
it.each(["projects-doctor", "taskflow-restoration"])(
|
||||
"plans %s only for its exact published writer without registry or credential fixtures",
|
||||
(scenario) => {
|
||||
const plan = planFor({
|
||||
selectedLaneNames: ["published-upgrade-survivor"],
|
||||
upgradeSurvivorBaselines: "2026.9.3 2026.9.4 2026.9.5 latest",
|
||||
upgradeSurvivorScenarios: scenario,
|
||||
});
|
||||
const name = `published-upgrade-survivor-2026.9.4-${scenario}`;
|
||||
expect(plan.lanes.map(summarizeLane)).toEqual([
|
||||
publishedUpgradeSurvivorLane(name, "openclaw@2026.9.4", scenario),
|
||||
]);
|
||||
expect(plan.requiredPrepublishPluginPackages).toEqual([]);
|
||||
expect(plan.credentials).toEqual([]);
|
||||
for (const alias of ["reported-issues", "far-reaching"]) {
|
||||
expect(
|
||||
planFor({
|
||||
selectedLaneNames: ["published-upgrade-survivor"],
|
||||
upgradeSurvivorBaselines: "2026.9.4",
|
||||
upgradeSurvivorScenarios: alias,
|
||||
}).lanes.map((lane) => lane.name),
|
||||
).not.toContain(name);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps platform survivors out of release aliases", () => {
|
||||
const scenariosFor = (
|
||||
upgradeSurvivorScenarios: string,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,13 @@
|
|||
import { spawnSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import {
|
||||
chmodSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
readFileSync,
|
||||
readdirSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
|
||||
|
|
@ -64,6 +71,19 @@ printf '{"dir":"%s"}\n' "$OPENCLAW_DOCKER_ALL_LOG_DIR/prepublish-plugin-registry
|
|||
set -euo pipefail
|
||||
printf '%s\n' "$*" >>"$CAPTURE_DIR/docker-args"
|
||||
if [ "\${1:-}" = run ]; then
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
*:/tmp/openclaw-worker-cleanup)
|
||||
printf '%s\\0' "$@" >"$CAPTURE_DIR/docker-cleanup-args"
|
||||
test -f "$CAPTURE_DIR/main-run-finished"
|
||||
if [ "\${FIXTURE_CLEANUP_EXIT:-0}" != 0 ]; then
|
||||
exit "$FIXTURE_CLEANUP_EXIT"
|
||||
fi
|
||||
rm -rf "\${arg%%:*}/runtime"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
printf '%s\\0' "$@" >"$CAPTURE_DIR/docker-run-args"
|
||||
if [ -n "\${FIXTURE_PAYLOAD_SHELL:-}" ]; then
|
||||
exec "$FIXTURE_PAYLOAD_SHELL" -c "\${!#}"
|
||||
|
|
@ -71,12 +91,21 @@ if [ "\${1:-}" = run ]; then
|
|||
fi
|
||||
previous=""
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
*/worker-runtime.*:*)
|
||||
mkdir -p "\${arg%%:*}/runtime"
|
||||
printf 'synthetic state' >"\${arg%%:*}/runtime/state-marker"
|
||||
;;
|
||||
esac
|
||||
if [ "$previous" = "--cidfile" ]; then
|
||||
printf 'fake-container\n' >"$arg"
|
||||
fi
|
||||
previous="$arg"
|
||||
done
|
||||
[ "\${1:-}" != run ] || exit "\${FIXTURE_RUN_EXIT:-0}"
|
||||
if [ "\${1:-}" = run ]; then
|
||||
touch "$CAPTURE_DIR/main-run-finished"
|
||||
exit "\${FIXTURE_RUN_EXIT:-0}"
|
||||
fi
|
||||
`,
|
||||
);
|
||||
|
||||
|
|
@ -108,6 +137,41 @@ done
|
|||
}
|
||||
|
||||
describe("standalone upgrade survivor plugin registry", () => {
|
||||
it("keeps synthetic state through inner finalization until the Docker owner joins", () => {
|
||||
const root = tempDirs.make("worker-cell-inner-finalization-");
|
||||
const runtime = join(root, "runtime");
|
||||
mkdirSync(runtime);
|
||||
const marker = join(runtime, "state-marker");
|
||||
writeFileSync(marker, "synthetic state");
|
||||
const source = readFileSync("scripts/e2e/lib/upgrade-survivor/run.sh", "utf8");
|
||||
const firstPhase = source.indexOf("phase storage-preflight");
|
||||
expect(firstPhase).toBeGreaterThan(0);
|
||||
const runner = join(root, "inner.sh");
|
||||
writeFileSync(
|
||||
runner,
|
||||
`${source.slice(0, firstPhase)}
|
||||
cleanup() { :; }
|
||||
write_summary() { :; }
|
||||
run_completed=1
|
||||
on_exit 0
|
||||
`,
|
||||
);
|
||||
const result = spawnSync("bash", [runner], {
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
OPENCLAW_UPGRADE_SURVIVOR_BASELINE: "openclaw@2026.9.4",
|
||||
OPENCLAW_UPGRADE_SURVIVOR_SCENARIO: "projects-doctor",
|
||||
OPENCLAW_UPGRADE_SURVIVOR_LIVE_OPENAI: "0",
|
||||
OPENCLAW_UPGRADE_SURVIVOR_RUNTIME_ROOT: runtime,
|
||||
OPENCLAW_UPGRADE_SURVIVOR_SUMMARY_JSON: join(root, "artifacts", "summary.json"),
|
||||
},
|
||||
timeout: 30_000,
|
||||
});
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
expect(readFileSync(marker, "utf8")).toBe("synthetic state");
|
||||
});
|
||||
|
||||
// macOS /bin/bash is 3.2; PATH may select a newer Bash. Exercise both owners.
|
||||
describe.each(process.platform === "darwin" ? ["/bin/bash", "bash"] : ["bash"])(
|
||||
"%s wrapper",
|
||||
|
|
@ -273,6 +337,85 @@ describe("standalone upgrade survivor plugin registry", () => {
|
|||
);
|
||||
expect(existsSync(packageTarball)).toBe(true);
|
||||
});
|
||||
|
||||
it.each(["projects-doctor", "taskflow-restoration"])(
|
||||
"isolates each %s run from retained evidence and preserves a failed runtime",
|
||||
(scenario) => {
|
||||
const artifacts = tempDirs.make("worker-cell-retained-artifacts-");
|
||||
const retained = join(artifacts, "projects-inventory.json");
|
||||
writeFileSync(retained, "previous evidence");
|
||||
const directories = [];
|
||||
for (const exitCode of ["0", "42"]) {
|
||||
const { captureDir, result } = runSurvivor({
|
||||
OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_DIR: artifacts,
|
||||
OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC: "openclaw@2026.9.4",
|
||||
OPENCLAW_UPGRADE_SURVIVOR_SCENARIO: scenario,
|
||||
OPENCLAW_UPGRADE_SURVIVOR_E2E_IMAGE: "worker-cleanup-fixture",
|
||||
FIXTURE_RUN_EXIT: exitCode,
|
||||
});
|
||||
expect(result.status, result.stderr).toBe(Number(exitCode));
|
||||
expect(existsSync(join(captureDir, "node-args"))).toBe(false);
|
||||
const directory = result.stdout.match(/Worker survivor artifacts: ([^\n]+)/u)?.[1] ?? "";
|
||||
expect(existsSync(directory)).toBe(true);
|
||||
directories.push(directory);
|
||||
const runtimes = readdirSync(directory).filter((name) =>
|
||||
name.startsWith("worker-runtime."),
|
||||
);
|
||||
expect(runtimes).toHaveLength(exitCode === "0" ? 0 : 1);
|
||||
const cleanupArgsPath = join(captureDir, "docker-cleanup-args");
|
||||
expect(existsSync(cleanupArgsPath)).toBe(exitCode === "0");
|
||||
if (exitCode === "0") {
|
||||
const args = readFileSync(cleanupArgsPath, "utf8").split("\0").slice(0, -1);
|
||||
expect(args[args.indexOf("--network") + 1]).toBe("none");
|
||||
expect(args[args.indexOf("--entrypoint") + 1]).toBe("rm");
|
||||
expect(args).not.toContain("--user");
|
||||
expect(args.filter((arg) => arg === "-v")).toHaveLength(1);
|
||||
expect(args[args.indexOf("-v") + 1]).toMatch(
|
||||
/\/worker-runtime\.[^:]+:\/tmp\/openclaw-worker-cleanup$/u,
|
||||
);
|
||||
expect(args.slice(-4)).toEqual([
|
||||
"worker-cleanup-fixture",
|
||||
"-rf",
|
||||
"--",
|
||||
"/tmp/openclaw-worker-cleanup/runtime",
|
||||
]);
|
||||
expect(readFileSync(join(captureDir, "docker-run-args"), "utf8")).toContain(
|
||||
"worker-cleanup-fixture\0",
|
||||
);
|
||||
}
|
||||
if (exitCode !== "0") {
|
||||
expect(result.stderr).toContain("Preserved failed synthetic worker-cell state:");
|
||||
for (const runtime of runtimes) {
|
||||
expect(readFileSync(join(directory, runtime, "runtime", "state-marker"), "utf8")).toBe(
|
||||
"synthetic state",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(new Set(directories).size).toBe(2);
|
||||
expect(readFileSync(retained, "utf8")).toBe("previous evidence");
|
||||
},
|
||||
);
|
||||
|
||||
it("fails and retains state when container-owned cleanup fails", () => {
|
||||
const { captureDir, result } = runSurvivor({
|
||||
OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC: "openclaw@2026.9.4",
|
||||
OPENCLAW_UPGRADE_SURVIVOR_SCENARIO: "taskflow-restoration",
|
||||
FIXTURE_CLEANUP_EXIT: "43",
|
||||
});
|
||||
expect(result.status, result.stderr).toBe(1);
|
||||
expectFinalFailure(result.stderr, 1);
|
||||
expect(result.stderr).toContain("Worker-cell runtime cleanup failed:");
|
||||
const args = readFileSync(join(captureDir, "docker-cleanup-args"), "utf8").split("\0");
|
||||
const mount = args[args.indexOf("-v") + 1];
|
||||
if (!mount) {
|
||||
throw new Error("Cleanup did not mount the synthetic runtime");
|
||||
}
|
||||
const runtimeRoot = mount.slice(0, mount.lastIndexOf(":"));
|
||||
expect(readFileSync(join(runtimeRoot, "runtime", "state-marker"), "utf8")).toBe(
|
||||
"synthetic state",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("standalone upgrade survivor live OpenAI probe", () => {
|
||||
|
|
|
|||
58
test/scripts/upgrade-survivor-projects-doctor.test.ts
Normal file
58
test/scripts/upgrade-survivor-projects-doctor.test.ts
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
assertProjectsDoctorResult,
|
||||
assertProjectsInventory,
|
||||
} from "../../scripts/e2e/lib/upgrade-survivor/projects-doctor.mjs";
|
||||
|
||||
const inventory = {
|
||||
rows: [{ id: "retained", source: "registered", updated_at_ms: 123 }],
|
||||
schema: [{ name: "projects", sql: "fixture schema" }],
|
||||
workspace: "/fixture/workspace",
|
||||
sentinels: { "/fixture/workspace/PROJECTS-PROOF.txt": "original" },
|
||||
sqliteFamily: { main: "database", "-wal": "wal" },
|
||||
configHash: "config",
|
||||
retainedSnapshots: [],
|
||||
};
|
||||
const clean = { ok: true, checksRun: 1, checksSkipped: 27, findings: [] };
|
||||
|
||||
describe("Projects upgrade Doctor evidence", () => {
|
||||
it("accepts the selected check and preserved state without pinning unrelated check counts", () => {
|
||||
expect(() =>
|
||||
assertProjectsDoctorResult(clean, inventory, structuredClone(inventory)),
|
||||
).not.toThrow();
|
||||
expect(() =>
|
||||
assertProjectsDoctorResult({ ...clean, checksSkipped: 28 }, inventory, inventory),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ ...clean, ok: false },
|
||||
{ ...clean, checksRun: 0 },
|
||||
{ ...clean, checksRun: 2 },
|
||||
{ ...clean, findings: [{ checkId: "core/doctor/project-clone-shape", severity: "warning" }] },
|
||||
{ ...clean, findings: [{ checkId: "core/doctor/lint-selection", severity: "error" }] },
|
||||
])("rejects a skipped, failed, or incorrectly selected Doctor result %#", (report) => {
|
||||
expect(() => assertProjectsDoctorResult(report, inventory, inventory)).toThrow();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ rows: [] },
|
||||
{ rows: [{ id: "retained", source: "registered", updated_at_ms: 124 }] },
|
||||
{ schema: [] },
|
||||
{ workspace: "/fixture/other" },
|
||||
{ sentinels: {} },
|
||||
])("rejects changed persisted inventory or files %#", (change) => {
|
||||
expect(() => assertProjectsInventory({ ...inventory, ...change }, inventory)).toThrow();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ sqliteFamily: { main: "changed", "-wal": "wal" } },
|
||||
{ sqliteFamily: { main: "database" } },
|
||||
{ configHash: "rewritten" },
|
||||
{ retainedSnapshots: ["/fixture/cache/openclaw-sqlite-readonly-retained"] },
|
||||
])("rejects Doctor mutation or incomplete snapshot disposal %#", (change) => {
|
||||
expect(() =>
|
||||
assertProjectsDoctorResult(clean, inventory, { ...inventory, ...change }),
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
120
test/scripts/upgrade-survivor-taskflow-restoration.test.ts
Normal file
120
test/scripts/upgrade-survivor-taskflow-restoration.test.ts
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
assertTaskflowSnapshot,
|
||||
createTaskflowFixture,
|
||||
normalizeTaskflowSnapshot,
|
||||
TASKFLOW_PLUGIN_MANIFEST,
|
||||
} from "../../scripts/e2e/lib/upgrade-survivor/taskflow-restoration-fixture.mjs";
|
||||
import { resolveWorkerCellExport } from "../../scripts/e2e/lib/upgrade-survivor/worker-cell-package.mjs";
|
||||
import { resolveGatewayStartupPluginPlanFromRegistry } from "../../src/plugins/gateway-startup-plugin-plan.js";
|
||||
import type { PluginManifestRecord } from "../../src/plugins/manifest-registry.js";
|
||||
import type { PluginRegistrySnapshot } from "../../src/plugins/plugin-registry-snapshot.js";
|
||||
|
||||
describe("taskflow survivor evidence", () => {
|
||||
it("loads the SDK fixture through the actual Gateway startup plan", () => {
|
||||
const manifest: PluginManifestRecord = {
|
||||
...TASKFLOW_PLUGIN_MANIFEST,
|
||||
origin: "config",
|
||||
channels: [],
|
||||
providers: [],
|
||||
cliBackends: [],
|
||||
skills: [],
|
||||
hooks: [],
|
||||
rootDir: "/fixtures/taskflow-survivor",
|
||||
source: "/fixtures/taskflow-survivor/taskflow-restoration-plugin.mjs",
|
||||
manifestPath: "/fixtures/taskflow-survivor/openclaw.plugin.json",
|
||||
};
|
||||
const index: PluginRegistrySnapshot = {
|
||||
version: 1,
|
||||
hostContractVersion: "test",
|
||||
compatRegistryVersion: "test",
|
||||
migrationVersion: 1,
|
||||
policyHash: "test",
|
||||
generatedAtMs: 0,
|
||||
installRecords: {},
|
||||
diagnostics: [],
|
||||
plugins: [
|
||||
{
|
||||
pluginId: manifest.id,
|
||||
manifestPath: manifest.manifestPath,
|
||||
manifestHash: "fixture",
|
||||
rootDir: manifest.rootDir,
|
||||
origin: manifest.origin,
|
||||
enabled: true,
|
||||
startup: {
|
||||
sidecar: manifest.activation?.onStartup === true,
|
||||
memory: false,
|
||||
agentHarnesses: [],
|
||||
configPaths: [],
|
||||
},
|
||||
compat: [],
|
||||
},
|
||||
],
|
||||
};
|
||||
const plan = resolveGatewayStartupPluginPlanFromRegistry({
|
||||
config: { plugins: { allow: [manifest.id], entries: { [manifest.id]: { enabled: true } } } },
|
||||
env: {},
|
||||
index,
|
||||
manifestRegistry: { plugins: [manifest], diagnostics: [] },
|
||||
});
|
||||
expect(plan.pluginIds).toEqual([manifest.id]);
|
||||
});
|
||||
|
||||
it("compares complete persisted records regardless of owner Map insertion order", () => {
|
||||
const fixture = createTaskflowFixture(1_800_000_000_000);
|
||||
const snapshot = {
|
||||
tasks: new Map(fixture.tasks.toReversed().map((task) => [task.taskId, task])),
|
||||
flows: new Map(fixture.flows.toReversed().map((flow) => [flow.flowId, flow])),
|
||||
deliveryStates: new Map(fixture.deliveryStates.map((row) => [row.taskId, row])),
|
||||
};
|
||||
expect(() => assertTaskflowSnapshot(snapshot, fixture)).not.toThrow();
|
||||
expect(normalizeTaskflowSnapshot(snapshot)).toEqual(fixture);
|
||||
const changed = structuredClone(fixture);
|
||||
for (const task of changed.tasks) {
|
||||
task.detail.payload.enabled = false;
|
||||
}
|
||||
expect(() => assertTaskflowSnapshot(changed, fixture)).toThrow();
|
||||
const missing = structuredClone(fixture);
|
||||
missing.deliveryStates.pop();
|
||||
expect(() => assertTaskflowSnapshot(missing, fixture)).toThrow();
|
||||
const revision = structuredClone(fixture);
|
||||
for (const flow of revision.flows) {
|
||||
flow.revision += 1;
|
||||
}
|
||||
expect(() => assertTaskflowSnapshot(revision, fixture)).toThrow();
|
||||
});
|
||||
|
||||
it("seeds only settled tasks with existing parent flows and no execution owner", () => {
|
||||
const now = 1_800_000_000_000;
|
||||
const fixture = createTaskflowFixture(now);
|
||||
expect(fixture.tasks).toHaveLength(3);
|
||||
for (const task of fixture.tasks) {
|
||||
expect(task.status).toBe("succeeded");
|
||||
expect(task.notifyPolicy).toBe("silent");
|
||||
expect(task.deliveryStatus).toBe("not_applicable");
|
||||
expect(task.endedAt).toBeLessThan(now);
|
||||
expect(task.cleanupAfter).toBeGreaterThan(now);
|
||||
expect(task).not.toHaveProperty("executionOwner");
|
||||
expect(task).not.toHaveProperty("childSessionKey");
|
||||
expect(fixture.flows.some((flow) => flow.flowId === task.parentFlowId)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("resolves named or minified owner exports without substituting another symbol", () => {
|
||||
expect(resolveWorkerCellExport("export { loadSnapshot, other as a };", "loadSnapshot")).toBe(
|
||||
"loadSnapshot",
|
||||
);
|
||||
expect(
|
||||
resolveWorkerCellExport("export { loadSnapshot as c, close as r };", "loadSnapshot"),
|
||||
).toBe("c");
|
||||
expect(
|
||||
resolveWorkerCellExport("export { other as loadSnapshot };", "loadSnapshot"),
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
resolveWorkerCellExport("export { loadSnapshot } from './different.mjs';", "loadSnapshot"),
|
||||
).toBeUndefined();
|
||||
expect(() =>
|
||||
resolveWorkerCellExport("export { loadSnapshot as a, loadSnapshot as b };", "loadSnapshot"),
|
||||
).toThrow("Ambiguous");
|
||||
});
|
||||
});
|
||||
57
test/scripts/upgrade-survivor-worker-package.test.ts
Normal file
57
test/scripts/upgrade-survivor-worker-package.test.ts
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
assertWorkerCellPackageIdentity,
|
||||
readWorkerCellPackageIdentity,
|
||||
} from "../../scripts/e2e/lib/upgrade-survivor/worker-cell-package.mjs";
|
||||
import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
function packageFixture() {
|
||||
const root = tempDirs.make("worker-package-identity-");
|
||||
fs.mkdirSync(path.join(root, "dist"));
|
||||
fs.writeFileSync(path.join(root, "package.json"), '{"name":"openclaw","version":"2026.9.4"}');
|
||||
fs.writeFileSync(path.join(root, "openclaw.mjs"), "export {};\n");
|
||||
fs.writeFileSync(
|
||||
path.join(root, "dist/build-info.json"),
|
||||
JSON.stringify({ version: "2026.9.4", commit: "a".repeat(40) }),
|
||||
);
|
||||
fs.writeFileSync(path.join(root, "dist/entry.mjs"), "export const answer = 1;\n");
|
||||
fs.symlinkSync("entry.mjs", path.join(root, "dist/alias.mjs"));
|
||||
return root;
|
||||
}
|
||||
|
||||
describe("worker survivor installed payload identity", () => {
|
||||
it.each(["changed", "extra", "missing", "symlink"])(
|
||||
"rejects a %s dist entry even when package version and build commit agree",
|
||||
(mutation) => {
|
||||
const root = packageFixture();
|
||||
const expected = readWorkerCellPackageIdentity(root);
|
||||
if (mutation === "changed") {
|
||||
fs.writeFileSync(path.join(root, "dist/entry.mjs"), "export const answer = 2;\n");
|
||||
} else if (mutation === "extra") {
|
||||
fs.writeFileSync(path.join(root, "dist/stale.mjs"), "export {};\n");
|
||||
} else if (mutation === "missing") {
|
||||
fs.unlinkSync(path.join(root, "dist/entry.mjs"));
|
||||
} else {
|
||||
fs.unlinkSync(path.join(root, "dist/alias.mjs"));
|
||||
fs.symlinkSync("build-info.json", path.join(root, "dist/alias.mjs"));
|
||||
}
|
||||
expect(() =>
|
||||
assertWorkerCellPackageIdentity(readWorkerCellPackageIdentity(root), expected),
|
||||
).toThrow("Installed application payload differs");
|
||||
},
|
||||
);
|
||||
|
||||
it("compares application bytes while npm reifies its installed dependency tree", () => {
|
||||
const root = packageFixture();
|
||||
const expected = readWorkerCellPackageIdentity(root);
|
||||
fs.mkdirSync(path.join(root, "node_modules"));
|
||||
fs.writeFileSync(path.join(root, "node_modules/.package-lock.json"), "{}");
|
||||
expect(() =>
|
||||
assertWorkerCellPackageIdentity(readWorkerCellPackageIdentity(root), expected),
|
||||
).not.toThrow();
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue