mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 09:39:25 +00:00
fix(update): continue after baseline fingerprint timeouts (#144758)
* fix(update): tolerate baseline package fingerprint timeouts Keep the full package scan bounded, but record a recoverable warning when it times out. Verify retained and restored package identity/version and launchers when the baseline fingerprint is unavailable. Stop hashing at the observed file size while preserving before/after metadata checks, removing one redundant EOF read per file. Release-note context: npm updates continue after baseline fingerprint scan timeouts and report the reduced rollback verification as a warning. Refs #143767. * fix(update): retain fingerprint warnings after rollback Let shared completion persist result steps before recording the terminal rolled-back outcome. Keep the original failure reason and downtime. Release-note context: rollback fingerprint advisories remain visible in update status and reports after the update process exits. Refs #143767.
This commit is contained in:
Peter Steinberger
GitHub
parent
d9ad56cbe3
commit
f50d4ce71e
8 changed files with 294 additions and 63 deletions
|
|
@ -44,6 +44,13 @@ reported cause, then run the warning's exact cleanup command or
|
|||
that cannot boot or pass readiness still block completion. See
|
||||
[Status and history](/cli/update/status-and-history) to inspect recorded warnings.
|
||||
|
||||
The baseline package fingerprint is best effort. If its bounded scan times out,
|
||||
the update records a warning and continues with the retained package copy.
|
||||
Rollback then verifies the restored directory identity, package version, and
|
||||
affected launchers, and records that full fingerprint verification was unavailable.
|
||||
A timeout alone does not fail the update or rollback; detected changes to the
|
||||
retained copy still refuse restoration.
|
||||
|
||||
Interrupting a fresh local update before activation records a failed,
|
||||
`interrupted` history entry while its installation owner is still held.
|
||||
An interrupted update is not a successful update or a verified rollback.
|
||||
|
|
|
|||
|
|
@ -32,14 +32,24 @@ import {
|
|||
withUpdatePreviewSignals,
|
||||
} from "./update-command-run.js";
|
||||
import * as servicePlan from "./update-command-service-plan.js";
|
||||
import { publishUpdateCommandTerminalResult } from "./update-command-terminal.js";
|
||||
|
||||
const dirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
it.each([0, 86])("persists and surfaces successful Doctor warnings (exit %s)", (exitCode) => {
|
||||
it.each([
|
||||
{ kind: "package-post-install-doctor", name: "openclaw doctor", exitCode: 0 },
|
||||
{ kind: "package-post-install-doctor", name: "openclaw doctor", exitCode: 86 },
|
||||
{ kind: "recoverable-maintenance", name: "global install swap", exitCode: 0 },
|
||||
] as const)("persists and surfaces $kind warnings (exit $exitCode)", ({ kind, name, exitCode }) => {
|
||||
const env = { OPENCLAW_STATE_DIR: dirs.make("update-warning-ledger-") };
|
||||
const run = { runId: createUpdateRun({ trigger: "cli" }, { env }).runId, env };
|
||||
const message = "Skipped derived cache cleanup: permission denied. Run openclaw doctor --fix.";
|
||||
const message =
|
||||
kind === "recoverable-maintenance"
|
||||
? "baseline package fingerprint incomplete after 30 s; rollback will be verified by the retained package copy"
|
||||
: "Skipped derived cache cleanup: permission denied. Run openclaw doctor --fix.";
|
||||
const otherWarning =
|
||||
"Skipped legacy cache cleanup: read-only directory. Run openclaw doctor --fix.";
|
||||
kind === "recoverable-maintenance"
|
||||
? "Package fingerprint verification unavailable; rollback verified by the retained package copy's directory identity and version."
|
||||
: "Skipped legacy cache cleanup: read-only directory. Run openclaw doctor --fix.";
|
||||
const result = completeUpdateCommandRun(
|
||||
{
|
||||
status: "ok",
|
||||
|
|
@ -47,12 +57,12 @@ it.each([0, 86])("persists and surfaces successful Doctor warnings (exit %s)", (
|
|||
durationMs: 1,
|
||||
steps: [
|
||||
{
|
||||
name: "openclaw doctor",
|
||||
command: "openclaw doctor --fix",
|
||||
name,
|
||||
command: name,
|
||||
cwd: "/tmp/update-fixture",
|
||||
durationMs: 1,
|
||||
exitCode,
|
||||
advisory: { kind: "package-post-install-doctor", message },
|
||||
advisory: { kind, message },
|
||||
warnings: [message, otherWarning],
|
||||
},
|
||||
],
|
||||
|
|
@ -65,12 +75,12 @@ it.each([0, 86])("persists and surfaces successful Doctor warnings (exit %s)", (
|
|||
status: "succeeded",
|
||||
steps: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
step: "warning:openclaw doctor",
|
||||
step: `warning:${name}`,
|
||||
status: "completed",
|
||||
detail: message,
|
||||
}),
|
||||
expect.objectContaining({
|
||||
step: "warning:openclaw doctor:2",
|
||||
step: `warning:${name}:2`,
|
||||
status: "completed",
|
||||
detail: otherWarning,
|
||||
}),
|
||||
|
|
@ -85,6 +95,60 @@ afterEach(() => {
|
|||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("persists fingerprint warnings before closing a rolled-back run", () => {
|
||||
const env = { OPENCLAW_STATE_DIR: dirs.make("rollback-fingerprint-warning-") };
|
||||
const run = { runId: createUpdateRun({ trigger: "cli" }, { env }).runId, env };
|
||||
const warnings = [
|
||||
"baseline package fingerprint incomplete after 30 s; rollback will be verified by the retained package copy",
|
||||
"Package fingerprint verification unavailable; rollback verified by the retained package copy's directory identity and version.",
|
||||
];
|
||||
vi.spyOn(defaultRuntime, "writeJson").mockImplementation(() => {});
|
||||
const result = publishUpdateCommandTerminalResult(
|
||||
{ opts: { json: true, run }, ownedManagedUpdateEnv: env },
|
||||
{
|
||||
status: "error",
|
||||
mode: "npm",
|
||||
reason: "doctor-failed",
|
||||
before: { version: "1.0.0" },
|
||||
after: { version: "1.0.0" },
|
||||
recovery: {
|
||||
serviceRestartSafe: true,
|
||||
packageRollbackVerified: true,
|
||||
service: "healthy",
|
||||
version: "1.0.0",
|
||||
},
|
||||
durationMs: 50,
|
||||
steps: [
|
||||
{
|
||||
name: "global install rollback",
|
||||
command: "restore",
|
||||
cwd: env.OPENCLAW_STATE_DIR,
|
||||
durationMs: 1,
|
||||
exitCode: 0,
|
||||
advisory: { kind: "recoverable-maintenance", message: warnings.join("\n") },
|
||||
warnings,
|
||||
},
|
||||
],
|
||||
},
|
||||
{ rolledBack: true, downtimeMs: 25 },
|
||||
);
|
||||
expect(result).toMatchObject({ status: "error", reason: "doctor-failed" });
|
||||
const recorded = getUpdateRun(run.runId, { env })!;
|
||||
expect(recorded).toMatchObject({
|
||||
status: "rolled-back",
|
||||
reason: "doctor-failed",
|
||||
downtimeMs: 25,
|
||||
});
|
||||
expect(recorded.steps).toEqual(
|
||||
expect.arrayContaining(
|
||||
warnings.map((detail) => expect.objectContaining({ status: "completed", detail })),
|
||||
),
|
||||
);
|
||||
for (const warning of warnings) {
|
||||
expect(renderUpdateRunReport(recorded).markdown).toContain(warning);
|
||||
}
|
||||
});
|
||||
|
||||
it("presents committed steps without reopening the ledger for display", () => {
|
||||
const env = { OPENCLAW_STATE_DIR: dirs.make("update-progress-committed-") };
|
||||
const run = { runId: createUpdateRun({ trigger: "cli" }, { env }).runId, env };
|
||||
|
|
|
|||
|
|
@ -337,7 +337,7 @@ export function createUpdateRunProgress(
|
|||
export function completeUpdateCommandRun(
|
||||
result: UpdateRunResult,
|
||||
run: UpdateCommandOptions["run"],
|
||||
downtimeMs?: number,
|
||||
completion: { rolledBack?: boolean; downtimeMs?: number } = {},
|
||||
): UpdateRunResult {
|
||||
if (!run) {
|
||||
return result;
|
||||
|
|
@ -402,15 +402,16 @@ export function completeUpdateCommandRun(
|
|||
finishUpdateRun(
|
||||
run.runId,
|
||||
{
|
||||
status:
|
||||
normalized.status === "ok"
|
||||
status: completion.rolledBack
|
||||
? "rolled-back"
|
||||
: normalized.status === "ok"
|
||||
? "succeeded"
|
||||
: normalized.status === "error"
|
||||
? "failed"
|
||||
: "skipped",
|
||||
reason: normalized.reason,
|
||||
after: normalized.after,
|
||||
downtimeMs,
|
||||
downtimeMs: completion.downtimeMs,
|
||||
},
|
||||
recordOptions,
|
||||
);
|
||||
|
|
|
|||
|
|
@ -4,11 +4,7 @@ import {
|
|||
type ControlPlaneUpdateSentinelMetaFile,
|
||||
} from "../../infra/update-control-plane-sentinel.js";
|
||||
import { verifyPackageUpdateRecovery } from "../../infra/update-global.js";
|
||||
import {
|
||||
finishUpdateRun,
|
||||
getUpdateRun,
|
||||
recordUpdateRunPhase,
|
||||
} from "../../infra/update-run-ledger.js";
|
||||
import { getUpdateRun, recordUpdateRunPhase } from "../../infra/update-run-ledger.js";
|
||||
import { assertUpdateRecoveryAdmission } from "../../infra/update-run-recovery-admission.js";
|
||||
import { readCurrentGitUpdateRecovery } from "../../infra/update-runner-git-recovery.js";
|
||||
import type { UpdateRunResult, UpdateStepResult } from "../../infra/update-runner.js";
|
||||
|
|
@ -245,16 +241,7 @@ export function publishUpdateCommandTerminalResult(
|
|||
outcome: { rolledBack: boolean; downtimeMs?: number },
|
||||
): UpdateRunResult {
|
||||
const nextAction = recordUpdateResultNextAction(params, input);
|
||||
const run = params.opts.run;
|
||||
const { downtimeMs } = outcome;
|
||||
if (run && outcome.rolledBack) {
|
||||
finishUpdateRun(
|
||||
run.runId,
|
||||
{ status: "rolled-back", reason: input.reason, after: input.after, downtimeMs },
|
||||
{ env: run.env },
|
||||
);
|
||||
}
|
||||
const result = completeUpdateCommandRun(input, run, downtimeMs);
|
||||
const result = completeUpdateCommandRun(input, params.opts.run, outcome);
|
||||
printResult(result, params.opts, { nextAction });
|
||||
return result;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,13 @@ const log = createSubsystemLogger("update/package-integrity");
|
|||
let readerSequence = 0;
|
||||
|
||||
export type PackageIntegrityFingerprint = { digest: string; identity: string; version: string };
|
||||
export type PackageDirectoryIdentity = Pick<PackageIntegrityFingerprint, "identity" | "version">;
|
||||
|
||||
export class PackageIntegrityTimeoutError extends Error {
|
||||
constructor(readonly budgetMs: number) {
|
||||
super("Package rollback verification timed out");
|
||||
}
|
||||
}
|
||||
|
||||
export type PackageRootIntegrityFingerprint =
|
||||
| { kind: "directory"; tree: PackageIntegrityFingerprint }
|
||||
|
|
@ -121,7 +128,7 @@ export function createPackageIntegrityReader(timeoutMs = MAX_SCAN_MS) {
|
|||
)
|
||||
.catch(() => {});
|
||||
}
|
||||
throw new Error("Package rollback verification timed out");
|
||||
throw new PackageIntegrityTimeoutError(budget);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
|
@ -170,18 +177,17 @@ export function createPackageIntegrityReader(timeoutMs = MAX_SCAN_MS) {
|
|||
}
|
||||
const hash = createHash("sha256");
|
||||
const buffer = Buffer.allocUnsafe(64 * 1024);
|
||||
const size = Number(stat.size);
|
||||
let position = 0;
|
||||
while (true) {
|
||||
// The final stat detects growth; an extra EOF read costs one OS call per file.
|
||||
while (position < size) {
|
||||
const { bytesRead } = await read(() =>
|
||||
handle.read(buffer, 0, Math.min(buffer.length, remainingBytes - position + 1), position),
|
||||
handle.read(buffer, 0, Math.min(buffer.length, size - position), position),
|
||||
);
|
||||
if (bytesRead === 0) {
|
||||
break;
|
||||
throw new Error("Package rollback file changed while reading");
|
||||
}
|
||||
position += bytesRead;
|
||||
if (position > remainingBytes) {
|
||||
throw new Error("Package rollback verification byte limit exceeded");
|
||||
}
|
||||
hash.update(buffer.subarray(0, bytesRead));
|
||||
}
|
||||
if (!unchanged(stat, await read(() => handle.stat({ bigint: true })))) {
|
||||
|
|
@ -306,6 +312,21 @@ export function createPackageIntegrityReader(timeoutMs = MAX_SCAN_MS) {
|
|||
return { kind: "link", metadata: metadata(stat).slice(0, -1), target };
|
||||
}
|
||||
|
||||
async function directoryIdentity(root: string): Promise<PackageDirectoryIdentity | null> {
|
||||
const stat = await read(() => fs.lstat(root, { bigint: true }));
|
||||
if (stat.isSymbolicLink()) {
|
||||
return null;
|
||||
}
|
||||
if (!stat.isDirectory() || stat.ino === 0n) {
|
||||
throw new Error("Package rollback filesystem identity is unavailable");
|
||||
}
|
||||
const version = await read(() => readPackageVersion(root, { maxBytes: MAX_MANIFEST_BYTES }));
|
||||
if (!version || !unchanged(stat, await read(() => fs.lstat(root, { bigint: true })))) {
|
||||
throw new Error("Package rollback identity changed or version is unavailable");
|
||||
}
|
||||
return { identity: identity(stat), version };
|
||||
}
|
||||
|
||||
async function launcher(file: string): Promise<string> {
|
||||
const stat = await read(() => fs.lstat(file, { bigint: true }));
|
||||
const contents = stat.isSymbolicLink()
|
||||
|
|
@ -337,5 +358,5 @@ export function createPackageIntegrityReader(timeoutMs = MAX_SCAN_MS) {
|
|||
}
|
||||
}
|
||||
|
||||
return { tree, rootEntry, launcher, exists, entries, observe };
|
||||
return { tree, rootEntry, directoryIdentity, launcher, exists, entries, observe };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { isDeepStrictEqual } from "node:util";
|
|||
import { formatErrorMessage } from "./errors.js";
|
||||
import {
|
||||
createPackageIntegrityReader,
|
||||
type PackageDirectoryIdentity,
|
||||
type PackageRootIntegrityFingerprint,
|
||||
} from "./package-update-integrity.js";
|
||||
|
||||
|
|
@ -63,6 +64,7 @@ export async function verifyNpmRootRecovery(
|
|||
fromBackup: boolean;
|
||||
hadPackage: boolean;
|
||||
previousRoot: PackageRootIntegrityFingerprint | undefined;
|
||||
previousIdentity?: PackageDirectoryIdentity;
|
||||
targetSwapRoot: string;
|
||||
shims: readonly { destination: string; backup: string | null; fingerprint?: string }[];
|
||||
},
|
||||
|
|
@ -73,11 +75,13 @@ export async function verifyNpmRootRecovery(
|
|||
await reader.observe(fromBackup ? "retained" : "restored", async () => {
|
||||
if (
|
||||
hadPackage
|
||||
? !previousRoot ||
|
||||
!isDeepStrictEqual(
|
||||
await reader.rootEntry(root, targetSwapRoot, previousRoot.kind),
|
||||
previousRoot,
|
||||
)
|
||||
? previousRoot
|
||||
? !isDeepStrictEqual(
|
||||
await reader.rootEntry(root, targetSwapRoot, previousRoot.kind),
|
||||
previousRoot,
|
||||
)
|
||||
: !params.previousIdentity ||
|
||||
!isDeepStrictEqual(await reader.directoryIdentity(root), params.previousIdentity)
|
||||
: !fromBackup && (await reader.exists(root))
|
||||
) {
|
||||
throw new Error(
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import { createDeferredCore } from "../shared/deferred.js";
|
|||
import { withTestDir } from "../test-helpers/temp-dir.js";
|
||||
import { swapStagedPackageInstall, type PackageUpdateTransaction } from "./package-update-swap.js";
|
||||
import { createPackageSwapFixture } from "./package-update-swap.test-support.js";
|
||||
import { updateRunStepsFromResultStep } from "./update-run-step.js";
|
||||
|
||||
afterEach(() => {
|
||||
setLoggerOverride(null);
|
||||
|
|
@ -29,6 +30,88 @@ function captureReaderLogs() {
|
|||
}
|
||||
|
||||
describe("package verification bounds", () => {
|
||||
it.each(["activation", "rollback", "changed identity", "changed version"] as const)(
|
||||
"handles %s after the baseline fingerprint times out",
|
||||
async (outcome) => {
|
||||
await withTestDir({ prefix: "openclaw-fingerprint-advisory-" }, async (base) => {
|
||||
const { params, packageRoot, launcher } = await createPackageSwapFixture(base);
|
||||
const original = await fs.stat(packageRoot);
|
||||
const open = fs.open.bind(fs);
|
||||
const blocked = createDeferredCore();
|
||||
let entered = false;
|
||||
vi.spyOn(fs, "open").mockImplementation(async (...args) => {
|
||||
if (!entered && String(args[0]) === path.join(packageRoot, "dist", "index.js")) {
|
||||
entered = true;
|
||||
await blocked.promise;
|
||||
}
|
||||
return open(...args);
|
||||
});
|
||||
let transaction: PackageUpdateTransaction | undefined;
|
||||
const beforeActivate = vi.fn();
|
||||
try {
|
||||
const result = await swapStagedPackageInstall({
|
||||
...params,
|
||||
timeoutMs: 200,
|
||||
beforeActivate,
|
||||
onTransaction: (value) => {
|
||||
transaction = value;
|
||||
},
|
||||
});
|
||||
expect(entered).toBe(true);
|
||||
expect(result.status, result.step.stderrTail ?? "").toBe("committed");
|
||||
expect(beforeActivate).toHaveBeenCalledOnce();
|
||||
expect(result.step.advisory?.message).toContain(
|
||||
"baseline package fingerprint incomplete",
|
||||
);
|
||||
expect(updateRunStepsFromResultStep(result.step)).toContainEqual(
|
||||
expect.objectContaining({ step: "warning:global install swap", status: "completed" }),
|
||||
);
|
||||
expect(await fs.readFile(launcher, "utf8")).toBe("candidate launcher\n");
|
||||
if (!transaction) {
|
||||
throw new Error("Missing package transaction");
|
||||
}
|
||||
if (outcome === "changed identity" || outcome === "changed version") {
|
||||
if (outcome === "changed identity") {
|
||||
await fs.rename(transaction.backupRoot, `${transaction.backupRoot}.original`);
|
||||
await fs.cp(`${transaction.backupRoot}.original`, transaction.backupRoot, {
|
||||
recursive: true,
|
||||
});
|
||||
} else {
|
||||
await fs.writeFile(
|
||||
path.join(transaction.backupRoot, "package.json"),
|
||||
'{"version":"3.0.0"}',
|
||||
);
|
||||
}
|
||||
const refused = await transaction.rollback(() => {});
|
||||
expect(refused.exitCode).toBe(1);
|
||||
expect(refused.advisory).toBeUndefined();
|
||||
expect(refused.stderrTail).toContain("retained package tree changed");
|
||||
expect(await fs.readFile(launcher, "utf8")).toBe("candidate launcher\n");
|
||||
await expect(fs.stat(transaction.backupRoot)).resolves.toBeDefined();
|
||||
return;
|
||||
}
|
||||
if (outcome === "rollback") {
|
||||
const restored = await transaction.rollback(() => {});
|
||||
expect(restored).toMatchObject({ exitCode: 0, activePackageRoot: packageRoot });
|
||||
expect(restored.advisory?.message).toContain("fingerprint verification unavailable");
|
||||
expect(restored.stderrTail ?? "").not.toMatch(/unverified|verification failed/);
|
||||
expect(await fs.readFile(launcher, "utf8")).toBe("old launcher\n");
|
||||
expect(await fs.readFile(path.join(packageRoot, "package.json"), "utf8")).toContain(
|
||||
'"version":"1.0.0"',
|
||||
);
|
||||
const actual = await fs.stat(packageRoot);
|
||||
expect([actual.dev, actual.ino]).toEqual([original.dev, original.ino]);
|
||||
}
|
||||
expect(
|
||||
await transaction.complete({ activationVerified: outcome === "activation" }, () => {}),
|
||||
).toBeUndefined();
|
||||
} finally {
|
||||
blocked.resolve();
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each([1024 * 1024 + 1, 1024 * 1024 * 1024 + 1])(
|
||||
"rejects manifest growth to %i bytes without attempting an oversized metadata allocation",
|
||||
async (size) => {
|
||||
|
|
@ -95,8 +178,13 @@ describe("package verification bounds", () => {
|
|||
expect((await transactions[0]!.rollback(() => {})).exitCode).toBe(0);
|
||||
await expect(fs.readFile(manifest, "utf8")).resolves.toHaveLength(1024 * 1024);
|
||||
const finished = observations.filter((record) => record.event === "reader-settled");
|
||||
expect(finished.map((record) => record.phase)).toEqual(["baseline", "retained", "restored"]);
|
||||
expect(new Set(finished.map((record) => record.readerId)).size).toBe(3);
|
||||
expect(finished.map((record) => record.phase)).toEqual([
|
||||
"baseline",
|
||||
"baseline",
|
||||
"retained",
|
||||
"restored",
|
||||
]);
|
||||
expect(new Set(finished.map((record) => record.readerId)).size).toBe(4);
|
||||
for (const record of finished) {
|
||||
expect(record).toMatchObject({ outcome: "completed", budgetMs: 30_000, pendingIo: 0 });
|
||||
expect(record.timeoutObservedAtMonotonicMs).toBeUndefined();
|
||||
|
|
@ -179,6 +267,9 @@ describe("package verification bounds", () => {
|
|||
const close = vi.spyOn(handle, "close");
|
||||
const read = vi.spyOn(handle, "read");
|
||||
const open = vi.spyOn(fs, "open").mockImplementation(async (...args) => {
|
||||
if (String(args[0]) !== path.join(packageRoot, "dist", "index.js")) {
|
||||
return realOpen(...args);
|
||||
}
|
||||
if (operation === "open") {
|
||||
return late.promise;
|
||||
}
|
||||
|
|
@ -197,13 +288,21 @@ describe("package verification bounds", () => {
|
|||
onLiveMutation,
|
||||
timeoutMs: 40,
|
||||
});
|
||||
expect(result.status).toBe("failed");
|
||||
expect(result.step.stderrTail).toContain("timed out");
|
||||
expect(result.status).toBe("committed");
|
||||
expect(result.step.advisory?.message).toContain(
|
||||
"baseline package fingerprint incomplete",
|
||||
);
|
||||
expect(Date.now() - started).toBeLessThan(2000);
|
||||
expect(beforeActivate).not.toHaveBeenCalled();
|
||||
expect(onLiveMutation).not.toHaveBeenCalled();
|
||||
expect(open).toHaveBeenCalledTimes(1);
|
||||
const baseline = observations.filter((record) => record.phase === "baseline");
|
||||
expect(beforeActivate).toHaveBeenCalledOnce();
|
||||
expect(onLiveMutation).toHaveBeenCalledOnce();
|
||||
expect(
|
||||
open.mock.calls.filter(
|
||||
([file]) => String(file) === path.join(packageRoot, "dist", "index.js"),
|
||||
),
|
||||
).toHaveLength(1);
|
||||
const baseline = observations.filter(
|
||||
(record) => record.readerId === observations[0]?.readerId,
|
||||
);
|
||||
expect(baseline).toHaveLength(2);
|
||||
const [begin, settled] = baseline;
|
||||
expect(begin).toMatchObject({ event: "reader-started", budgetMs: 40 });
|
||||
|
|
@ -230,8 +329,8 @@ describe("package verification bounds", () => {
|
|||
}
|
||||
await expect(
|
||||
fs.readFile(path.join(packageRoot, "package.json"), "utf8"),
|
||||
).resolves.toContain('"version":"1.0.0"');
|
||||
await expect(fs.readFile(launcher, "utf8")).resolves.toBe("old launcher\n");
|
||||
).resolves.toContain('"version":"2.0.0"');
|
||||
await expect(fs.readFile(launcher, "utf8")).resolves.toBe("candidate launcher\n");
|
||||
} finally {
|
||||
late.resolve(handle);
|
||||
await handle.close();
|
||||
|
|
@ -289,7 +388,7 @@ describe("package verification bounds", () => {
|
|||
const result = await swapStagedPackageInstall({ ...params, timeoutMs: 40 });
|
||||
// Preserve the existing best-effort close policy, but report its timeout.
|
||||
expect(result.status).toBe("committed");
|
||||
expect(observations.find((record) => record.event === "reader-settled")).toMatchObject({
|
||||
expect(observations.findLast((record) => record.event === "reader-settled")).toMatchObject({
|
||||
phase: "baseline",
|
||||
outcome: "timed-out",
|
||||
pendingIo: 1,
|
||||
|
|
|
|||
|
|
@ -18,7 +18,9 @@ import {
|
|||
} from "./package-update-filesystem.js";
|
||||
import {
|
||||
createPackageIntegrityReader,
|
||||
PackageIntegrityTimeoutError,
|
||||
readPackageVersionIfPresent,
|
||||
type PackageDirectoryIdentity,
|
||||
type PackageRootIntegrityFingerprint,
|
||||
} from "./package-update-integrity.js";
|
||||
import {
|
||||
|
|
@ -82,6 +84,7 @@ export async function swapStagedPackageInstall(params: {
|
|||
: params.installTarget.packageRoot;
|
||||
const targetSwapRoot = native?.liveProjectRoot ?? targetPackageRoot;
|
||||
const stagedSwapRoot = native?.projectRoot ?? params.stage.packageRoot;
|
||||
const warnings: string[] = [];
|
||||
const step = (
|
||||
exitCode: number,
|
||||
stdoutTail: string | null,
|
||||
|
|
@ -94,6 +97,15 @@ export async function swapStagedPackageInstall(params: {
|
|||
exitCode,
|
||||
stdoutTail,
|
||||
stderrTail,
|
||||
...(exitCode === 0 && warnings.length > 0
|
||||
? {
|
||||
advisory: {
|
||||
kind: "recoverable-maintenance" as const,
|
||||
message: warnings.join("\n"),
|
||||
},
|
||||
warnings: [...warnings],
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
if (!targetLayout || !targetPackageRoot || !targetSwapRoot) {
|
||||
return {
|
||||
|
|
@ -115,6 +127,7 @@ export async function swapStagedPackageInstall(params: {
|
|||
let previousVersion: string | null = null;
|
||||
let previousDistFiles: string[] | undefined;
|
||||
let previousRoot: PackageRootIntegrityFingerprint | undefined;
|
||||
let previousIdentity: PackageDirectoryIdentity | undefined;
|
||||
let rootLink: ReturnType<typeof createNpmPackageRootLinkLifecycle> | undefined;
|
||||
let packageBackedUp = false;
|
||||
let displacedCandidateRoot: string | undefined;
|
||||
|
|
@ -132,7 +145,7 @@ export async function swapStagedPackageInstall(params: {
|
|||
let activationCompleted = false;
|
||||
const verifyNpmRecovery = (root: string, fromBackup: boolean) =>
|
||||
verifyNpmRootRecovery(
|
||||
{ root, fromBackup, hadPackage, previousRoot, targetSwapRoot, shims },
|
||||
{ root, fromBackup, hadPackage, previousRoot, previousIdentity, targetSwapRoot, shims },
|
||||
params.timeoutMs,
|
||||
);
|
||||
const restoreSwap = async (assertCurrent = () => {}): Promise<string[]> => {
|
||||
|
|
@ -191,7 +204,15 @@ export async function swapStagedPackageInstall(params: {
|
|||
await verifyNpmRecovery(targetSwapRoot, false);
|
||||
// Returning to absence cannot establish a verified previous runtime.
|
||||
packageRollbackVerified =
|
||||
hadPackage && previousRoot?.kind === "directory" && messages.length === 0;
|
||||
hadPackage &&
|
||||
(previousRoot?.kind === "directory" ||
|
||||
(!previousRoot && previousIdentity !== undefined)) &&
|
||||
messages.length === 0;
|
||||
if (packageRollbackVerified && !previousRoot) {
|
||||
warnings.push(
|
||||
"Package fingerprint verification unavailable; rollback verified by the retained package copy's directory identity and version.",
|
||||
);
|
||||
}
|
||||
if (previousRoot?.kind === "link" && messages.length === 0) {
|
||||
messages.push(
|
||||
`${rollback.length > 0 ? "Restored" : "Verified"} the npm package link and affected launchers; external checkout runtime integrity is unverified.`,
|
||||
|
|
@ -263,11 +284,29 @@ export async function swapStagedPackageInstall(params: {
|
|||
? await readPackageVersionIfPresent(params.installTarget.packageRoot)
|
||||
: null;
|
||||
if (hadPackage && !native) {
|
||||
// Unreadable or unbounded rollback material must fail while the old
|
||||
// package is still live, before beforeActivate may stop its service.
|
||||
previousRoot = await baseline.rootEntry(targetSwapRoot);
|
||||
previousVersion = previousRoot.kind === "directory" ? previousRoot.tree.version : null;
|
||||
if (previousRoot.kind === "link") {
|
||||
try {
|
||||
previousRoot = await baseline.rootEntry(targetSwapRoot);
|
||||
} catch (error) {
|
||||
if (!(error instanceof PackageIntegrityTimeoutError)) {
|
||||
throw error;
|
||||
}
|
||||
// Capture the identity before mutation even when the full walk exhausted its budget.
|
||||
previousIdentity =
|
||||
(await createPackageIntegrityReader(params.timeoutMs).directoryIdentity(
|
||||
targetSwapRoot,
|
||||
)) ?? undefined;
|
||||
if (!previousIdentity) {
|
||||
throw error;
|
||||
}
|
||||
warnings.push(
|
||||
`baseline package fingerprint incomplete after ${error.budgetMs / 1000} s; rollback will be verified by the retained package copy`,
|
||||
);
|
||||
}
|
||||
previousVersion =
|
||||
previousRoot?.kind === "directory"
|
||||
? previousRoot.tree.version
|
||||
: (previousIdentity?.version ?? null);
|
||||
if (previousRoot?.kind === "link") {
|
||||
rootLink = createNpmPackageRootLinkLifecycle({
|
||||
liveRoot: targetSwapRoot,
|
||||
backupRoot,
|
||||
|
|
@ -282,6 +321,8 @@ export async function swapStagedPackageInstall(params: {
|
|||
(await collectPackageDistInventory(params.installTarget.packageRoot!));
|
||||
}
|
||||
packageRollbackVerified = hadPackage && previousVersion !== null;
|
||||
};
|
||||
const readLaunchers = async (launcherReader: ReturnType<typeof createPackageIntegrityReader>) => {
|
||||
await fs.mkdir(targetLayout.globalRoot, { recursive: true });
|
||||
const shimNames = new Set([params.packageName, "openclaw"]);
|
||||
const shimEntries =
|
||||
|
|
@ -291,7 +332,7 @@ export async function swapStagedPackageInstall(params: {
|
|||
await (
|
||||
native
|
||||
? fs.readdir(params.stage.layout.binDir)
|
||||
: baseline.entries(params.stage.layout.binDir)
|
||||
: launcherReader.entries(params.stage.layout.binDir)
|
||||
).catch((error: unknown) => {
|
||||
if (hasErrnoCode(error, "ENOENT")) {
|
||||
return [];
|
||||
|
|
@ -312,13 +353,14 @@ export async function swapStagedPackageInstall(params: {
|
|||
const destination = path.join(targetLayout.binDir, entry);
|
||||
const backup = (await (native
|
||||
? pathEntryExists(destination)
|
||||
: baseline.exists(destination)))
|
||||
: launcherReader.exists(destination)))
|
||||
? path.join(shimBackupDir, entry)
|
||||
: null;
|
||||
const fingerprint = backup && !native ? await baseline.launcher(destination) : undefined;
|
||||
const fingerprint =
|
||||
backup && !native ? await launcherReader.launcher(destination) : undefined;
|
||||
if (backup) {
|
||||
await copyPathEntry(destination, backup);
|
||||
if (!native && (await baseline.launcher(backup)) !== fingerprint) {
|
||||
if (!native && (await launcherReader.launcher(backup)) !== fingerprint) {
|
||||
throw new Error(`Package rollback launcher backup changed: ${destination}`);
|
||||
}
|
||||
}
|
||||
|
|
@ -333,6 +375,9 @@ export async function swapStagedPackageInstall(params: {
|
|||
};
|
||||
try {
|
||||
await (native ? readBaseline() : baseline.observe("baseline", readBaseline));
|
||||
// The optional tree scan must not consume the launcher backup's deadline.
|
||||
const launcherReader = createPackageIntegrityReader(params.timeoutMs);
|
||||
await launcherReader.observe("baseline", () => readLaunchers(launcherReader));
|
||||
// Validation and launcher backup finish while the old Gateway is serving.
|
||||
// Only this boundary authorizes the orchestrator to suspend the service.
|
||||
const assertProjectUnchanged = native
|
||||
|
|
@ -536,7 +581,10 @@ export async function swapStagedPackageInstall(params: {
|
|||
}
|
||||
activePackageRoot = null;
|
||||
packageBackedUp = true;
|
||||
packageRollbackVerified = native !== undefined || previousRoot?.kind === "directory";
|
||||
packageRollbackVerified =
|
||||
native !== undefined ||
|
||||
previousRoot?.kind === "directory" ||
|
||||
previousIdentity !== undefined;
|
||||
}
|
||||
rollback.push(async (assertCurrent) => {
|
||||
if (!native && hadPackage) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue