fix(update): let finalization finish without an implicit deadline (#153085)

* fix: preserve omitted update finalization deadlines

* test: align RPC finalizer omission expectation

* test(update): type the migrated stdin iterator return

* docs(update): clarify aggregate deadline omission
This commit is contained in:
Jason (Json) 2026-09-19 14:26:19 -06:00 • committed by GitHub
parent f9494701bb
commit d79393ea91
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 305 additions and 77 deletions

View file

@ -193,18 +193,22 @@ deferred-install activation checks.
Post-core repair Doctor and `openclaw update finalize` run without a separate
per-Doctor deadline unless the operator supplies `--timeout`. A fresh post-core
process receives the operator choice separately from its internal step allowance.
Older targets retain their existing allowance and deadline behavior. Existing
install, build, plugin-operation and
enclosing activation deadlines still apply. An explicit `--timeout <seconds>`
limits each finalization phase and its child commands. Admission and config phases
scale with shared SQLite state.
Older targets retain their existing allowance and deadline behavior.
When `--timeout` is omitted, current CLI and RPC finalization do not add an aggregate
activation deadline. Explicit operator limits and inherited activation allowances
still apply; older or unrecognized handoffs retain their existing finite-deadline
behavior. Independent install, build, plugin-operation, readiness, and cleanup
bounds still apply. An explicit `--timeout <seconds>` limits each finalization phase
and its child commands. Admission and config phases scale with shared SQLite state.
Post-plugin config validation and readiness checks use the measured shared and
agent database sizes after Doctor finishes, including WAL files. Serial plugin
operations retain individual deadlines within the enclosing activation budget. That
budget uses the measured database sizes, observed candidate startup, plugin count,
and the caller's step allowance. Migrated finalization receives the same allowance;
it does not choose a separate default. Expiry reports `update-activation-timeout`
and retains ownership until writers settle; it does not authorize rollback or restart.
operations retain individual deadlines. When an aggregate activation budget is
present, it uses the measured database sizes, observed candidate startup, plugin
count, and the caller's step allowance. Migrated finalization preserves explicit or
inherited allowances. Aggregate expiry reports `update-activation-timeout` and
retains ownership until writers settle; it does not authorize rollback or restart.
Use `openclaw update status` and Doctor for recovery guidance.
| Flag | Description |

View file

@ -18,7 +18,8 @@ const mocks = vi.hoisted(() => ({
hasSchemaRefusal: vi.fn(),
maybeRestartService: vi.fn(),
maybeStopService: vi.fn(),
prepareMutableUpdate: vi.fn<(env?: NodeJS.ProcessEnv) => Promise<void>>(),
prepareMutableUpdate:
vi.fn<(env?: NodeJS.ProcessEnv, activationTimeoutMs?: number) => Promise<void>>(),
pluginPreflight: vi.fn(),
pluginTargets: vi.fn(),
pluginRecords: vi.fn(),

View file

@ -38,6 +38,51 @@ const { executionParams, inspectOrStopService, mocks, schemaContext, successfulU
await import("./update-command-execution.test-support.js");
describe("mutable update execution", () => {
it.each(
(["package", "git"] as const).flatMap((kind) =>
[undefined, 30_000].map((timeoutMs) => ({ kind, timeoutMs })),
),
)(
"preserves aggregate work intent at $kind activation ($timeoutMs)",
async ({ kind, timeoutMs }) => {
const budgets = await import("../../infra/update-finalization-budget.js");
const budget = vi
.spyOn(budgets, "resolveUpdateFinalizationTimeoutMs")
.mockResolvedValue(180_000);
mocks.runPackageUpdate.mockImplementation(async ({ beforeActivate }) => {
await beforeActivate();
return successfulUpdate;
});
mocks.runGitUpdate.mockImplementation(
async (
params: Parameters<typeof import("./update-command-git.js").updateGitInstall>[0],
) => {
if (!params.inspectGitTarget || !params.beforeGitMutation) {
throw new Error("Expected both real Git admission callbacks");
}
const target = { schemaVersions: { state: 15, agent: 19 } };
await params.inspectGitTarget(target);
await params.beforeGitMutation(target);
return { ...successfulUpdate, mode: "git" };
},
);
const execution = await executeMutableUpdate({
...executionParams(kind),
timeoutMs,
updateStepTimeoutMs: timeoutMs ?? 30 * 60_000,
});
expect(execution?.result.status).toBe("ok");
expect(execution?.mutationStarted).toBe(true);
expect(mocks.prepareMutableUpdate).toHaveBeenCalledTimes(kind === "package" ? 2 : 3);
expect(mocks.prepareMutableUpdate.mock.calls.at(-1)?.[1]).toBe(
timeoutMs === undefined ? undefined : 180_000,
);
expect(budget).toHaveBeenCalledTimes(timeoutMs === undefined ? 0 : 1);
},
);
it.each(["package", "git"] as const)(
"continues the %s update with the recorded readiness warning instead of inference repair",
async (kind) => {

View file

@ -534,13 +534,16 @@ export async function executeMutableUpdate(
// Health and candidate work can outlive the inspected service/config generation.
await recheckSchemas(admittedTargetSchemaVersions);
assertExecutionCurrent();
const activationTimeoutMs = await resolveUpdateFinalizationTimeoutMs(updateStepTimeoutMs, {
env,
databases: schemaVersions,
observedStartupMs: observedGatewayStartupMs,
pluginCount: Object.keys(config.plugins?.entries ?? {}).length,
nodeRunner: params.packageUpdateNodeRunner,
});
const activationTimeoutMs =
params.timeoutMs === undefined
? undefined
: await resolveUpdateFinalizationTimeoutMs(updateStepTimeoutMs, {
env,
databases: schemaVersions,
observedStartupMs: observedGatewayStartupMs,
pluginCount: Object.keys(config.plugins?.entries ?? {}).length,
nodeRunner: params.packageUpdateNodeRunner,
});
assertExecutionCurrent();
await params.prepareMutableUpdate(env, activationTimeoutMs);
assertExecutionCurrent();

View file

@ -6,6 +6,7 @@ import type {
import type { UpdateRunStep } from "../../infra/update-run-record.js";
import type { UpdateRecoveryHandoff } from "../../infra/update-run-recovery.js";
import type { UpdateRunResult } from "../../infra/update-runner.js";
import type { UpdateTimeoutHandoff } from "../../infra/update-timeout-provenance.js";
import type { UpdateCommandChildGrant } from "./update-command-executor.js";
import type { FinishUpdateParams } from "./update-command-finish-types.js";
@ -20,7 +21,7 @@ export type UpdateDoctorInput = {
workspaceSuggestions?: boolean;
};
export type MigratedUpdateFinalizationInput = {
export type MigratedUpdateFinalizationInput = Partial<UpdateTimeoutHandoff> & {
params: Omit<FinishUpdateParams, "packageTransaction" | "preManagedServiceStop" | "opts"> & {
opts: Omit<FinishUpdateParams["opts"], "run" | "recovery"> & {
run?: Omit<

View file

@ -14,6 +14,7 @@ import {
import { resolveUpdateFinalizationTimeoutMs } from "../../infra/update-finalization-budget.js";
import type { UpdateRunStep } from "../../infra/update-run-record.js";
import { isUpdateGatewayReadinessPending } from "../../infra/update-run-step.js";
import { createUpdateTimeoutHandoff } from "../../infra/update-timeout-provenance.js";
import { runUtf8CommandWithTimeout } from "../../process/exec.js";
import type { OpenClawSchemaVersions } from "../../state/openclaw-schema-versions.js";
import { resolveOpenClawStateSqlitePath } from "../../state/openclaw-state-db.paths.js";
@ -198,23 +199,28 @@ export async function continueMigratedUpdateInFreshProcess(
const { windowsTaskAutoStartRecovery: _windows, ...serializableStop } = preManagedServiceStop;
stopState = serializableStop;
}
run.activationTimeoutMs ??= await resolveUpdateFinalizationTimeoutMs(
params.updateStepTimeoutMs,
{
env: params.ownedManagedUpdateEnv ?? run.env,
databases: params.schemaVersions,
pluginCount: Object.keys(params.preUpdatePluginInstallRecords).length,
nodeRunner: params.packageUpdateNodeRunner,
},
);
if (params.opts.timeout !== undefined) {
run.activationTimeoutMs ??= await resolveUpdateFinalizationTimeoutMs(
params.updateStepTimeoutMs,
{
env: params.ownedManagedUpdateEnv ?? run.env,
databases: params.schemaVersions,
pluginCount: Object.keys(params.preUpdatePluginInstallRecords).length,
nodeRunner: params.packageUpdateNodeRunner,
},
);
}
const handoff = createUpdateTimeoutHandoff(params.opts.timeout, params.updateStepTimeoutMs);
assertCurrent();
const resultPath = path.join(scratchDir, "result.json");
const { requesterAuthority, executorFence, ...runIdentity } = run;
const input: MigratedUpdateFinalizationInput = {
...handoff,
params: {
...serializable,
opts: {
...params.opts,
timeout: handoff.timeout.serialized,
run: {
...runIdentity,
...(requesterAuthority
@ -239,8 +245,8 @@ export async function continueMigratedUpdateInFreshProcess(
env: workerEnv,
input: JSON.stringify({ ...input, ...(grant ? { executor: grant } : {}) }),
beforeInput: bindChild,
// This continuation includes bounded plugin steps as well as service
// verification; the whole-process bound must exceed one step's budget.
// Only an operator deadline bounds forward finalization. Probes and
// cancellation settlement keep their separate finite allowances.
timeoutMs: run.activationTimeoutMs,
killProcessTree: true,
requireProcessTreeExtinction: true,

View file

@ -42,6 +42,10 @@ import {
} from "../../infra/update-post-core-context.js";
import { UpdateFailureFactSchema } from "../../infra/update-run-schema.js";
import type { UpdateRunResult } from "../../infra/update-runner.js";
import {
createUpdateTimeoutHandoff,
isOmittedUpdateTimeout,
} from "../../infra/update-timeout-provenance.js";
import { getWindowsSystem32ExePath } from "../../infra/windows-install-roots.js";
import { writePersistedInstalledPluginIndexInstallRecordsWithLease } from "../../plugins/installed-plugin-index-records.js";
import { restorePersistedInstalledPluginIndexIfCurrent } from "../../plugins/installed-plugin-index-store-write.js";
@ -93,18 +97,10 @@ export async function resolvePostCoreUpdateOperatorOptions(params: {
if (!params.resultPath || params.opts.timeout === undefined) {
return params.opts;
}
const handoff = await readJsonIfExists<{
completionOwner?: unknown;
timeout?: { version?: unknown; serialized?: unknown; operator?: unknown };
}>(path.join(path.dirname(params.resultPath), "handoff.json"));
const timeout = handoff?.timeout;
if (
handoff?.completionOwner !== "parent" ||
timeout?.version !== 1 ||
timeout.operator !== null ||
timeout.serialized !== params.opts.timeout ||
!parseStrictPositiveInteger(params.opts.timeout)
) {
const handoff = await readJsonIfExists<unknown>(
path.join(path.dirname(params.resultPath), "handoff.json"),
);
if (!isOmittedUpdateTimeout(params.opts.timeout, handoff)) {
// Shipped parents have no provenance. Their received deadline remains explicit-looking.
return params.opts;
}
@ -380,7 +376,8 @@ export async function continuePostCoreUpdateInFreshProcess(params: {
}
// Older targets need the existing allowance. New targets recover operator intent
// from the private handoff instead of treating this compatibility value as explicit.
const serializedTimeout = params.opts.timeout ?? String(Math.ceil(params.timeoutMs / 1000));
const handoff = createUpdateTimeoutHandoff(params.opts.timeout, params.timeoutMs);
const serializedTimeout = handoff.timeout.serialized;
argv.push("--timeout", serializedTimeout);
const resultDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-update-post-core-"));
const resultPath = path.join(resultDir, "plugins.json");
@ -422,18 +419,7 @@ export async function continuePostCoreUpdateInFreshProcess(params: {
}
await writePostCorePluginInstallRecordsFile(installRecordsPath, pluginInstallRecords);
await writePostCoreSourceConfigFile(sourceConfigPath, params.preUpdateConfig);
await writeJson(
path.join(resultDir, "handoff.json"),
{
completionOwner: "parent",
timeout: {
version: 1,
serialized: serializedTimeout,
operator: params.opts.timeout ?? null,
},
},
{ dirMode: 0o700 },
);
await writeJson(path.join(resultDir, "handoff.json"), handoff, { dirMode: 0o700 });
const jsonMode = params.opts.json === true;
const childStdio = resolvePostCoreUpdateChildStdio(process.platform, jsonMode);
const handoffEnv = buildPostCoreHandoffEnv({

View file

@ -502,10 +502,13 @@ async function updateCommandInternal(
const activateCurrentCore = async () => {
run.executorFence = await executor.enter(root, {
preflight: true,
activationTimeoutMs: (run.activationTimeoutMs ??= await resolveUpdateFinalizationTimeoutMs(
updateStepTimeoutMs,
{ env: run.env, pluginCount },
)),
activationTimeoutMs: (run.activationTimeoutMs ??=
timeoutMs === undefined
? undefined
: await resolveUpdateFinalizationTimeoutMs(updateStepTimeoutMs, {
env: run.env,
pluginCount,
})),
});
};
if (packageAlreadyCurrent) {

View file

@ -9,6 +9,7 @@ import { createDeferredCore } from "../shared/deferred.js";
const fixture = vi.hoisted(() => ({
close: vi.fn<() => Promise<void>>(),
budget: vi.fn(),
activation: vi.fn(),
finish: vi.fn(),
terminal: vi.fn(),
writeFile: vi.fn(),
@ -28,7 +29,11 @@ vi.mock("../cli/update-cli/update-command-executor.js", () => ({
_runId: string,
_root: string,
run: (fence: { assertCurrent: () => void }) => Promise<unknown>,
) => run({ assertCurrent: vi.fn() }),
options?: { activationTimeoutMs: number },
) => {
fixture.activation(options);
return run({ assertCurrent: vi.fn() });
},
withUpdateCommandExecutor: async (
_runId: string,
run: (executor: { enter: () => Promise<typeof fixture.fence> }) => Promise<unknown>,
@ -262,3 +267,118 @@ it("binds migrated worker finalization to its local candidate runtime", async ()
{ mode: 0o600 },
);
});
it.each([
{
name: "supported omission",
version: 1,
operator: null,
owner: "parent",
serialized: "1800",
expected: undefined,
},
{
name: "explicit deadline",
version: 1,
operator: "1800",
owner: "parent",
serialized: "1800",
expected: 10_800_000,
},
{
name: "unknown version",
version: 2,
operator: null,
owner: "parent",
serialized: "1800",
expected: 10_800_000,
},
{
name: "mismatched serialization",
version: 1,
operator: null,
owner: "parent",
serialized: "900",
expected: 10_800_000,
},
{
name: "malformed operator",
version: 1,
operator: false,
owner: "parent",
serialized: "1800",
expected: 10_800_000,
},
{
name: "wrong completion owner",
version: 1,
operator: null,
owner: "child",
serialized: "1800",
expected: 10_800_000,
},
{ name: "legacy producer", expected: 10_800_000 },
{
name: "inherited explicit allowance",
version: 1,
operator: null,
owner: "parent",
serialized: "1800",
inherited: 12_345,
expected: 12_345,
},
])("preserves aggregate deadline intent for $name", async (row) => {
const input = {
executor: {},
completionOwner: row.owner,
timeout:
row.version === undefined
? undefined
: {
version: row.version,
serialized: row.serialized,
operator: row.operator,
},
bufferedSteps: [],
resultPath: "/synthetic/result.json",
params: {
root: "/synthetic",
opts: {
json: true,
timeout: row.version === undefined ? undefined : "1800",
run: {
runId: "synthetic-run",
env: {},
activationTimeoutMs: row.inherited,
},
},
updateStepTimeoutMs: 1_800_000,
rollbackBlockedReason: "state-migrated-no-rollback",
preUpdatePluginInstallRecords: {},
result,
},
};
const settled = createDeferredCore();
fixture.close.mockImplementation(async () => settled.resolve());
fixture.budget.mockResolvedValue(10_800_000);
fixture.finish.mockResolvedValue(result);
fixture.terminal.mockReturnValue({ runId: "synthetic-run", status: "ok" });
process.argv = [process.execPath, "update-migrated-finalize.worker.js"];
vi.spyOn(process.stdin, Symbol.asyncIterator).mockImplementation(async function* () {
yield JSON.stringify(input);
return undefined;
});
await import("./update-migrated-finalize.worker.js");
await settled.promise;
expect(process.exitCode).toBe(originalExitCode);
expect(fixture.activation).toHaveBeenCalledExactlyOnceWith(
row.expected === undefined ? undefined : { activationTimeoutMs: row.expected },
);
expect(fixture.budget).toHaveBeenCalledTimes(
row.expected === undefined || row.inherited !== undefined ? 0 : 1,
);
expect(fixture.finish).toHaveBeenCalledOnce();
expect(fixture.writeFile).toHaveBeenCalledOnce();
});

View file

@ -40,6 +40,7 @@ import {
import { adoptUpdateRun, getUpdateRun, recordUpdateRunStep } from "./update-run-ledger.js";
import type { UpdateRunRecord } from "./update-run-record.js";
import type { UpdateRecoveryFence } from "./update-run-recovery.js";
import { isOmittedUpdateTimeout } from "./update-timeout-provenance.js";
async function finalizeMigratedUpdate(): Promise<void> {
// Validation imports this whole candidate graph before activation. The helper
@ -84,13 +85,19 @@ async function finalizeMigratedUpdate(): Promise<void> {
"Full-state checkpoint recovery is deferred; retained state was left unchanged.",
);
}
const omittedOperatorTimeout = isOmittedUpdateTimeout(input.params.opts.timeout, input);
if (omittedOperatorTimeout) {
input.params.opts.timeout = undefined;
}
const activationTimeoutMs =
input.params.opts.run?.activationTimeoutMs ??
(await resolveUpdateFinalizationTimeoutMs(input.params.updateStepTimeoutMs, {
env: input.params.ownedManagedUpdateEnv ?? input.params.opts.run?.env,
databases: input.params.schemaVersions,
pluginCount: Object.keys(input.params.preUpdatePluginInstallRecords).length,
}));
(omittedOperatorTimeout
? undefined
: await resolveUpdateFinalizationTimeoutMs(input.params.updateStepTimeoutMs, {
env: input.params.ownedManagedUpdateEnv ?? input.params.opts.run?.env,
databases: input.params.schemaVersions,
pluginCount: Object.keys(input.params.preUpdatePluginInstallRecords).length,
}));
let publishedRecord: UpdateRunRecord | undefined;
const finalized = await withUpdateCommandTerminalResult(
async (registerRun) => {
@ -100,9 +107,7 @@ async function finalizeMigratedUpdate(): Promise<void> {
input.params.opts.run?.runId ?? "",
input.params.result.root ?? input.params.root,
async (fence) => finalizeInput(input, fence, registerRun),
{
activationTimeoutMs,
},
activationTimeoutMs === undefined ? undefined : { activationTimeoutMs },
);
}
// The shipped v2026.9.3 producer overrides these selectors for worker

View file

@ -111,6 +111,20 @@ describe("runPostCoreFinalizeAfterGatewayUpdate", () => {
expect(call.timeoutMs).toBeGreaterThanOrEqual(120_000);
});
it("leaves forward finalization unbounded when the caller omits its work deadline", async () => {
const spawnFinalize = vi.fn<PostCoreFinalizeSpawner>(async () => ({ code: 0 }));
await expect(
runPostCoreFinalizeAfterGatewayUpdate({
result: gitOkResult(),
resolveEntrypoint: resolveEntrypointOk,
spawnFinalize,
}),
).resolves.toEqual({ status: "ok", entrypoint: ENTRYPOINT });
const call = expectDefined(spawnFinalize.mock.calls[0], "finalizer was started")[0];
expect(call.argv).not.toContain("--timeout");
expect(call.timeoutMs).toBeUndefined();
});
it("strips the gateway service identity from the finalizer child env", async () => {
const spawnFinalize = vi.fn<PostCoreFinalizeSpawner>(async () => ({ code: 0 }));
await runPostCoreFinalizeAfterGatewayUpdate({
@ -235,8 +249,7 @@ fs.writeFileSync(process.env.OPENCLAW_TEST_OUTPUT_PATH, JSON.stringify({
expect(call.env.OPENCLAW_UPDATE_EFFECTIVE_CHANNEL).toBe("dev");
expect(call.argv).not.toContain("--channel");
expect(call.argv).not.toContain("--timeout");
// Doctor has no separate automatic deadline; the enclosing activation is bounded.
expect(call.timeoutMs).toBeGreaterThanOrEqual(20 * 60_000);
expect(call.timeoutMs).toBeUndefined();
});
it("passes and removes the pre-update config payload for channel restoration", async () => {

View file

@ -106,7 +106,7 @@ type FinalizeSpawnResult = { code: number | null; stdout?: string; stderr?: stri
type PostCoreFinalizeSpawner = (params: {
argv: string[];
cwd: string;
timeoutMs: number;
timeoutMs?: number;
env: NodeJS.ProcessEnv;
}) => Promise<FinalizeSpawnResult>;
@ -196,12 +196,16 @@ export async function runPostCoreFinalizeAfterGatewayUpdate(params: {
// Pin the finalizer's host-compat resolution to the just-installed core
// version so plugins reconcile against the new core, not the running process.
const compatHostVersion = result.after?.version ?? undefined;
// Outer whole-process backstop, decoupled from the per-step `--timeout` above.
const processTimeoutMs = await resolveUpdateFinalizationTimeoutMs(perStepTimeoutMs, {
env: params.env,
pluginCount: Object.keys(params.preUpdateConfig?.sourceConfig.plugins?.entries ?? {}).length,
nodeRunner: nodePath,
});
// An omitted operator deadline must not become an outer work deadline.
const processTimeoutMs =
perStepTimeoutMs === undefined
? undefined
: await resolveUpdateFinalizationTimeoutMs(perStepTimeoutMs, {
env: params.env,
pluginCount: Object.keys(params.preUpdateConfig?.sourceConfig.plugins?.entries ?? {})
.length,
nodeRunner: nodePath,
});
let sourceConfigDir: string | undefined;
try {

View file

@ -0,0 +1,37 @@
import { parseStrictPositiveInteger } from "@openclaw/normalization-core/number-coercion";
import { isRecord } from "@openclaw/normalization-core/record-coerce";
export type UpdateTimeoutHandoff = {
completionOwner: "parent";
timeout: { version: 1; serialized: string; operator: string | null };
};
/** Keep a compatibility allowance for shipped receivers and retain the caller's intent. */
export function createUpdateTimeoutHandoff(
operatorTimeout: string | undefined,
fallbackTimeoutMs: number,
): UpdateTimeoutHandoff {
return {
completionOwner: "parent",
timeout: {
version: 1,
serialized: operatorTimeout ?? String(Math.ceil(fallbackTimeoutMs / 1000)),
operator: operatorTimeout ?? null,
},
};
}
/** Unknown or mismatched private input cannot remove a shipped caller's deadline. */
export function isOmittedUpdateTimeout(serialized: string | undefined, handoff: unknown): boolean {
if (!serialized || !parseStrictPositiveInteger(serialized) || !isRecord(handoff)) {
return false;
}
const timeout = handoff.timeout;
return (
handoff.completionOwner === "parent" &&
isRecord(timeout) &&
timeout.version === 1 &&
timeout.operator === null &&
timeout.serialized === serialized
);
}