fix(update): release repair leases before recovery maintenance (#157733)

* fix(update): release repair leases before recovery maintenance

* fix(update): keep maintenance tool name module-private

* fix(update): end agent deadline before maintenance handoff
This commit is contained in:
Jason (Json) 2026-09-24 20:58:43 -06:00 • committed by GitHub
parent afa3ba3ae4
commit 136ce55f29
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 1238 additions and 85 deletions

View file

@ -171,6 +171,13 @@ because the updater owns service changes. They preserve an operator's
coordinators and agent-database lease checks. An external deployment owner still
owns stopping and restarting its Gateway.
Automatic repair finishes its embedded agent turn and releases that turn's database
and process resources before asking the update owner to run Doctor or update repair.
This prevents the repair agent's own credential writes from blocking maintenance.
Other live agent leases still block repair. Maintenance preserves the original
Gateway activation intent, including `--no-restart` and intentional stops. A
successful maintenance command alone does not verify the original symptom.
Repair invoked within the owning update can continue when its inherited run ID
and live process identity match that owner. Standalone repair records the same
continuation for its new run and passes that run ID to its Doctor children.

View file

@ -6,6 +6,7 @@ import { formatErrorMessage } from "../../../infra/errors.js";
import type { ProviderRouteOverridePresence } from "../../../plugin-sdk/provider-model-types.js";
import { resolveProviderModelRoutes } from "../../../plugins/provider-model-routes.js";
import { looksLikeSecretSentinel, resolveSecretSentinel } from "../../../secrets/sentinel.js";
import { getOpenClawDatabaseMaintenanceScope } from "../../../state/openclaw-state-db-async-lifecycle.js";
import type { AuthProfileStore } from "../../auth-profiles.js";
import { markAuthProfileSuccess } from "../../auth-profiles.js";
import { recordRuntimeAuthMaterialization } from "../../auth-profiles/runtime-materializations.js";
@ -41,7 +42,7 @@ export function markEmbeddedRunAuthProfileSuccess(input: {
const successProvider =
input.profileStore.profiles[successProfileId]?.provider.trim() || input.provider;
const successStarted = Date.now();
void markAuthProfileSuccess({
const bookkeeping = markAuthProfileSuccess({
store: input.profileStore,
provider: successProvider,
profileId: successProfileId,
@ -70,6 +71,9 @@ export function markEmbeddedRunAuthProfileSuccess(input: {
`error=${formatErrorMessage(error)}`,
);
});
// Capture the entire operation before it waits in the auth writer queue.
// Ordinary turns remain nonblocking; a repair owner must join it before Doctor.
void getOpenClawDatabaseMaintenanceScope()?.track(bookkeeping);
}
export function reportEmbeddedRunSuccessfulAuthBinding(input: {

View file

@ -0,0 +1,128 @@
import { exitCliAfterOutput } from "../cli/one-shot-exit.js";
import {
withInstallationTarget,
type InstallationTarget,
} from "../infra/installation-target-context.js";
import { redactSupportString } from "../logging/diagnostic-support-redaction.js";
import type { RuntimeEnv } from "../runtime.js";
/** Automatic triage owns maintenance only after the embedded turn has fully settled. */
export async function runAutomaticTriageRepair(params: {
runtime: RuntimeEnv;
target: InstallationTarget;
targetEnv: NodeJS.ProcessEnv;
installRoot: string;
prompt: string;
signal: AbortSignal;
allowGatewayActivation: boolean;
isCurrent: () => boolean;
formatError: (error: unknown) => string;
}): Promise<void> {
const { runtime, target, targetEnv, prompt, isCurrent } = params;
const redaction = { env: targetEnv, stateDir: target.stateDir };
const deadline = Date.now() + 600_000;
const controller = new AbortController();
const signal = AbortSignal.any([params.signal, controller.signal]);
const timer = setTimeout(
() => controller.abort(new Error("Automatic triage timed out.")),
600_000,
);
try {
const result = await withInstallationTarget(target, async () => {
const { prepareUpdateRepairInference, runUpdateRepairTurn } =
await import("../infra/update-repair-agent.runtime.js");
if (!isCurrent()) {
return {
status: "unavailable" as const,
reason: "Repair authority is no longer current.",
};
}
const selected = await prepareUpdateRepairInference(
signal,
Math.max(1, deadline - Date.now()),
);
if (!isCurrent()) {
return {
status: "unavailable" as const,
reason: "Repair authority is no longer current.",
};
}
if (!selected.ok) {
return { status: "unavailable" as const, reason: selected.reason };
}
signal.throwIfAborted();
return runUpdateRepairTurn({
target: {
stateDir: target.stateDir,
configPath: target.configPath,
workspaceDir: target.defaultWorkspaceDir,
installRoot: params.installRoot,
},
route: selected.route,
modelFallbacks: selected.modelFallbacks,
prompt,
signal,
timeoutMs: Math.max(1, deadline - Date.now()),
maxToolCalls: 40,
isCurrent,
maintenanceHandoff: true,
});
});
// Inference is bounded; settled maintenance keeps its own phase budgets and
// remains cancellable by the original owner, not by the expired agent timer.
clearTimeout(timer);
if (result.status === "unavailable") {
runtime.error(params.formatError(result.reason));
exitCliAfterOutput(runtime, controller.signal.aborted ? 2 : 1);
}
if (result.envelope.final) {
runtime.log(redactSupportString(result.envelope.final, redaction, { maxLength: 32 * 1024 }));
}
if (result.envelope.error?.message) {
runtime.error(params.formatError(result.envelope.error.message));
}
if (controller.signal.aborted || result.envelope.status !== "ok") {
exitCliAfterOutput(
runtime,
controller.signal.aborted || result.envelope.status === "timeout" ? 2 : 1,
);
}
if (result.maintenance) {
const { runUpdateRepairMaintenance } = await import("../infra/update-repair-maintenance.js");
const maintenance = await runUpdateRepairMaintenance({
request: result.maintenance,
target: {
stateDir: target.stateDir,
configPath: target.configPath,
workspaceDir: target.defaultWorkspaceDir,
installRoot: params.installRoot,
},
env: targetEnv,
allowGatewayActivation: params.allowGatewayActivation,
signal: params.signal,
assertCurrent: () => {
if (!isCurrent()) {
throw new Error("Repair authority is no longer current.");
}
},
});
for (const output of [maintenance.stdout, maintenance.stderr]) {
if (output.trim()) {
runtime.log(redactSupportString(output, redaction, { maxLength: 32 * 1024 }));
}
}
params.signal.throwIfAborted();
if (!isCurrent() || maintenance.termination !== "exit" || maintenance.code !== 0) {
runtime.error(
"Updater-owned maintenance did not complete; use the manual recovery command above.",
);
exitCliAfterOutput(runtime, 1);
}
runtime.log(
"Maintenance completed. Verify the original symptom and intended Gateway state before claiming recovery.",
);
}
} finally {
clearTimeout(timer);
}
}

View file

@ -116,6 +116,7 @@ export function renderTriagePrompt(params: {
redaction: SupportRedactionContext;
updateFailure?: TriageUpdateFailure;
failure?: TriageFailureContext;
maintenanceHandoff?: true;
}): string {
const { bundle, redaction, failure } = params;
const findings = params.findings.toSorted((left, right) => {
@ -124,7 +125,7 @@ export function renderTriagePrompt(params: {
return severity || left.checkId.localeCompare(right.checkId);
});
const lines = [
"You are repairing THIS machine's OpenClaw installation. Diagnose the root cause, apply the repair autonomously within your existing permissions, and verify the result. Preserve configuration, history, and databases. Use local `openclaw doctor`, `openclaw doctor --fix`, `openclaw status --all`, and `openclaw logs` as needed. Product documentation: https://docs.openclaw.ai.",
"You are repairing THIS machine's OpenClaw installation. Diagnose the root cause, apply the repair autonomously within your existing permissions, and verify the result. Preserve configuration, history, and databases. Use read-only `openclaw doctor --lint --json`, `openclaw status --all`, and `openclaw logs` for diagnostics. Product documentation: https://docs.openclaw.ai.",
"",
"## Environment",
"",
@ -138,7 +139,9 @@ export function renderTriagePrompt(params: {
"",
"## Completion goal",
"",
"Diagnose and repair the original symptom using existing repair commands, including `openclaw doctor --fix` and, for unfinished updates, `openclaw update repair`. Respect installation ownership, locks, schema and capability approval refusals. If maintenance refuses to stop the Gateway from this fixing subtree, use read-only diagnosis or safe offline artifact repair and atomic restart, or report that an independent operator must run maintenance outside triage. Do not bypass the refusal.",
params.maintenanceHandoff
? "Diagnose and repair the original symptom. Never run Doctor maintenance or update repair through exec: this turn owns live credential database resources. Call request_update_maintenance with operation doctor-fix or update-repair to end the turn; the update owner will run that fixed command only after agent work and database resources settle. Respect all lease, service, schema, and capability refusals. The tool is a request, not proof of recovery."
: "Diagnose and repair the original symptom using existing repair commands, including `openclaw doctor --fix` and, for unfinished updates, `openclaw update repair`. Respect installation ownership, locks, schema and capability approval refusals. If maintenance refuses to stop the Gateway from this fixing subtree, use read-only diagnosis or safe offline artifact repair and atomic restart, or report that an independent operator must run maintenance outside triage. Do not bypass the refusal.",
failure?.gateway === "preserve"
? "Do not start or restart the Gateway: this invocation did not authorize activation. Preserve --no-restart and intentional stops. Use read-only status checks and report live health verification as deferred while it is intentionally stopped."
: "Only activate a Gateway intended to run. For managed recovery, use atomic `openclaw gateway restart` when needed, never stop then start: an explicit stop after native scope attachment cancels this recovery and its children. Preserve later operator stops and report cancellation or infeasibility instead of claiming recovery.",

View file

@ -0,0 +1,201 @@
import path from "node:path";
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { createManagedHandoffTestBinding } from "../../test/helpers/managed-handoff-isolation.js";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import { resolveManagedUpdateLeaseDatabasePath } from "../infra/update-managed-service-handoff-lease.js";
import { createDeferredCore } from "../shared/deferred.js";
import { runAutomaticTriageRepair } from "./triage-automatic-repair.js";
import { triageCommand } from "./triage.js";
import {
createTriageInferenceSelection,
createTriageRuntime,
withTriageTerminal,
} from "./triage.test-support.js";
const mocks = vi.hoisted(() => ({
turn: vi.fn(),
selected: vi.fn(),
maintenance: vi.fn(),
handoff: undefined as ReturnType<typeof createManagedHandoffTestBinding> | undefined,
}));
vi.mock("./doctor-lint.js", () => ({ collectDoctorFindings: async () => [] }));
vi.mock("./triage-update.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./triage-update.js")>()),
readPendingTriageUpdateFailure: async () => undefined,
}));
vi.mock("../config/config.js", () => ({
readConfigFileSnapshot: async () => ({
exists: true,
valid: true,
config: { agents: { defaults: { model: "fixture/repair" } } },
}),
}));
vi.mock("../infra/executable-path.js", () => ({ resolveExecutablePath: () => undefined }));
vi.mock("../infra/update-repair-agent.runtime.js", () => ({
prepareUpdateRepairInference: mocks.selected,
runUpdateRepairTurn: mocks.turn,
}));
vi.mock("../infra/update-repair-maintenance.js", () => ({
runUpdateRepairMaintenance: mocks.maintenance,
}));
vi.mock("../infra/tmp-openclaw-dir.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../infra/tmp-openclaw-dir.js")>()),
resolvePreferredOpenClawTmpDir: () => {
if (!mocks.handoff) {
throw new Error("Private handoff binding required");
}
mocks.handoff.assertPath();
return mocks.handoff.directory;
},
}));
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
let stateDir: string;
beforeEach(() => {
vi.clearAllMocks();
stateDir = tempDirs.make("triage-maintenance-");
mocks.handoff = createManagedHandoffTestBinding(stateDir);
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
vi.stubEnv("OPENCLAW_CONFIG_PATH", path.join(stateDir, "openclaw.json"));
vi.stubEnv(
"NODE_OPTIONS",
[process.env.NODE_OPTIONS, mocks.handoff.nodeOption].filter(Boolean).join(" "),
);
mocks.handoff.assertPath(resolveManagedUpdateLeaseDatabasePath());
mocks.selected.mockResolvedValue(createTriageInferenceSelection(stateDir));
mocks.maintenance.mockResolvedValue({
termination: "exit",
code: 0,
stdout: "Maintenance result",
stderr: "",
});
});
afterEach(() => {
vi.unstubAllEnvs();
mocks.handoff = undefined;
});
const completed = {
status: "completed",
envelope: { status: "ok", final: "" },
maintenance: { operation: "update-repair" },
};
function run(gateway: "preserve" | "verify-running" = "preserve") {
return withTriageTerminal(false, () =>
triageCommand(
createTriageRuntime(),
{ noExport: true },
{
failure: {
kind: "update",
phase: "doctor",
error: "original failure",
installationRoot: stateDir,
gateway,
},
signal: new AbortController().signal,
assertCurrent: () => {},
},
),
);
}
it.each(["preserve", "verify-running"] as const)(
"settles the automatic repair turn before maintenance with %s intent",
async (gateway) => {
const entered = createDeferredCore();
const settled = createDeferredCore<typeof completed>();
mocks.turn.mockImplementation(async (params) => {
expect(params.maintenanceHandoff).toBe(true);
expect(params.prompt).toContain("Never run Doctor maintenance or update repair through exec");
expect(params.prompt).toContain(
gateway === "preserve"
? "Do not start or restart the Gateway"
: "Only activate a Gateway intended to run",
);
entered.resolve();
return settled.promise;
});
const pending = run(gateway);
await entered.promise;
expect(mocks.maintenance).not.toHaveBeenCalled();
settled.resolve(completed);
await pending;
expect(mocks.maintenance).toHaveBeenCalledExactlyOnceWith(
expect.objectContaining({
request: { operation: "update-repair" },
allowGatewayActivation: gateway === "verify-running",
target: expect.objectContaining({ installRoot: stateDir, stateDir }),
}),
);
},
);
it.each(["error", "timeout", "cleanup"])(
"never starts maintenance after a %s turn",
async (failure) => {
if (failure === "cleanup") {
mocks.turn.mockRejectedValue(new Error("resource cleanup failed"));
} else {
mocks.turn.mockResolvedValue({ ...completed, envelope: { status: failure, final: "" } });
}
await expect(run()).rejects.toBeDefined();
expect(mocks.maintenance).not.toHaveBeenCalled();
},
);
it("preserves a failed maintenance command as failure", async () => {
mocks.turn.mockResolvedValue(completed);
mocks.maintenance.mockResolvedValue({
termination: "exit",
code: 1,
stdout: "",
stderr: "Agent database is still open",
});
await expect(run()).rejects.toMatchObject({ code: 1 });
expect(mocks.maintenance).toHaveBeenCalledOnce();
});
it.each([false, true])(
"maintenance outlives the agent deadline but retains owner cancellation (cancel=%s)",
async (cancel) => {
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
const owner = new AbortController();
mocks.turn.mockImplementation(async () => {
await vi.advanceTimersByTimeAsync(599_999);
return completed;
});
mocks.maintenance.mockImplementation(async ({ signal }: { signal: AbortSignal }) => {
await vi.advanceTimersByTimeAsync(2);
expect(signal.aborted).toBe(false);
if (cancel) {
owner.abort(new Error("owner cancelled"));
expect(signal.aborted).toBe(true);
}
return { termination: "exit", code: 0, stdout: "", stderr: "" };
});
try {
const pending = runAutomaticTriageRepair({
runtime: createTriageRuntime(),
target: {
stateDir,
configPath: path.join(stateDir, "openclaw.json"),
defaultWorkspaceDir: stateDir,
},
targetEnv: { OPENCLAW_STATE_DIR: stateDir },
installRoot: stateDir,
prompt: "Repair.",
signal: owner.signal,
allowGatewayActivation: false,
isCurrent: () => true,
formatError: String,
});
if (cancel) {
await expect(pending).rejects.toThrow("owner cancelled");
} else {
await pending;
}
expect(mocks.maintenance).toHaveBeenCalledOnce();
} finally {
vi.useRealTimers();
}
},
);

View file

@ -23,7 +23,6 @@ import { resolveExecutablePath } from "../infra/executable-path.js";
import {
installationTargetEnv,
resolveInstallationTarget,
withInstallationTarget,
type InstallationTarget,
} from "../infra/installation-target-context.js";
import { resolveOpenClawPackageRoot } from "../infra/openclaw-root.js";
@ -229,13 +228,7 @@ export async function triageCommand(
const bundle: TriageBundle = deferDiagnostics
? { kind: "deferred" }
: await collectTriageBundle(options.noExport === true, redaction);
const prompt = renderTriagePrompt({
findings,
bundle,
redaction,
updateFailure,
failure: automatic?.failure,
});
// Packaged OpenClaw/Bun hosts cannot interpret npm shim entrypoints. Reuse the
// active Node runtime or require an installed node.exe before choosing a shim.
const nodeExecutable = isNodeRuntime(process.execPath)
@ -281,6 +274,16 @@ export async function triageCommand(
resolveAgentEffectiveModelPrimary(config, agentId),
);
}
const prompt = renderTriagePrompt({
findings,
bundle,
redaction,
updateFailure,
failure: automatic?.failure,
...(runEmbedded && automatic && !automatic.diagnosticOnly
? { maintenanceHandoff: true as const }
: {}),
});
const canStartAgent = allowAgent && (runEmbedded || handoff !== undefined);
const now = new Date().toISOString().replace(/[:.]/gu, "-");
const outputDir = path.join(target.stateDir, "logs", "support");
@ -542,75 +545,18 @@ export async function triageCommand(
}
if (automatic && !automatic.diagnosticOnly) {
const deadline = Date.now() + 600_000;
const controller = new AbortController();
const signal = AbortSignal.any([automatic.signal, controller.signal]);
const timer = setTimeout(
() => controller.abort(new Error("Automatic triage timed out.")),
600_000,
);
try {
const result = await withInstallationTarget(target, async () => {
const { prepareUpdateRepairInference, runUpdateRepairTurn } =
await import("../infra/update-repair-agent.runtime.js");
if (!isCurrent()) {
return {
status: "unavailable" as const,
reason: "Repair authority is no longer current.",
};
}
const selected = await prepareUpdateRepairInference(
signal,
Math.max(1, deadline - Date.now()),
);
if (!isCurrent()) {
return {
status: "unavailable" as const,
reason: "Repair authority is no longer current.",
};
}
if (!selected.ok) {
return { status: "unavailable" as const, reason: selected.reason };
}
signal.throwIfAborted();
return runUpdateRepairTurn({
target: {
stateDir: target.stateDir,
configPath: target.configPath,
workspaceDir: target.defaultWorkspaceDir,
installRoot: agentCwd ?? process.cwd(),
},
route: selected.route,
modelFallbacks: selected.modelFallbacks,
prompt,
signal,
timeoutMs: Math.max(1, deadline - Date.now()),
maxToolCalls: 40,
isCurrent,
});
});
if (result.status === "unavailable") {
runtime.error(triageCollectionError(result.reason, redaction));
exitCliAfterOutput(runtime, controller.signal.aborted ? 2 : 1);
}
if (result.envelope.final) {
runtime.log(
redactSupportString(result.envelope.final, redaction, { maxLength: 32 * 1024 }),
);
}
if (result.envelope.error?.message) {
runtime.error(triageCollectionError(result.envelope.error.message, redaction));
}
if (controller.signal.aborted || result.envelope.status !== "ok") {
exitCliAfterOutput(
runtime,
controller.signal.aborted || result.envelope.status === "timeout" ? 2 : 1,
);
}
} finally {
clearTimeout(timer);
}
return;
const { runAutomaticTriageRepair } = await import("./triage-automatic-repair.js");
return runAutomaticTriageRepair({
runtime,
target,
targetEnv,
prompt,
isCurrent,
installRoot: agentCwd ?? process.cwd(),
signal: automatic.signal,
allowGatewayActivation: automatic.failure.gateway === "verify-running",
formatError: (error) => triageCollectionError(error, redaction),
});
}
const { runUpdateRepairLoop } = await import("../infra/update-repair-agent.js");

View file

@ -0,0 +1,40 @@
import { expect, it, vi } from "vitest";
import { markEmbeddedRunAuthProfileSuccess } from "../agents/embedded-agent-runner/run/auth-profile-success.js";
import { createDeferredCore } from "../shared/deferred.js";
import { createOpenClawDatabaseMaintenanceScope } from "../state/openclaw-state-db-async-lifecycle.js";
const auth = vi.hoisted(() => ({ success: vi.fn() }));
vi.mock("../agents/auth-profiles.js", () => ({ markAuthProfileSuccess: auth.success }));
it.each([false, true])(
"settles deferred auth bookkeeping before retiring repair resources (failure=%s)",
async (fails) => {
const pending = createDeferredCore();
auth.success.mockReturnValueOnce(pending.promise);
const resources = createOpenClawDatabaseMaintenanceScope();
const retired = vi.fn();
resources.own({}, "agent-handles", retired);
const result = resources.run(() =>
markEmbeddedRunAuthProfileSuccess({
profileId: "fixture:repair",
profileStore: { version: 1, profiles: {} },
provider: "fixture",
runId: "fixture-run",
sessionId: "fixture-session",
}),
);
expect(result).toBeUndefined();
const closed = resources.close();
try {
expect(retired).not.toHaveBeenCalled();
} finally {
if (fails) {
pending.reject(new Error("synthetic bookkeeping refusal"));
} else {
pending.resolve();
}
await closed;
}
expect(retired).toHaveBeenCalledOnce();
},
);

View file

@ -0,0 +1,188 @@
import { afterEach, expect, it, vi } from "vitest";
import { createManagedHandoffTestBinding } from "../../test/helpers/managed-handoff-isolation.js";
import { createExternalAuthRuntime } from "../agents/auth-profiles/external-auth.js";
import { createAuthProfileStoreRuntime } from "../agents/auth-profiles/store.js";
import type { StreamFn } from "../agents/runtime/index.js";
import { beginDoctorMaintenance } from "../commands/doctor-maintenance.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { createAssistantMessageEventStream } from "../llm/utils/event-stream.js";
import { readActiveOpenClawAgentDatabaseLeasesReadOnly } from "../state/openclaw-agent-db-lease.js";
import { createOpenClawDatabaseMaintenanceScope } from "../state/openclaw-state-db-async-lifecycle.js";
import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js";
import { resolveManagedUpdateLeaseDatabasePath } from "./update-managed-service-handoff-lease.js";
import { runUpdateRepairTurn } from "./update-repair-agent.runtime.js";
const fixture = vi.hoisted(() => ({
stream: vi.fn<StreamFn>(),
handoff: undefined as ReturnType<typeof createManagedHandoffTestBinding> | undefined,
}));
// Only inference is synthetic; admission, tool preparation/execution, terminal
// resolution, auth bookkeeping, and database lifecycle are production owners.
vi.mock("../agents/provider-stream.js", () => ({
registerProviderStreamForModel: () => fixture.stream,
}));
vi.mock("./tmp-openclaw-dir.js", () => ({
resolvePreferredOpenClawTmpDir: () => {
if (!fixture.handoff) {
throw new Error("Private handoff binding required");
}
fixture.handoff.assertPath();
return fixture.handoff.directory;
},
}));
afterEach(() => {
vi.unstubAllEnvs();
fixture.handoff = undefined;
fixture.stream.mockReset();
});
it("executes the real terminal client tool before settling credentials and admitting Doctor", async () => {
await withOpenClawTestState({ layout: "home" }, async (state) => {
fixture.handoff = createManagedHandoffTestBinding(state.root);
vi.stubEnv(
"NODE_OPTIONS",
[process.env.NODE_OPTIONS, fixture.handoff.nodeOption].filter(Boolean).join(" "),
);
expect(fixture.handoff.assertPath(resolveManagedUpdateLeaseDatabasePath())).toBe(
fixture.handoff.databasePath,
);
vi.stubEnv("OPENCLAW_SERVICE_REPAIR_POLICY", "external");
const agentDir = state.agentDir("owner");
const auth = createAuthProfileStoreRuntime(createExternalAuthRuntime(() => []));
const seed = createOpenClawDatabaseMaintenanceScope();
try {
seed.run(() =>
auth.saveAuthProfileStore(
{
version: 1,
profiles: {
"fixture:repair": { type: "token", provider: "fixture", token: "synthetic-token" },
},
},
agentDir,
),
);
} finally {
await seed.close();
}
expect(readActiveOpenClawAgentDatabaseLeasesReadOnly({ env: state.env })).toEqual([]);
const config: OpenClawConfig = {
plugins: { enabled: false },
agents: { defaults: { skipBootstrap: true }, entries: { owner: { agentDir } } },
tools: { toolSearch: { enabled: false } },
models: {
providers: {
fixture: {
baseUrl: "https://repair.invalid/v1",
api: "openai-responses",
models: [
{
id: "repair",
name: "Synthetic repair",
reasoning: false,
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 128000,
maxTokens: 1024,
},
],
},
},
},
};
await state.writeConfig(config);
fixture.stream.mockImplementation((model, context) => {
expect(context.tools?.map((tool) => tool.name)).toContain("request_update_maintenance");
const stream = createAssistantMessageEventStream();
queueMicrotask(() => {
stream.push({
type: "done",
reason: "toolUse",
message: {
role: "assistant",
content: [
{
type: "toolCall",
id: "maintenance-request",
name: "request_update_maintenance",
arguments: { operation: "doctor-fix" },
},
],
api: model.api,
provider: model.provider,
model: model.id,
usage: {
input: 1,
output: 1,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 2,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "toolUse",
timestamp: Date.now(),
},
});
stream.end();
});
return stream;
});
// Cold module loading is outside the deterministic model turn budget. The
// automatic caller allows ten minutes; this fixture exercises one immediate inference.
await Promise.all([
import("../agents/embedded-agent.js"),
import("../agents/embedded-agent-runner/run-entry.js"),
]);
const result = await runUpdateRepairTurn({
target: { ...state, installRoot: state.workspaceDir },
route: {
runner: "embedded",
provider: "fixture",
model: "repair",
modelLabel: "fixture/repair",
agentId: "owner",
agentDir,
authProfileId: "fixture:repair",
runConfig: config,
sourceConfig: config,
},
modelFallbacks: [],
prompt: "Request Doctor repair using the maintenance tool.",
timeoutMs: 30000,
maxToolCalls: 1,
signal: new AbortController().signal,
maintenanceHandoff: true,
});
expect(result, JSON.stringify(result)).toMatchObject({
status: "completed",
envelope: { status: "ok" },
maintenance: { operation: "doctor-fix" },
});
expect(fixture.stream).toHaveBeenCalledTimes(1);
// These checks must precede fixture teardown: cleanup must not hide a leaked lease.
expect(readActiveOpenClawAgentDatabaseLeasesReadOnly({ env: state.env })).toEqual([]);
const saved = auth.loadAuthProfileStoreForRuntime(agentDir, {
readOnly: true,
externalCli: { mode: "none" },
});
expect(saved.usageStats?.["fixture:repair"]?.lastUsed).toBeGreaterThan(0);
const doctor = await beginDoctorMaintenance({
options: { repair: true, nonInteractive: true },
root: null,
runtime: { log() {}, error() {}, exit() {} },
});
expect(doctor).toBeDefined();
try {
doctor!.run(() => auth.saveAuthProfileStore(saved, agentDir));
} finally {
await doctor?.release();
}
expect(readActiveOpenClawAgentDatabaseLeasesReadOnly({ env: state.env })).toEqual([]);
expect(
auth.loadAuthProfileStoreForRuntime(agentDir, {
readOnly: true,
externalCli: { mode: "none" },
}),
).toEqual(saved);
});
});

View file

@ -1,23 +1,47 @@
import { execFileSync } from "node:child_process";
import fs from "node:fs/promises";
import path from "node:path";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { createManagedHandoffTestBinding } from "../../test/helpers/managed-handoff-isolation.js";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import { captureAgentToolExecutionBudget } from "../agents/agent-tool-source-execution-guard.js";
import { createExternalAuthRuntime } from "../agents/auth-profiles/external-auth.js";
import { createAuthProfileStoreRuntime } from "../agents/auth-profiles/store.js";
import type { RunEmbeddedAgentParams } from "../agents/embedded-agent-runner/run/params.js";
import { createAgentCleanupScope } from "../agents/run-cleanup-timeout.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { getOpenClawDatabaseMaintenanceScope } from "../state/openclaw-state-db-async-lifecycle.js";
import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js";
import { getInstallationTarget } from "./installation-target-context.js";
import { openNodeSqliteDatabase } from "./node-sqlite.js";
import { resolveManagedUpdateLeaseDatabasePath } from "./update-managed-service-handoff-lease.js";
import {
prepareUpdateRepairInference,
runUpdateRepairTurn,
withUpdateRepairEnvironment,
} from "./update-repair-agent.runtime.js";
const mocks = vi.hoisted(() => ({ entry: vi.fn(), run: vi.fn(), cleanup: vi.fn() }));
const mocks = vi.hoisted(() => ({
entry: vi.fn(),
run: vi.fn(),
cleanup: vi.fn(),
handoff: undefined as ReturnType<typeof createManagedHandoffTestBinding> | undefined,
}));
vi.mock("./tmp-openclaw-dir.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./tmp-openclaw-dir.js")>()),
resolvePreferredOpenClawTmpDir: () => {
if (!mocks.handoff) {
throw new Error("Private handoff binding required");
}
mocks.handoff.assertPath();
return mocks.handoff.directory;
},
}));
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
afterEach(() => {
vi.unstubAllEnvs();
mocks.handoff = undefined;
});
vi.mock("../agents/embedded-agent.js", () => ({ runEmbeddedAgent: mocks.run }));
vi.mock("../agents/embedded-agent-runner/run-entry.js", () => ({
runEmbeddedAgentEntry: mocks.entry,
@ -27,6 +51,12 @@ vi.mock("../process/supervisor/index.js", () => ({
}));
beforeEach(() => {
mocks.handoff = createManagedHandoffTestBinding(tempDirs.make("repair-runtime-handoff-"));
vi.stubEnv(
"NODE_OPTIONS",
[process.env.NODE_OPTIONS, mocks.handoff.nodeOption].filter(Boolean).join(" "),
);
mocks.handoff.assertPath(resolveManagedUpdateLeaseDatabasePath());
mocks.run.mockReset();
mocks.cleanup.mockReset().mockResolvedValue(undefined);
mocks.entry
@ -170,6 +200,56 @@ describe("post-failure repair execution", () => {
},
);
it("preserves both process and database cleanup failures and refuses clean completion", async () => {
await withOpenClawTestState({ layout: "home" }, async (state) => {
const config: OpenClawConfig = { plugins: { enabled: false } };
const processFailure = new Error("Synthetic process cleanup failure");
const databaseFailure = new Error("Synthetic database cleanup failure");
const close = vi.fn().mockRejectedValueOnce(databaseFailure).mockResolvedValue(undefined);
let resources: ReturnType<typeof getOpenClawDatabaseMaintenanceScope>;
mocks.run.mockImplementation(async () => {
resources = getOpenClawDatabaseMaintenanceScope();
expect(resources).toBeDefined();
resources!.own({}, "agent-resources", close);
return { meta: { durationMs: 1 } };
});
mocks.cleanup.mockRejectedValueOnce(processFailure);
const cleanup = createAgentCleanupScope();
try {
await expect(
cleanup.run(() =>
runUpdateRepairTurn({
target: { ...state, installRoot: state.workspaceDir },
route: {
runner: "embedded",
provider: "fixture",
model: "repair",
modelLabel: "fixture/repair",
agentId: "owner",
agentDir: state.agentDir("owner"),
runConfig: config,
sourceConfig: config,
},
modelFallbacks: [],
prompt: "Repair.",
timeoutMs: 10000,
maxToolCalls: 1,
signal: new AbortController().signal,
}),
),
).rejects.toMatchObject({
message: "Repair turn and database resource cleanup failed.",
errors: [processFailure, databaseFailure],
});
expect(cleanup.outcome).toBe("uncertain");
expect(close).toHaveBeenCalledOnce();
} finally {
// The fixture owns the synthetic refusal and must retire it after the assertion.
await resources?.close();
}
});
});
it("refuses revoked repair authority before starting the runner", async () => {
await withOpenClawTestState({ layout: "home" }, async (state) => {
const config: OpenClawConfig = { plugins: { enabled: false } };

View file

@ -13,6 +13,7 @@ import { isToolAllowedByPolicies } from "../agents/tool-policy-match.js";
import { mergeAlsoAllowPolicy, resolveToolProfilePolicy } from "../agents/tool-policy.js";
import { buildExecRunConfig } from "../commands/agent-exec-input.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { createOpenClawDatabaseMaintenanceScope } from "../state/openclaw-state-db-async-lifecycle.js";
import type { SystemAgentConfiguredRoute } from "../system-agent/inference-route.js";
import { sanitizeHostExecEnv, withHostExecInheritedEnvOmitted } from "./host-env-security.js";
import {
@ -20,6 +21,11 @@ import {
withInstallationTarget,
LOCAL_INSTALLATION_TARGET_UNSUPPORTED,
} from "./installation-target-context.js";
import {
readUpdateRepairMaintenanceRequest,
updateRepairMaintenanceTool,
type UpdateRepairMaintenanceRequest,
} from "./update-repair-maintenance.js";
import type { UpdateRepairTarget } from "./update-repair-protocol.js";
import { buildUpdateDoctorEnv } from "./update-runner-doctor.js";
@ -105,7 +111,33 @@ export async function withUpdateRepairEnvironment<T>(
}
}
async function withRepairResources<T>(run: () => Promise<T>): Promise<T> {
const resources = createOpenClawDatabaseMaintenanceScope();
const [outcome] = await Promise.allSettled([Promise.resolve().then(() => resources.run(run))]);
try {
await resources.close();
} catch (error) {
recordAgentCleanupFailure();
if (outcome.status === "rejected") {
throw new AggregateError(
[outcome.reason, error],
"Repair turn and database resource cleanup failed.",
{ cause: error },
);
}
throw error;
}
if (outcome.status === "rejected") {
throw outcome.reason;
}
return outcome.value;
}
export async function prepareUpdateRepairInference(signal: AbortSignal, timeoutMs: number) {
return withRepairResources(() => prepareRepairInference(signal, timeoutMs));
}
async function prepareRepairInference(signal: AbortSignal, timeoutMs: number) {
signal.throwIfAborted();
const { getRuntimeConfig } = await import("../config/io.js");
signal.throwIfAborted();
@ -219,7 +251,7 @@ function repairRunConfig(
});
}
export async function runUpdateRepairTurn(params: {
type UpdateRepairTurnParams = {
target: UpdateRepairTarget;
route: Extract<SystemAgentConfiguredRoute, { runner: "embedded" }>;
modelFallbacks: string[];
@ -228,7 +260,14 @@ export async function runUpdateRepairTurn(params: {
maxToolCalls: number;
signal: AbortSignal;
isCurrent?: () => boolean;
}) {
maintenanceHandoff?: true;
};
export async function runUpdateRepairTurn(params: UpdateRepairTurnParams) {
return withRepairResources(() => runScopedUpdateRepairTurn(params));
}
async function runScopedUpdateRepairTurn(params: UpdateRepairTurnParams) {
params.signal.throwIfAborted();
const { route, target } = params;
const config = repairRunConfig(route, params.modelFallbacks);
@ -267,6 +306,7 @@ export async function runUpdateRepairTurn(params: {
controller.abort(new Error("per-turn-budget"));
}, params.timeoutMs);
let cleanupProcessScope: (() => Promise<void>) | undefined;
let maintenance: UpdateRepairMaintenanceRequest | undefined;
let envelope: {
model: string;
provider: string;
@ -341,6 +381,9 @@ export async function runUpdateRepairTurn(params: {
cwd: target.installRoot,
config: runConfig,
prompt: params.prompt,
clientTools: params.maintenanceHandoff
? [updateRepairMaintenanceTool]
: undefined,
provider,
model,
...(route.authProfileId && provider === route.provider
@ -360,6 +403,10 @@ export async function runUpdateRepairTurn(params: {
),
);
const error = extractAgentRunTerminalError(result.result);
if (params.maintenanceHandoff && result.terminal.outcome.status === "ok" && !error) {
assertCurrent();
maintenance = readUpdateRepairMaintenanceRequest(result.result.meta);
}
envelope = {
model: result.model,
provider: result.provider,
@ -386,5 +433,10 @@ export async function runUpdateRepairTurn(params: {
recordAgentCleanupFailure();
throw error;
}
return { status: "completed" as const, toolCalls: toolBudget.toolCalls, envelope };
return {
status: "completed" as const,
toolCalls: toolBudget.toolCalls,
envelope,
...(maintenance ? { maintenance } : {}),
};
}

View file

@ -0,0 +1,202 @@
import { afterEach, expect, it, vi } from "vitest";
import { createManagedHandoffTestBinding } from "../../test/helpers/managed-handoff-isolation.js";
import { createExternalAuthRuntime } from "../agents/auth-profiles/external-auth.js";
import { createAuthProfileStoreRuntime } from "../agents/auth-profiles/store.js";
import { markEmbeddedRunAuthProfileSuccess } from "../agents/embedded-agent-runner/run/auth-profile-success.js";
import type { RunEmbeddedAgentParams } from "../agents/embedded-agent-runner/run/params.js";
import { beginDoctorMaintenance } from "../commands/doctor-maintenance.js";
import {
assertNoOpenClawAgentDatabaseLeasesReadOnly,
readActiveOpenClawAgentDatabaseLeasesReadOnly,
} from "../state/openclaw-agent-db-lease.js";
import { createOpenClawDatabaseMaintenanceScope } from "../state/openclaw-state-db-async-lifecycle.js";
import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js";
import { resolveManagedUpdateLeaseDatabasePath } from "./update-managed-service-handoff-lease.js";
import { runUpdateRepairTurn } from "./update-repair-agent.runtime.js";
const fixture = vi.hoisted(() => ({
run: vi.fn(),
handoff: undefined as ReturnType<typeof createManagedHandoffTestBinding> | undefined,
}));
vi.mock("./tmp-openclaw-dir.js", () => ({
resolvePreferredOpenClawTmpDir: () => {
if (!fixture.handoff) {
throw new Error("Private handoff binding required before database admission");
}
fixture.handoff.assertPath();
return fixture.handoff.directory;
},
}));
vi.mock("../agents/embedded-agent.js", () => ({ runEmbeddedAgent: fixture.run }));
vi.mock("../agents/embedded-agent-runner/run-entry.js", () => ({
runEmbeddedAgentEntry: async (params: {
runCandidate: (provider: string, model: string, options: object) => Promise<unknown>;
}) => ({
result: await params.runCandidate("fixture", "repair", {}),
provider: "fixture",
model: "repair",
terminal: { outcome: { status: "ok" } },
}),
}));
afterEach(() => {
vi.unstubAllEnvs();
fixture.handoff = undefined;
fixture.run.mockReset();
});
it.each([false, true])(
"releases only repair credential leases after the terminal handoff (parent lease=%s)",
async (parentLease) => {
await withOpenClawTestState({ layout: "home" }, async (state) => {
fixture.handoff = createManagedHandoffTestBinding(state.root);
vi.stubEnv(
"NODE_OPTIONS",
[process.env.NODE_OPTIONS, fixture.handoff.nodeOption].filter(Boolean).join(" "),
);
expect(fixture.handoff.assertPath(resolveManagedUpdateLeaseDatabasePath())).toBe(
fixture.handoff.databasePath,
);
vi.stubEnv("OPENCLAW_SERVICE_REPAIR_POLICY", "external");
const agentDir = state.agentDir("owner");
const auth = createAuthProfileStoreRuntime(createExternalAuthRuntime(() => []));
const parent = createOpenClawDatabaseMaintenanceScope();
if (parentLease) {
parent.run(() =>
auth.saveAuthProfileStore({ version: 1, profiles: {} }, state.agentDir("independent")),
);
}
try {
fixture.run.mockImplementation(async (input: RunEmbeddedAgentParams) => {
expect(input.sessionPersistence).toBe("detached");
expect(input.agentDir).toBe(agentDir);
auth.saveAuthProfileStore(
{
version: 1,
profiles: {
"fixture:repair": { type: "token", provider: "fixture", token: "synthetic-token" },
},
},
agentDir,
);
const leases = readActiveOpenClawAgentDatabaseLeasesReadOnly({ env: state.env });
expect(leases.map((lease) => lease.agent_id).toSorted()).toEqual(
parentLease ? ["independent", "owner"] : ["owner"],
);
expect(leases.every((lease) => lease.owner_pid === process.pid)).toBe(true);
expect(() => assertNoOpenClawAgentDatabaseLeasesReadOnly({ env: state.env })).toThrow(
/still open in process/,
);
// The embedded runner starts this durable write without awaiting its completion.
markEmbeddedRunAuthProfileSuccess({
agentDir,
profileId: "fixture:repair",
provider: "fixture",
profileStore: auth.loadAuthProfileStoreForRuntime(agentDir, {
readOnly: true,
externalCli: { mode: "none" },
}),
runId: input.runId,
sessionId: input.sessionId,
});
return {
meta: {
durationMs: 1,
stopReason: "tool_calls",
pendingToolCalls: [
{
name: "request_update_maintenance",
arguments: '{"operation":"update-repair"}',
},
],
},
};
});
const config = {
plugins: { enabled: false },
agents: { entries: { owner: { agentDir } } },
};
const result = await parent.run(() =>
runUpdateRepairTurn({
target: { ...state, installRoot: state.workspaceDir },
route: {
runner: "embedded",
provider: "fixture",
model: "repair",
modelLabel: "fixture/repair",
agentId: "owner",
agentDir,
runConfig: config,
sourceConfig: config,
},
modelFallbacks: [],
prompt: "Repair.",
timeoutMs: 10_000,
maxToolCalls: 1,
signal: new AbortController().signal,
maintenanceHandoff: true,
}),
);
// Check before test cleanup: otherwise fixture teardown hides the production leak.
expect(
readActiveOpenClawAgentDatabaseLeasesReadOnly({ env: state.env }).map(
(lease) => lease.agent_id,
),
).toEqual(parentLease ? ["independent"] : []);
if (parentLease) {
expect(() => assertNoOpenClawAgentDatabaseLeasesReadOnly({ env: state.env })).toThrow(
/Agent independent/,
);
} else {
expect(() =>
assertNoOpenClawAgentDatabaseLeasesReadOnly({ env: state.env }),
).not.toThrow();
}
expect(result).toMatchObject({
status: "completed",
envelope: { status: "ok" },
maintenance: { operation: "update-repair" },
});
expect(
fixture.run.mock.calls[0]?.[0].clientTools?.map(
(tool: { function: { name: string } }) => tool.function.name,
),
).toEqual(["request_update_maintenance"]);
const saved = auth.loadAuthProfileStoreForRuntime(agentDir, {
readOnly: true,
externalCli: { mode: "none" },
});
expect(saved.usageStats?.["fixture:repair"]?.lastUsed).toBeGreaterThan(0);
const admitDoctor = () =>
beginDoctorMaintenance({
options: { repair: true, nonInteractive: true },
root: null,
runtime: { log() {}, error() {}, exit() {} },
});
if (parentLease) {
await expect(admitDoctor()).rejects.toMatchObject({
refusal: { kind: "deferred", reason: "agent-database-in-use" },
});
} else {
const doctor = await admitDoctor();
expect(doctor).toBeDefined();
try {
// Exercise the real maintenance owner after the repair, then reopen
// durable credentials as the next startup would.
doctor!.run(() => auth.saveAuthProfileStore(saved, agentDir));
} finally {
await doctor?.release();
}
expect(readActiveOpenClawAgentDatabaseLeasesReadOnly({ env: state.env })).toEqual([]);
expect(
auth.loadAuthProfileStoreForRuntime(agentDir, {
readOnly: true,
externalCli: { mode: "none" },
}),
).toEqual(saved);
}
} finally {
await parent.close();
}
});
},
);

View file

@ -0,0 +1,185 @@
import { beforeEach, expect, it, vi } from "vitest";
import { createAgentCleanupScope } from "../agents/run-cleanup-timeout.js";
import { CommandProcessCleanupError } from "../process/exec-result.js";
import {
readUpdateRepairMaintenanceRequest,
runUpdateRepairMaintenance,
} from "./update-repair-maintenance.js";
const external = vi.hoisted(() => ({ entry: vi.fn(), command: vi.fn() }));
vi.mock("../daemon/gateway-entrypoint.js", () => ({
resolveGatewayInstallEntrypoint: external.entry,
}));
vi.mock("../process/exec.js", () => ({ runUtf8CommandWithTimeout: external.command }));
beforeEach(() => {
external.entry.mockReset().mockResolvedValue("/synthetic/install/dist/index.js");
external.command
.mockReset()
.mockResolvedValue({ termination: "exit", code: 0, stdout: "", stderr: "" });
});
const target = {
installRoot: "/synthetic/install",
stateDir: "/synthetic/state",
configPath: "/synthetic/config",
workspaceDir: "/synthetic/workspace",
};
it.each(["doctor-fix", "update-repair"] as const)(
"executes only the typed %s continuation on the selected installation",
async (operation) => {
const request = readUpdateRepairMaintenanceRequest({
stopReason: "tool_calls",
pendingToolCalls: [
{ name: "request_update_maintenance", arguments: JSON.stringify({ operation }) },
],
});
expect(request).toEqual({ operation });
const current = vi.fn();
const signal = new AbortController().signal;
await runUpdateRepairMaintenance({
request: request!,
allowGatewayActivation: false,
target,
env: {
OPENCLAW_STATE_DIR: target.stateDir,
OPENCLAW_UPDATE_PARENT_ALLOWS_GATEWAY_ACTIVATION: "1",
},
signal,
assertCurrent: current,
});
expect(current).toHaveBeenCalledOnce();
expect(external.command).toHaveBeenCalledExactlyOnceWith(
[
expect.any(String),
"/synthetic/install/dist/index.js",
...(operation === "update-repair"
? ["update", "repair", "--yes", "--json", "--no-restart"]
: ["doctor", "--fix", "--non-interactive"]),
],
expect.objectContaining({
cwd: target.installRoot,
baseEnv: {},
env: expect.objectContaining({
OPENCLAW_STATE_DIR: target.stateDir,
OPENCLAW_SHELL: "exec",
OPENCLAW_UPDATE_IN_PROGRESS: "1",
OPENCLAW_UPDATE_PARENT_ALLOWS_GATEWAY_SERVICE_REPAIR: "0",
OPENCLAW_UPDATE_PARENT_ALLOWS_GATEWAY_ACTIVATION: "0",
OPENCLAW_SERVICE_REPAIR_POLICY: "external",
}),
signal,
killProcessTree: true,
requireProcessTreeExtinction: true,
}),
);
},
);
it.each(["revoked", "cancelled"])(
"does not launch maintenance when %s during entrypoint resolution",
async (cause) => {
const controller = new AbortController();
let current = true;
external.entry.mockImplementation(async () => {
current = false;
if (cause === "cancelled") {
controller.abort(new Error("cancelled"));
}
return "/synthetic/install/dist/index.js";
});
await expect(
runUpdateRepairMaintenance({
request: { operation: "update-repair" },
allowGatewayActivation: false,
target,
env: {},
signal: controller.signal,
assertCurrent: () => {
if (!current) {
throw new Error("revoked");
}
},
}),
).rejects.toThrow(cause);
expect(external.command).not.toHaveBeenCalled();
},
);
it.each([
{
stopReason: "stop",
pendingToolCalls: [
{ name: "request_update_maintenance", arguments: '{"operation":"doctor-fix"}' },
],
},
{
stopReason: "tool_calls",
pendingToolCalls: [{ name: "exec", arguments: '{"operation":"doctor-fix"}' }],
},
{
stopReason: "tool_calls",
pendingToolCalls: [
{
name: "request_update_maintenance",
arguments: '{"operation":"doctor-fix","command":"anything"}',
},
],
},
{
stopReason: "tool_calls",
pendingToolCalls: [
{ name: "request_update_maintenance", arguments: '{"operation":"doctor-fix"}' },
{ name: "request_update_maintenance", arguments: '{"operation":"update-repair"}' },
],
},
])("refuses malformed or ambiguous terminal maintenance requests: %j", (meta) => {
expect(() => readUpdateRepairMaintenanceRequest(meta)).toThrow();
});
it.each(["forced", "uncertain", "rejected"] as const)(
"cannot certify the repair owner after %s maintenance cleanup",
async (cleanup) => {
if (cleanup === "rejected") {
external.command.mockRejectedValue(new CommandProcessCleanupError());
} else {
external.command.mockResolvedValue({
termination: "exit",
code: 0,
stdout: "",
stderr: "",
cleanup,
});
}
const owner = createAgentCleanupScope();
await expect(
owner.run(() =>
runUpdateRepairMaintenance({
request: { operation: "doctor-fix" },
allowGatewayActivation: false,
target,
env: {},
signal: new AbortController().signal,
assertCurrent: () => {},
}),
),
).rejects.toThrow(/cleanup/i);
expect(owner.outcome).toBe("uncertain");
},
);
it("leaves an intended-running recovery with the native maintenance owner", async () => {
await runUpdateRepairMaintenance({
request: { operation: "update-repair" },
allowGatewayActivation: true,
target,
env: { OPENCLAW_SERVICE_REPAIR_POLICY: "external" },
signal: new AbortController().signal,
assertCurrent: () => {},
});
expect(external.command).toHaveBeenCalledExactlyOnceWith(
[expect.any(String), "/synthetic/install/dist/index.js", "update", "repair", "--yes", "--json"],
expect.objectContaining({
env: { OPENCLAW_SERVICE_REPAIR_POLICY: "external", OPENCLAW_SHELL: "exec" },
}),
);
});

View file

@ -0,0 +1,117 @@
import { z } from "zod";
import type { ClientToolDefinition } from "../agents/command/shared-types.js";
import { recordAgentCleanupFailure } from "../agents/run-cleanup-timeout.js";
import { hasCommandProcessCleanupError } from "../process/exec-result.js";
import type { UpdateRepairTarget } from "./update-repair-protocol.js";
import { buildUpdateDoctorEnv } from "./update-runner-doctor.js";
const UPDATE_REPAIR_MAINTENANCE_TOOL = "request_update_maintenance";
const requestSchema = z.strictObject({ operation: z.enum(["doctor-fix", "update-repair"]) });
export type UpdateRepairMaintenanceRequest = z.infer<typeof requestSchema>;
/** A terminal client tool requests work; it never runs maintenance inside the agent. */
export const updateRepairMaintenanceTool: ClientToolDefinition = {
type: "function",
function: {
name: UPDATE_REPAIR_MAINTENANCE_TOOL,
description:
"End this repair turn and ask the update owner to run Doctor repair or finish an interrupted update after all agent database and process resources settle. Do not run these maintenance commands through exec. The owner preserves all lease, service, and capability refusals.",
parameters: {
type: "object",
properties: { operation: { type: "string", enum: ["doctor-fix", "update-repair"] } },
required: ["operation"],
additionalProperties: false,
},
strict: true,
},
};
export function readUpdateRepairMaintenanceRequest(meta: {
stopReason?: string;
pendingToolCalls?: Array<{ name: string; arguments: string }>;
}): UpdateRepairMaintenanceRequest | undefined {
if (!meta.pendingToolCalls?.length) {
return undefined;
}
const [call] = meta.pendingToolCalls;
if (
meta.stopReason !== "tool_calls" ||
meta.pendingToolCalls.length !== 1 ||
call?.name !== UPDATE_REPAIR_MAINTENANCE_TOOL
) {
throw new Error("Repair returned an invalid maintenance handoff.");
}
return requestSchema.parse(JSON.parse(call.arguments));
}
/** Called only after inference, the agent turn, and their resource scopes have closed. */
export async function runUpdateRepairMaintenance(params: {
request: UpdateRepairMaintenanceRequest;
target: UpdateRepairTarget;
env: NodeJS.ProcessEnv;
signal: AbortSignal;
assertCurrent: () => void;
allowGatewayActivation: boolean;
}) {
const [{ resolveGatewayInstallEntrypoint }, { isNodeRuntime }, { runUtf8CommandWithTimeout }] =
await Promise.all([
import("../daemon/gateway-entrypoint.js"),
import("../daemon/runtime-binary.js"),
import("../process/exec.js"),
]);
const entrypoint = await resolveGatewayInstallEntrypoint(params.target.installRoot);
params.signal.throwIfAborted();
params.assertCurrent();
if (!entrypoint) {
throw new Error("The installed OpenClaw entrypoint is unavailable.");
}
const args =
params.request.operation === "update-repair"
? [
"update",
"repair",
"--yes",
"--json",
...(params.allowGatewayActivation ? [] : ["--no-restart"]),
]
: ["doctor", "--fix", "--non-interactive"];
try {
const result = await runUtf8CommandWithTimeout(
[isNodeRuntime(process.execPath) ? process.execPath : "node", entrypoint, ...args],
{
cwd: params.target.installRoot,
baseEnv: {},
// The fixing subtree must not recursively start another automatic repair.
env: {
...params.env,
// Preserve an intentional stop; otherwise let the command's existing
// maintenance owner enforce its native service and activation policy.
...(!params.allowGatewayActivation
? buildUpdateDoctorEnv({
allowGatewayServiceRepair: false,
allowGatewayActivation: false,
serviceRepairPolicy: "external",
})
: {}),
OPENCLAW_SHELL: "exec",
},
input: "",
signal: params.signal,
killProcessTree: true,
requireProcessTreeExtinction: true,
outputCapture: "tail",
maxOutputBytes: 32 * 1024,
},
);
if (result.cleanup === "uncertain" || result.cleanup === "forced") {
recordAgentCleanupFailure();
throw new Error("Maintenance subprocess cleanup is unconfirmed.");
}
return result;
} catch (error) {
if (hasCommandProcessCleanupError(error)) {
recordAgentCleanupFailure();
}
throw error;
}
}