mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-10 00:11:19 +00:00
fix(subagents): killed subagent runs stay running in the task list (#99806)
* fix(subagents): reconcile killed task outcomes Co-authored-by: masatohoshino <g515hoshino@gmail.com> * fix(subagents): retain reconciliation marker narrowing * fix(subagents): honor generations in latest-run views * test(subagents): align reconciliation fixtures * fix(subagents): keep steer recovery best effort * style(tasks): use type-only runtime contract import * refactor(subagents): keep generation ordering leaf-only * fix(subagents): sanitize persisted task owner ids * fix(subagents): preserve failed replay lifecycle reason * fix(agents): clear killed lifecycle timeout grace * fix(tasks): preserve canonical subagent outcomes * fix(agents): continue requester-wide cancellation * fix(agents): fence yield revival races * test(agents): mock detached task lookup * fix: fence subagent cleanup deletion --------- Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
parent
9c5ea29e38
commit
153ee2abba
46 changed files with 8471 additions and 520 deletions
|
|
@ -395,6 +395,7 @@ vi.mock("../tasks/detached-task-runtime.js", () => ({
|
|||
completeTaskRunByRunId: vi.fn(),
|
||||
createRunningTaskRun: vi.fn(),
|
||||
failTaskRunByRunId: vi.fn(),
|
||||
findDetachedTaskRun: vi.fn(() => ({ lookup: "available" as const })),
|
||||
setDetachedTaskDeliveryStatusByRunId: vi.fn(),
|
||||
}));
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import {
|
|||
applySubagentWaitOutcome,
|
||||
buildCompactAnnounceStatsLine,
|
||||
buildChildCompletionFindings,
|
||||
dedupeLatestChildCompletionRows,
|
||||
readSubagentOutput,
|
||||
} from "./subagent-announce-output.js";
|
||||
|
||||
|
|
@ -63,6 +64,22 @@ function sessionsYieldTurn(message = "Waiting for subagent completion.") {
|
|||
];
|
||||
}
|
||||
|
||||
describe("dedupeLatestChildCompletionRows", () => {
|
||||
it("prefers the newer generation when child runs share a creation timestamp", () => {
|
||||
const childSessionKey = "agent:main:subagent:reused";
|
||||
const older = {
|
||||
runId: "run-older",
|
||||
generation: 1,
|
||||
childSessionKey,
|
||||
task: "older",
|
||||
createdAt: 1_000,
|
||||
};
|
||||
const newer = { ...older, runId: "run-newer", generation: 2, task: "newer" };
|
||||
|
||||
expect(dedupeLatestChildCompletionRows([older, newer])).toStrictEqual([newer]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildCompactAnnounceStatsLine", () => {
|
||||
afterEach(() => {
|
||||
testing.setDepsForTest();
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import {
|
|||
resolveAgentIdFromSessionKey,
|
||||
resolveStorePath,
|
||||
} from "./subagent-announce.runtime.js";
|
||||
import { compareSubagentRunGeneration } from "./subagent-run-generation.js";
|
||||
import { assistantCallsSessionsYield, isSessionsYieldToolResult } from "./subagent-yield-output.js";
|
||||
import { extractAssistantText, sanitizeTextContent } from "./tools/chat-history-text.js";
|
||||
import { isAnnounceSkip } from "./tools/sessions-send-tokens.js";
|
||||
|
|
@ -416,9 +417,11 @@ export function buildChildCompletionFindings(
|
|||
|
||||
export function dedupeLatestChildCompletionRows(
|
||||
children: Array<{
|
||||
runId: string;
|
||||
childSessionKey: string;
|
||||
task: string;
|
||||
label?: string;
|
||||
generation?: number;
|
||||
createdAt: number;
|
||||
endedAt?: number;
|
||||
frozenResultText?: string | null;
|
||||
|
|
@ -438,7 +441,7 @@ export function dedupeLatestChildCompletionRows(
|
|||
const latestByChildSessionKey = new Map<string, (typeof children)[number]>();
|
||||
for (const child of children) {
|
||||
const existing = latestByChildSessionKey.get(child.childSessionKey);
|
||||
if (!existing || child.createdAt > existing.createdAt) {
|
||||
if (!existing || compareSubagentRunGeneration(child, existing) > 0) {
|
||||
latestByChildSessionKey.set(child.childSessionKey, child);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -336,6 +336,25 @@ describe("subagent announce seam flow", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("skips delete cleanup when the lifecycle owner invalidates the attempt", async () => {
|
||||
const didAnnounce = await runSubagentAnnounceFlow({
|
||||
childSessionKey: "agent:main:subagent:test",
|
||||
childRunId: "run-invalidated-delete",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "do thing",
|
||||
timeoutMs: 10,
|
||||
cleanup: "delete",
|
||||
waitForCompletion: false,
|
||||
outcome: { status: "ok" },
|
||||
roundOneReply: "ANNOUNCE_SKIP",
|
||||
onBeforeDeleteChildSession: () => false,
|
||||
});
|
||||
|
||||
expect(didAnnounce).toBe(true);
|
||||
expect(sessionsDeleteSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("warns when ANNOUNCE_SKIP suppresses a cron job completion", async () => {
|
||||
const logSpy = vi.spyOn(defaultRuntime, "log").mockImplementation(() => {});
|
||||
|
||||
|
|
|
|||
|
|
@ -261,6 +261,7 @@ export async function runSubagentAnnounceFlow(params: {
|
|||
signal?: AbortSignal;
|
||||
bestEffortDeliver?: boolean;
|
||||
onDeliveryResult?: (delivery: SubagentAnnounceDeliveryResult) => void;
|
||||
onBeforeDeleteChildSession?: () => boolean;
|
||||
}): Promise<boolean> {
|
||||
let didAnnounce = false;
|
||||
const expectsCompletionMessage = params.expectsCompletionMessage === true;
|
||||
|
|
@ -619,7 +620,7 @@ export async function runSubagentAnnounceFlow(params: {
|
|||
// Best-effort
|
||||
}
|
||||
}
|
||||
if (shouldDeleteChildSession) {
|
||||
if (shouldDeleteChildSession && (params.onBeforeDeleteChildSession?.() ?? true)) {
|
||||
await deleteSubagentSessionForCleanup({
|
||||
callGateway: subagentAnnounceDeps.callGateway,
|
||||
childSessionKey: params.childSessionKey,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import type {
|
|||
import type { SessionEntry } from "../config/sessions/types.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import type { CallGatewayOptions } from "../gateway/call.js";
|
||||
import { SUBAGENT_KILL_TASK_ERROR } from "../tasks/detached-task-runtime-contract.js";
|
||||
import {
|
||||
testing,
|
||||
killAllControlledSubagentRuns,
|
||||
|
|
@ -21,6 +22,10 @@ import {
|
|||
sendControlledSubagentMessage,
|
||||
steerControlledSubagentRun,
|
||||
} from "./subagent-control.js";
|
||||
import {
|
||||
SUBAGENT_ENDED_REASON_COMPLETE,
|
||||
SUBAGENT_ENDED_REASON_KILLED,
|
||||
} from "./subagent-lifecycle-events.js";
|
||||
import {
|
||||
testing as subagentRegistryTesting,
|
||||
addSubagentRunForTests,
|
||||
|
|
@ -32,6 +37,23 @@ vi.mock("../gateway/call.js", () => ({
|
|||
callGateway: vi.fn(),
|
||||
}));
|
||||
|
||||
const detachedTaskRuntimeMocks = vi.hoisted(() => ({
|
||||
findDetachedTaskRun: vi.fn(() => ({ lookup: "available" as const })),
|
||||
finalizeTaskRunByRunId: vi.fn<(_params: unknown) => unknown[]>(() => []),
|
||||
}));
|
||||
|
||||
vi.mock("../tasks/detached-task-runtime.js", () => ({
|
||||
createQueuedTaskRun: vi.fn(() => null),
|
||||
createRunningTaskRun: vi.fn(() => null),
|
||||
startTaskRunByRunId: vi.fn(() => []),
|
||||
recordTaskRunProgressByRunId: vi.fn(() => []),
|
||||
finalizeTaskRunByRunId: detachedTaskRuntimeMocks.finalizeTaskRunByRunId,
|
||||
completeTaskRunByRunId: vi.fn(() => []),
|
||||
failTaskRunByRunId: vi.fn(() => []),
|
||||
setDetachedTaskDeliveryStatusByRunId: vi.fn(() => []),
|
||||
findDetachedTaskRun: detachedTaskRuntimeMocks.findDetachedTaskRun,
|
||||
}));
|
||||
|
||||
vi.mock("./run-wait.js", () => {
|
||||
const readLatestAssistantReplySnapshot = async (params: {
|
||||
sessionKey: string;
|
||||
|
|
@ -188,6 +210,7 @@ function writeSessionStoreFixture(label: string, store: Record<string, unknown>)
|
|||
}
|
||||
|
||||
beforeEach(() => {
|
||||
detachedTaskRuntimeMocks.finalizeTaskRunByRunId.mockClear();
|
||||
setSubagentControlDepsForTest();
|
||||
subagentRegistryTesting.setDepsForTest({
|
||||
cleanupBrowserSessionsForLifecycleEnd: async () => {},
|
||||
|
|
@ -502,7 +525,6 @@ describe("sendControlledSubagentMessage", () => {
|
|||
role: "assistant",
|
||||
content: [{ type: "text", text: "older reply from a previous run" }],
|
||||
};
|
||||
|
||||
setSubagentControlDepsForTest({
|
||||
callGateway: async <T = Record<string, unknown>>(request: CallGatewayOptions) => {
|
||||
if (request.method === "chat.history") {
|
||||
|
|
@ -590,9 +612,21 @@ describe("killSubagentRunAdmin", () => {
|
|||
|
||||
expect(result.found).toBe(true);
|
||||
expect(result.killed).toBe(true);
|
||||
if (!result.found) {
|
||||
throw new Error("expected tracked subagent run");
|
||||
}
|
||||
expect(result.runId).toBe("run-worker");
|
||||
expect(result.sessionKey).toBe(childSessionKey);
|
||||
expect(getSubagentRunByChildSessionKey(childSessionKey)?.endedAt).toBeTypeOf("number");
|
||||
expect(detachedTaskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalledTimes(1);
|
||||
expect(detachedTaskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
runId: "run-worker",
|
||||
runtime: "subagent",
|
||||
sessionKey: childSessionKey,
|
||||
status: "cancelled",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns found=false when the session key is not tracked as a subagent run", async () => {
|
||||
|
|
@ -604,6 +638,379 @@ describe("killSubagentRunAdmin", () => {
|
|||
expect(result).toEqual({ found: false, killed: false });
|
||||
});
|
||||
|
||||
it("retries task reconciliation for an already-killed run", async () => {
|
||||
const childSessionKey = "agent:main:subagent:already-killed";
|
||||
const endedAt = Date.now() - 1_000;
|
||||
addSubagentRunForTests({
|
||||
runId: "run-already-killed",
|
||||
childSessionKey,
|
||||
controllerSessionKey: "agent:main:controller",
|
||||
requesterSessionKey: "agent:main:requester",
|
||||
requesterDisplayKey: "requester",
|
||||
task: "repair task projection",
|
||||
cleanup: "keep",
|
||||
createdAt: endedAt - 4_000,
|
||||
startedAt: endedAt - 3_000,
|
||||
endedAt,
|
||||
endedReason: SUBAGENT_ENDED_REASON_KILLED,
|
||||
outcome: { status: "error", error: "killed" },
|
||||
suppressAnnounceReason: "killed",
|
||||
killReconciliation: { killedAt: endedAt },
|
||||
cleanupCompletedAt: endedAt + 500,
|
||||
});
|
||||
|
||||
const first = await killSubagentRunAdmin({ cfg: {}, sessionKey: childSessionKey });
|
||||
const second = await killSubagentRunAdmin({ cfg: {}, sessionKey: childSessionKey });
|
||||
|
||||
for (const result of [first, second]) {
|
||||
expect(result).toMatchObject({
|
||||
found: true,
|
||||
killed: false,
|
||||
targetState: {
|
||||
state: "terminal",
|
||||
task: {
|
||||
status: "cancelled",
|
||||
endedAt,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
expect(detachedTaskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("keeps a killed steer-restart run on its failed projection", async () => {
|
||||
const childSessionKey = "agent:main:subagent:steer-restart";
|
||||
const endedAt = Date.now() - 1_000;
|
||||
addSubagentRunForTests({
|
||||
runId: "run-steer-restart",
|
||||
childSessionKey,
|
||||
controllerSessionKey: "agent:main:controller",
|
||||
requesterSessionKey: "agent:main:requester",
|
||||
requesterDisplayKey: "requester",
|
||||
task: "replace active run",
|
||||
cleanup: "keep",
|
||||
createdAt: endedAt - 4_000,
|
||||
startedAt: endedAt - 3_000,
|
||||
endedAt,
|
||||
endedReason: SUBAGENT_ENDED_REASON_KILLED,
|
||||
suppressAnnounceReason: "steer-restart",
|
||||
outcome: { status: "error", error: "agent run aborted" },
|
||||
completion: { required: false, resultText: null, capturedAt: endedAt },
|
||||
});
|
||||
|
||||
const result = await killSubagentRunAdmin({ cfg: {}, sessionKey: childSessionKey });
|
||||
|
||||
expect(result).toMatchObject({
|
||||
found: true,
|
||||
killed: false,
|
||||
targetState: {
|
||||
state: "terminal",
|
||||
task: {
|
||||
status: "failed",
|
||||
endedAt,
|
||||
error: "agent run aborted",
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(detachedTaskRuntimeMocks.finalizeTaskRunByRunId).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("restores the recoverable task marker when abort lifecycle wins the race", async () => {
|
||||
const childSessionKey = "agent:main:subagent:abort-lifecycle-race";
|
||||
const storePath = writeSessionStoreFixture("admin-kill-abort-lifecycle-race", {
|
||||
[childSessionKey]: {
|
||||
sessionId: "sess-abort-lifecycle-race",
|
||||
updatedAt: Date.now(),
|
||||
},
|
||||
});
|
||||
const run = {
|
||||
runId: "run-abort-lifecycle-race",
|
||||
childSessionKey,
|
||||
controllerSessionKey: "agent:main:controller",
|
||||
requesterSessionKey: "agent:main:requester",
|
||||
requesterDisplayKey: "requester",
|
||||
task: "finish while aborting",
|
||||
cleanup: "keep" as const,
|
||||
createdAt: Date.now() - 5_000,
|
||||
startedAt: Date.now() - 4_000,
|
||||
};
|
||||
const abortedLastRunWrites: boolean[] = [];
|
||||
addSubagentRunForTests(run);
|
||||
setSubagentControlDepsForTest({
|
||||
isEmbeddedAgentRunActive: () => true,
|
||||
abortEmbeddedAgentRun: () => {
|
||||
const endedAt = Date.now();
|
||||
Object.assign(run, {
|
||||
endedAt,
|
||||
endedReason: SUBAGENT_ENDED_REASON_KILLED,
|
||||
outcome: { status: "error" as const, error: "agent run aborted" },
|
||||
suppressAnnounceReason: "killed" as const,
|
||||
killReconciliation: { killedAt: endedAt },
|
||||
});
|
||||
return true;
|
||||
},
|
||||
patchSessionEntry: async (_scope, patcher) => {
|
||||
const current = { sessionId: "sess-abort-lifecycle-race", updatedAt: Date.now() };
|
||||
const patch = await patcher(current, { existingEntry: { ...current } });
|
||||
abortedLastRunWrites.push(patch?.abortedLastRun === true);
|
||||
return patch ? { ...current, ...patch } : current;
|
||||
},
|
||||
});
|
||||
|
||||
const result = await killSubagentRunAdmin({
|
||||
cfg: cfgWithSessionStore(storePath),
|
||||
sessionKey: childSessionKey,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ found: true, killed: true });
|
||||
expect(detachedTaskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
runId: "run-abort-lifecycle-race",
|
||||
status: "cancelled",
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
}),
|
||||
);
|
||||
expect(abortedLastRunWrites).toEqual([true]);
|
||||
});
|
||||
|
||||
it("reports when completion wins while the kill path awaits persistence", async () => {
|
||||
const childSessionKey = "agent:main:subagent:completion-race";
|
||||
const storePath = writeSessionStoreFixture("admin-kill-completion-race", {
|
||||
[childSessionKey]: {
|
||||
sessionId: "sess-completion-race",
|
||||
updatedAt: Date.now(),
|
||||
},
|
||||
});
|
||||
const run = {
|
||||
runId: "run-completion-race",
|
||||
childSessionKey,
|
||||
controllerSessionKey: "agent:main:controller",
|
||||
requesterSessionKey: "agent:main:requester",
|
||||
requesterDisplayKey: "requester",
|
||||
task: "finish while cancellation starts",
|
||||
cleanup: "keep" as const,
|
||||
createdAt: Date.now() - 5_000,
|
||||
startedAt: Date.now() - 4_000,
|
||||
};
|
||||
const abortedLastRunWrites: boolean[] = [];
|
||||
addSubagentRunForTests({
|
||||
...run,
|
||||
runId: "run-stale-completion-race",
|
||||
task: "stale older row",
|
||||
createdAt: Date.now() - 9_000,
|
||||
startedAt: Date.now() - 8_000,
|
||||
});
|
||||
addSubagentRunForTests(run);
|
||||
setSubagentControlDepsForTest({
|
||||
isEmbeddedAgentRunActive: () => true,
|
||||
abortEmbeddedAgentRun: () => true,
|
||||
patchSessionEntry: async (_scope, patcher) => {
|
||||
const current = { sessionId: "sess-completion-race", updatedAt: Date.now() };
|
||||
const patch = await patcher(current, { existingEntry: { ...current } });
|
||||
abortedLastRunWrites.push(patch?.abortedLastRun === true);
|
||||
if (abortedLastRunWrites.length === 1) {
|
||||
Object.assign(run, {
|
||||
endedAt: Date.now(),
|
||||
endedReason: SUBAGENT_ENDED_REASON_COMPLETE,
|
||||
outcome: { status: "ok" as const },
|
||||
completion: {
|
||||
required: false,
|
||||
resultText: "done",
|
||||
capturedAt: Date.now(),
|
||||
},
|
||||
});
|
||||
}
|
||||
return patch ? { ...current, ...patch } : current;
|
||||
},
|
||||
});
|
||||
|
||||
const result = await killSubagentRunAdmin({
|
||||
cfg: cfgWithSessionStore(storePath),
|
||||
sessionKey: childSessionKey,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
found: true,
|
||||
killed: false,
|
||||
targetState: {
|
||||
state: "terminal",
|
||||
task: {
|
||||
status: "succeeded",
|
||||
endedAt: expect.any(Number),
|
||||
progressSummary: "done",
|
||||
terminalSummary: null,
|
||||
},
|
||||
},
|
||||
runId: "run-completion-race",
|
||||
});
|
||||
expect(abortedLastRunWrites).toEqual([true, false]);
|
||||
expect(run).toMatchObject({
|
||||
endedReason: SUBAGENT_ENDED_REASON_COMPLETE,
|
||||
outcome: { status: "ok" },
|
||||
});
|
||||
expect(detachedTaskRuntimeMocks.finalizeTaskRunByRunId).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("refreshes target completion after descendant cancellation settles", async () => {
|
||||
const childSessionKey = "agent:main:subagent:cascade-completion-race";
|
||||
const descendantSessionKey = "agent:main:subagent:cascade-completion-child";
|
||||
const storePath = writeSessionStoreFixture("admin-kill-cascade-completion-race", {
|
||||
[childSessionKey]: {
|
||||
sessionId: "sess-cascade-completion-race",
|
||||
updatedAt: Date.now(),
|
||||
},
|
||||
[descendantSessionKey]: {
|
||||
sessionId: "sess-cascade-completion-child",
|
||||
updatedAt: Date.now(),
|
||||
},
|
||||
});
|
||||
const run = {
|
||||
runId: "run-cascade-completion-race",
|
||||
childSessionKey,
|
||||
controllerSessionKey: "agent:main:controller",
|
||||
requesterSessionKey: "agent:main:requester",
|
||||
requesterDisplayKey: "requester",
|
||||
task: "finish while descendant cancellation settles",
|
||||
cleanup: "keep" as const,
|
||||
createdAt: Date.now() - 5_000,
|
||||
startedAt: Date.now() - 4_000,
|
||||
};
|
||||
const abortedLastRunWrites: boolean[] = [];
|
||||
addSubagentRunForTests(run);
|
||||
addSubagentRunForTests({
|
||||
runId: "run-cascade-completion-child",
|
||||
childSessionKey: descendantSessionKey,
|
||||
controllerSessionKey: childSessionKey,
|
||||
requesterSessionKey: childSessionKey,
|
||||
requesterDisplayKey: "parent",
|
||||
task: "descendant",
|
||||
cleanup: "keep",
|
||||
createdAt: Date.now() - 3_000,
|
||||
startedAt: Date.now() - 2_000,
|
||||
});
|
||||
setSubagentControlDepsForTest({
|
||||
isEmbeddedAgentRunActive: () => true,
|
||||
abortEmbeddedAgentRun: () => true,
|
||||
patchSessionEntry: async (scope, patcher) => {
|
||||
const current = { sessionId: `sess-${scope.sessionKey}`, updatedAt: Date.now() };
|
||||
const patch = await patcher(current, { existingEntry: { ...current } });
|
||||
if (scope.sessionKey === childSessionKey) {
|
||||
abortedLastRunWrites.push(patch?.abortedLastRun === true);
|
||||
}
|
||||
if (scope.sessionKey === descendantSessionKey) {
|
||||
const endedAt = Date.now();
|
||||
Object.assign(run, {
|
||||
endedAt,
|
||||
endedReason: SUBAGENT_ENDED_REASON_COMPLETE,
|
||||
outcome: { status: "ok" as const },
|
||||
completion: { required: false, resultText: "done", capturedAt: endedAt },
|
||||
});
|
||||
}
|
||||
return patch ? { ...current, ...patch } : current;
|
||||
},
|
||||
});
|
||||
|
||||
const result = await killSubagentRunAdmin({
|
||||
cfg: cfgWithSessionStore(storePath),
|
||||
sessionKey: childSessionKey,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
found: true,
|
||||
killed: true,
|
||||
targetState: {
|
||||
state: "terminal",
|
||||
task: {
|
||||
status: "succeeded",
|
||||
progressSummary: "done",
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(abortedLastRunWrites).toEqual([true, false]);
|
||||
});
|
||||
|
||||
it("kills a run that yields while the kill path awaits persistence", async () => {
|
||||
const childSessionKey = "agent:main:subagent:yield-race";
|
||||
const storePath = writeSessionStoreFixture("admin-kill-yield-race", {
|
||||
[childSessionKey]: {
|
||||
sessionId: "sess-yield-race",
|
||||
updatedAt: Date.now(),
|
||||
},
|
||||
});
|
||||
const run = {
|
||||
runId: "run-yield-race",
|
||||
childSessionKey,
|
||||
controllerSessionKey: "agent:main:controller",
|
||||
requesterSessionKey: "agent:main:requester",
|
||||
requesterDisplayKey: "requester",
|
||||
task: "yield while cancellation starts",
|
||||
cleanup: "keep" as const,
|
||||
createdAt: Date.now() - 5_000,
|
||||
startedAt: Date.now() - 4_000,
|
||||
};
|
||||
const yieldedAt = Date.now() - 1_000;
|
||||
addSubagentRunForTests(run);
|
||||
setSubagentControlDepsForTest({
|
||||
isEmbeddedAgentRunActive: () => true,
|
||||
abortEmbeddedAgentRun: () => true,
|
||||
patchSessionEntry: async () => {
|
||||
Object.assign(run, {
|
||||
endedAt: yieldedAt,
|
||||
pauseReason: "sessions_yield" as const,
|
||||
});
|
||||
return null;
|
||||
},
|
||||
});
|
||||
|
||||
const result = await killSubagentRunAdmin({
|
||||
cfg: cfgWithSessionStore(storePath),
|
||||
sessionKey: childSessionKey,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
found: true,
|
||||
killed: true,
|
||||
runId: "run-yield-race",
|
||||
targetState: {
|
||||
state: "terminal",
|
||||
task: { status: "cancelled", error: SUBAGENT_KILL_TASK_ERROR },
|
||||
},
|
||||
});
|
||||
expect(getSubagentRunByChildSessionKey(childSessionKey)).toMatchObject({
|
||||
endedReason: SUBAGENT_ENDED_REASON_KILLED,
|
||||
endedAt: yieldedAt,
|
||||
outcome: {
|
||||
status: "error",
|
||||
endedAt: yieldedAt,
|
||||
elapsedMs: yieldedAt - run.startedAt,
|
||||
},
|
||||
});
|
||||
expect(getSubagentRunByChildSessionKey(childSessionKey)?.pauseReason).toBeUndefined();
|
||||
expect(detachedTaskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ runId: "run-yield-race", status: "cancelled" }),
|
||||
);
|
||||
const [finalizeArgs] = detachedTaskRuntimeMocks.finalizeTaskRunByRunId.mock.calls[0] ?? [];
|
||||
const killedAt = (finalizeArgs as { endedAt?: number } | undefined)?.endedAt;
|
||||
expect(killedAt).toBeGreaterThan(yieldedAt);
|
||||
|
||||
const repeated = await killSubagentRunAdmin({
|
||||
cfg: cfgWithSessionStore(storePath),
|
||||
sessionKey: childSessionKey,
|
||||
});
|
||||
expect(repeated).toMatchObject({
|
||||
found: true,
|
||||
killed: false,
|
||||
targetState: {
|
||||
state: "terminal",
|
||||
task: { status: "cancelled", endedAt: killedAt },
|
||||
},
|
||||
});
|
||||
const [repeatedFinalizeArgs] =
|
||||
detachedTaskRuntimeMocks.finalizeTaskRunByRunId.mock.calls.at(-1) ?? [];
|
||||
expect((repeatedFinalizeArgs as { endedAt?: number } | undefined)?.endedAt).toBe(killedAt);
|
||||
});
|
||||
|
||||
it("does not mark a finalizing run killed when its abort is rejected", async () => {
|
||||
const childSessionKey = "agent:main:subagent:worker-finalizing";
|
||||
const storePath = writeSessionStoreFixture("admin-kill-finalizing", {
|
||||
|
|
@ -644,7 +1051,7 @@ describe("killSubagentRunAdmin", () => {
|
|||
expect(persisted[childSessionKey]?.abortedLastRun).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not kill a newest finished run when only a stale older row is still active", async () => {
|
||||
it("does not kill a newest finalizing run when only a stale older row is still active", async () => {
|
||||
const childSessionKey = "agent:main:subagent:worker-stale-admin";
|
||||
|
||||
addSubagentRunForTests({
|
||||
|
|
@ -679,6 +1086,10 @@ describe("killSubagentRunAdmin", () => {
|
|||
|
||||
expect(result.found).toBe(true);
|
||||
expect(result.killed).toBe(false);
|
||||
if (!result.found) {
|
||||
throw new Error("expected finalizing subagent run");
|
||||
}
|
||||
expect(result.targetState).toEqual({ state: "finalizing" });
|
||||
expect(result.runId).toBe("run-current-admin");
|
||||
expect(result.sessionKey).toBe(childSessionKey);
|
||||
});
|
||||
|
|
@ -717,6 +1128,9 @@ describe("killSubagentRunAdmin", () => {
|
|||
|
||||
expect(result.found).toBe(true);
|
||||
expect(result.killed).toBe(true);
|
||||
if (!result.found) {
|
||||
throw new Error("expected tracked subagent run");
|
||||
}
|
||||
expect(result.runId).toBe("run-worker-store-fail");
|
||||
expect(result.sessionKey).toBe(childSessionKey);
|
||||
expect(getSubagentRunByChildSessionKey(childSessionKey)?.endedAt).toBeTypeOf("number");
|
||||
|
|
@ -785,7 +1199,7 @@ describe("killControlledSubagentRun", () => {
|
|||
expect(getSubagentRunByChildSessionKey(childSessionKey)?.runId).toBe("run-current");
|
||||
});
|
||||
|
||||
it("does not kill a stale child row while cascading descendants from an ended current parent", async () => {
|
||||
it("kills a yielded descendant without reviving a stale child row", async () => {
|
||||
const parentSessionKey = "agent:main:subagent:kill-parent";
|
||||
const childSessionKey = `${parentSessionKey}:subagent:child`;
|
||||
const leafSessionKey = `${childSessionKey}:subagent:leaf`;
|
||||
|
|
@ -837,6 +1251,8 @@ describe("killControlledSubagentRun", () => {
|
|||
cleanup: "keep",
|
||||
createdAt: Date.now() - 1_000,
|
||||
startedAt: Date.now() - 900,
|
||||
endedAt: Date.now() - 800,
|
||||
pauseReason: "sessions_yield",
|
||||
});
|
||||
|
||||
const result = await killControlledSubagentRun({
|
||||
|
|
@ -980,6 +1396,67 @@ describe("killAllControlledSubagentRuns", () => {
|
|||
testing.setDepsForTest();
|
||||
});
|
||||
|
||||
it("continues bulk cancellation after one registry persistence failure", async () => {
|
||||
let persistenceAttempts = 0;
|
||||
subagentRegistryTesting.setDepsForTest({
|
||||
cleanupBrowserSessionsForLifecycleEnd: async () => {},
|
||||
ensureContextEnginesInitialized: () => {},
|
||||
ensureRuntimePluginsLoaded: () => {},
|
||||
getSubagentRunsSnapshotForRead: (runs) => new Map(runs),
|
||||
persistSubagentRunsToDisk: () => {},
|
||||
persistSubagentRunsToDiskOrThrow: () => {
|
||||
persistenceAttempts += 1;
|
||||
if (persistenceAttempts === 1) {
|
||||
throw new Error("sqlite busy");
|
||||
}
|
||||
},
|
||||
restoreSubagentRunsFromDisk: () => 0,
|
||||
resolveContextEngine: async () => ({
|
||||
info: { id: "test", name: "Test" },
|
||||
assemble: async ({ messages }) => ({ messages, estimatedTokens: 0 }),
|
||||
compact: async () => ({ ok: true, compacted: false }),
|
||||
ingest: async () => ({ ingested: false }),
|
||||
}),
|
||||
});
|
||||
const first = {
|
||||
runId: "run-bulk-persistence-failure-first",
|
||||
childSessionKey: "agent:main:subagent:bulk-persistence-failure-first",
|
||||
controllerSessionKey: "agent:main:main",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "first bulk task",
|
||||
cleanup: "keep" as const,
|
||||
createdAt: Date.now() - 2_000,
|
||||
startedAt: Date.now() - 1_900,
|
||||
};
|
||||
const second = {
|
||||
...first,
|
||||
runId: "run-bulk-persistence-failure-second",
|
||||
childSessionKey: "agent:main:subagent:bulk-persistence-failure-second",
|
||||
task: "second bulk task",
|
||||
createdAt: Date.now() - 1_000,
|
||||
startedAt: Date.now() - 900,
|
||||
};
|
||||
addSubagentRunForTests(first);
|
||||
addSubagentRunForTests(second);
|
||||
|
||||
const result = await killAllControlledSubagentRuns({
|
||||
cfg: cfgWithSessionStore(),
|
||||
controller: {
|
||||
controllerSessionKey: "agent:main:main",
|
||||
callerSessionKey: "agent:main:main",
|
||||
callerIsSubagent: false,
|
||||
controlScope: "children",
|
||||
},
|
||||
runs: [first, second],
|
||||
});
|
||||
|
||||
expect(result).toEqual({ status: "ok", killed: 1, labels: ["second bulk task"] });
|
||||
expect(persistenceAttempts).toBe(2);
|
||||
expect(getSubagentRunByChildSessionKey(first.childSessionKey)?.endedAt).toBeUndefined();
|
||||
expect(getSubagentRunByChildSessionKey(second.childSessionKey)?.endedAt).toBeTypeOf("number");
|
||||
});
|
||||
|
||||
it("ignores stale run snapshots in bulk kill requests", async () => {
|
||||
const childSessionKey = "agent:main:subagent:stale-kill-all-worker";
|
||||
const storePath = writeSessionStoreFixture("stale-kill-all", {
|
||||
|
|
@ -1036,7 +1513,7 @@ describe("killAllControlledSubagentRuns", () => {
|
|||
expect(getSubagentRunByChildSessionKey(childSessionKey)?.runId).toBe("run-current-bulk");
|
||||
});
|
||||
|
||||
it("does not let a stale bulk entry suppress the current live entry for the same child key", async () => {
|
||||
it("does not let a stale bulk entry suppress the current yielded entry", async () => {
|
||||
const childSessionKey = "agent:main:subagent:stale-kill-all-shadow-worker";
|
||||
const storePath = writeSessionStoreFixture("stale-kill-all-shadow", {
|
||||
[childSessionKey]: {
|
||||
|
|
@ -1054,6 +1531,8 @@ describe("killAllControlledSubagentRuns", () => {
|
|||
cleanup: "keep",
|
||||
createdAt: Date.now() - 4_000,
|
||||
startedAt: Date.now() - 3_000,
|
||||
endedAt: Date.now() - 2_000,
|
||||
pauseReason: "sessions_yield",
|
||||
});
|
||||
|
||||
const result = await killAllControlledSubagentRuns({
|
||||
|
|
@ -1086,6 +1565,8 @@ describe("killAllControlledSubagentRuns", () => {
|
|||
cleanup: "keep",
|
||||
createdAt: Date.now() - 4_000,
|
||||
startedAt: Date.now() - 3_000,
|
||||
endedAt: Date.now() - 2_000,
|
||||
pauseReason: "sessions_yield",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
|
|
|||
|
|
@ -15,6 +15,10 @@ import { logVerbose } from "../globals.js";
|
|||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import { isSubagentSessionKey, parseAgentSessionKey } from "../routing/session-key.js";
|
||||
import { createLazyImportLoader } from "../shared/lazy-promise.js";
|
||||
import {
|
||||
SUBAGENT_KILL_TASK_ERROR,
|
||||
type DetachedTaskTerminalState,
|
||||
} from "../tasks/detached-task-runtime-contract.js";
|
||||
import { INTERNAL_MESSAGE_CHANNEL } from "../utils/message-channel.js";
|
||||
import { AGENT_LANE_SUBAGENT } from "./lanes.js";
|
||||
import {
|
||||
|
|
@ -22,7 +26,12 @@ import {
|
|||
waitForAgentRunAndReadUpdatedAssistantReply,
|
||||
} from "./run-wait.js";
|
||||
import { resolveStoredSubagentCapabilities } from "./subagent-capabilities.js";
|
||||
import { SUBAGENT_ENDED_REASON_KILLED } from "./subagent-lifecycle-events.js";
|
||||
import { buildLatestSubagentRunIndex, resolveSessionEntryForKey } from "./subagent-list.js";
|
||||
import {
|
||||
resolveFinalizedSubagentTaskState,
|
||||
resolveKilledSubagentTaskEndedAt,
|
||||
} from "./subagent-registry-completion.js";
|
||||
import { subagentRuns } from "./subagent-registry-memory.js";
|
||||
import {
|
||||
getLatestSubagentRunByChildSessionKey,
|
||||
|
|
@ -178,12 +187,109 @@ function isFinishedForSteerControl(entry: SubagentRunRecord, hasPendingDescendan
|
|||
return Boolean(entry.endedAt) && entry.pauseReason !== "sessions_yield" && !hasPendingDescendants;
|
||||
}
|
||||
|
||||
type SubagentKillTargetState =
|
||||
| { state: "finalizing" }
|
||||
| { state: "terminal"; task: DetachedTaskTerminalState };
|
||||
|
||||
function resolveSubagentKillTargetState(
|
||||
entry: SubagentRunRecord,
|
||||
): SubagentKillTargetState | undefined {
|
||||
if (
|
||||
entry.endedReason === SUBAGENT_ENDED_REASON_KILLED &&
|
||||
entry.suppressAnnounceReason !== "steer-restart"
|
||||
) {
|
||||
const taskEndedAt = resolveKilledSubagentTaskEndedAt(entry);
|
||||
return typeof taskEndedAt === "number"
|
||||
? {
|
||||
state: "terminal",
|
||||
task: {
|
||||
status: "cancelled",
|
||||
endedAt: taskEndedAt,
|
||||
lastEventAt: taskEndedAt,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
progressSummary: entry.completion?.resultText ?? undefined,
|
||||
terminalSummary: null,
|
||||
},
|
||||
}
|
||||
: undefined;
|
||||
}
|
||||
const terminal = resolveFinalizedSubagentTaskState(entry);
|
||||
if (terminal) {
|
||||
return { state: "terminal", task: terminal };
|
||||
}
|
||||
return typeof entry.endedAt === "number" &&
|
||||
entry.pauseReason !== "sessions_yield" &&
|
||||
(entry.endedReason !== SUBAGENT_ENDED_REASON_KILLED ||
|
||||
entry.suppressAnnounceReason === "steer-restart")
|
||||
? { state: "finalizing" }
|
||||
: undefined;
|
||||
}
|
||||
|
||||
async function persistSubagentAbortedLastRun(params: {
|
||||
childSessionKey: string;
|
||||
storePath: string;
|
||||
hasSessionEntry: boolean;
|
||||
abortedLastRun: boolean;
|
||||
}): Promise<void> {
|
||||
if (!params.hasSessionEntry) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await subagentControlDeps.patchSessionEntry(
|
||||
{ storePath: params.storePath, sessionKey: params.childSessionKey },
|
||||
(current) => ({
|
||||
...current,
|
||||
abortedLastRun: params.abortedLastRun,
|
||||
updatedAt: Date.now(),
|
||||
}),
|
||||
{ replaceEntry: true },
|
||||
);
|
||||
} catch (error) {
|
||||
logVerbose(
|
||||
`subagents control kill: failed to persist abortedLastRun=${params.abortedLastRun} for ${params.childSessionKey}: ${formatErrorMessage(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function markSubagentRunTerminatedBestEffort(
|
||||
params: Parameters<typeof markSubagentRunTerminated>[0],
|
||||
): number {
|
||||
try {
|
||||
return markSubagentRunTerminated(params);
|
||||
} catch (error) {
|
||||
// The registry transition rolled back atomically. Keep multi-run control
|
||||
// moving so one persistence failure cannot leave siblings running.
|
||||
logVerbose(
|
||||
`subagents control kill: failed to persist ${params.runId ?? params.childSessionKey ?? "unknown"}: ${formatErrorMessage(error)}`,
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
async function killSubagentRun(params: {
|
||||
cfg: OpenClawConfig;
|
||||
entry: SubagentRunRecord;
|
||||
cache: Map<string, Record<string, SessionEntry>>;
|
||||
}): Promise<{ killed: boolean; sessionId?: string }> {
|
||||
if (params.entry.endedAt) {
|
||||
}): Promise<{
|
||||
killed: boolean;
|
||||
sessionId?: string;
|
||||
targetState?: SubagentKillTargetState;
|
||||
}> {
|
||||
const initialTargetState = resolveSubagentKillTargetState(params.entry);
|
||||
if (initialTargetState) {
|
||||
if (
|
||||
params.entry.endedReason === SUBAGENT_ENDED_REASON_KILLED &&
|
||||
params.entry.suppressAnnounceReason !== "steer-restart"
|
||||
) {
|
||||
markSubagentRunTerminatedBestEffort({
|
||||
runId: params.entry.runId,
|
||||
childSessionKey: params.entry.childSessionKey,
|
||||
reason: "killed",
|
||||
});
|
||||
}
|
||||
return { killed: false, targetState: initialTargetState };
|
||||
}
|
||||
if (params.entry.endedAt && params.entry.pauseReason !== "sessions_yield") {
|
||||
return { killed: false };
|
||||
}
|
||||
const childSessionKey = params.entry.childSessionKey;
|
||||
|
|
@ -194,6 +300,20 @@ async function killSubagentRun(params: {
|
|||
});
|
||||
const sessionId = resolved.entry?.sessionId;
|
||||
const runtime = await resolveSubagentControlRuntime();
|
||||
const targetStateAfterRuntimeLoad = resolveSubagentKillTargetState(params.entry);
|
||||
if (targetStateAfterRuntimeLoad) {
|
||||
if (
|
||||
params.entry.endedReason === SUBAGENT_ENDED_REASON_KILLED &&
|
||||
params.entry.suppressAnnounceReason !== "steer-restart"
|
||||
) {
|
||||
markSubagentRunTerminatedBestEffort({
|
||||
runId: params.entry.runId,
|
||||
childSessionKey,
|
||||
reason: "killed",
|
||||
});
|
||||
}
|
||||
return { killed: false, sessionId, targetState: targetStateAfterRuntimeLoad };
|
||||
}
|
||||
const active = sessionId ? runtime.isEmbeddedAgentRunActive(sessionId) : false;
|
||||
const aborted = sessionId ? runtime.abortEmbeddedAgentRun(sessionId) : false;
|
||||
const cleared = runtime.clearSessionQueues([childSessionKey, sessionId]);
|
||||
|
|
@ -205,30 +325,43 @@ async function killSubagentRun(params: {
|
|||
if (active && !aborted) {
|
||||
return { killed: false, sessionId };
|
||||
}
|
||||
if (resolved.entry) {
|
||||
try {
|
||||
await subagentControlDeps.patchSessionEntry(
|
||||
{ storePath: resolved.storePath, sessionKey: childSessionKey },
|
||||
(current) => ({
|
||||
...current,
|
||||
abortedLastRun: true,
|
||||
updatedAt: Date.now(),
|
||||
}),
|
||||
{ replaceEntry: true },
|
||||
);
|
||||
} catch (error) {
|
||||
logVerbose(
|
||||
`subagents control kill: failed to persist abortedLastRun for ${childSessionKey}: ${formatErrorMessage(error)}`,
|
||||
);
|
||||
const persistAbortedLastRun = (abortedLastRun: boolean) =>
|
||||
persistSubagentAbortedLastRun({
|
||||
childSessionKey,
|
||||
storePath: resolved.storePath,
|
||||
hasSessionEntry: resolved.entry !== undefined,
|
||||
abortedLastRun,
|
||||
});
|
||||
await persistAbortedLastRun(true);
|
||||
const targetState = resolveSubagentKillTargetState(params.entry);
|
||||
if (targetState) {
|
||||
const killedTarget =
|
||||
targetState.state === "terminal" &&
|
||||
targetState.task.status === "cancelled" &&
|
||||
targetState.task.error === SUBAGENT_KILL_TASK_ERROR;
|
||||
if (killedTarget) {
|
||||
markSubagentRunTerminatedBestEffort({
|
||||
runId: params.entry.runId,
|
||||
childSessionKey,
|
||||
reason: "killed",
|
||||
});
|
||||
} else {
|
||||
await persistAbortedLastRun(false);
|
||||
}
|
||||
const killed =
|
||||
killedTarget && (aborted || cleared.followupCleared > 0 || cleared.laneCleared > 0);
|
||||
return { killed, sessionId, targetState };
|
||||
}
|
||||
const marked = markSubagentRunTerminated({
|
||||
const marked = markSubagentRunTerminatedBestEffort({
|
||||
runId: params.entry.runId,
|
||||
childSessionKey,
|
||||
reason: "killed",
|
||||
});
|
||||
const killed = marked > 0 || aborted || cleared.followupCleared > 0 || cleared.laneCleared > 0;
|
||||
return { killed, sessionId };
|
||||
return {
|
||||
killed,
|
||||
sessionId,
|
||||
};
|
||||
}
|
||||
|
||||
async function cascadeKillChildren(params: {
|
||||
|
|
@ -253,10 +386,7 @@ async function cascadeKillChildren(params: {
|
|||
) {
|
||||
continue;
|
||||
}
|
||||
const existing = childRunsBySessionKey.get(childKey);
|
||||
if (!existing || run.createdAt >= existing.createdAt) {
|
||||
childRunsBySessionKey.set(childKey, run);
|
||||
}
|
||||
childRunsBySessionKey.set(childKey, run);
|
||||
}
|
||||
const childRuns = Array.from(childRunsBySessionKey.values());
|
||||
const seenChildSessionKeys = params.seenChildSessionKeys ?? new Set<string>();
|
||||
|
|
@ -270,7 +400,7 @@ async function cascadeKillChildren(params: {
|
|||
}
|
||||
seenChildSessionKeys.add(childKey);
|
||||
|
||||
if (!run.endedAt) {
|
||||
if (!run.endedAt || run.pauseReason === "sessions_yield") {
|
||||
const stopResult = await killSubagentRun({
|
||||
cfg: params.cfg,
|
||||
entry: run,
|
||||
|
|
@ -324,7 +454,7 @@ export async function killAllControlledSubagentRuns(params: {
|
|||
}
|
||||
seenChildSessionKeys.add(childKey);
|
||||
|
||||
if (!currentEntry.endedAt) {
|
||||
if (!currentEntry.endedAt || currentEntry.pauseReason === "sessions_yield") {
|
||||
const stopResult = await killSubagentRun({ cfg: params.cfg, entry: currentEntry, cache });
|
||||
if (stopResult.killed) {
|
||||
killed += 1;
|
||||
|
|
@ -445,10 +575,38 @@ export async function killSubagentRunAdmin(params: { cfg: OpenClawConfig; sessio
|
|||
cache: killCache,
|
||||
seenChildSessionKeys,
|
||||
});
|
||||
// Descendant cleanup can yield long enough for the target run to finish.
|
||||
// Return the freshest registry state so task cancellation cannot make a stale kill sticky.
|
||||
const targetState = resolveSubagentKillTargetState(entry) ?? stopResult.targetState;
|
||||
const killedTarget =
|
||||
targetState?.state === "terminal" &&
|
||||
targetState.task.status === "cancelled" &&
|
||||
targetState.task.error === SUBAGENT_KILL_TASK_ERROR;
|
||||
const stopResultAlreadyClearedAbort =
|
||||
stopResult.targetState !== undefined &&
|
||||
!(
|
||||
stopResult.targetState.state === "terminal" &&
|
||||
stopResult.targetState.task.status === "cancelled" &&
|
||||
stopResult.targetState.task.error === SUBAGENT_KILL_TASK_ERROR
|
||||
);
|
||||
if (targetState && !killedTarget && !stopResultAlreadyClearedAbort) {
|
||||
const resolved = resolveSessionEntryForKey({
|
||||
cfg: params.cfg,
|
||||
key: targetSessionKey,
|
||||
cache: killCache,
|
||||
});
|
||||
await persistSubagentAbortedLastRun({
|
||||
childSessionKey: targetSessionKey,
|
||||
storePath: resolved.storePath,
|
||||
hasSessionEntry: resolved.entry !== undefined,
|
||||
abortedLastRun: false,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
found: true as const,
|
||||
killed: stopResult.killed || cascade.killed > 0,
|
||||
...(targetState ? { targetState } : {}),
|
||||
runId: entry.runId,
|
||||
sessionKey: entry.childSessionKey,
|
||||
cascadeKilled: cascade.killed,
|
||||
|
|
|
|||
|
|
@ -23,6 +23,62 @@ function baseRun(overrides: Partial<LegacySubagentRunRecord> = {}): LegacySubage
|
|||
}
|
||||
|
||||
describe("normalizeSubagentRunState", () => {
|
||||
it("normalizes durable task ownership and generation metadata", () => {
|
||||
const entry = normalizeSubagentRunState(
|
||||
baseRun({ taskRunId: " run-task-owner ", generation: 2 }),
|
||||
);
|
||||
const malformed = normalizeSubagentRunState(
|
||||
baseRun({ taskRunId: " ", generation: Number.NaN }),
|
||||
);
|
||||
const nonString = normalizeSubagentRunState({
|
||||
...baseRun(),
|
||||
taskRunId: 42,
|
||||
} as unknown as SubagentRunRecord);
|
||||
|
||||
expect(entry).toMatchObject({ taskRunId: "run-task-owner", generation: 2 });
|
||||
expect(malformed.taskRunId).toBeUndefined();
|
||||
expect(malformed.generation).toBeUndefined();
|
||||
expect(nonString.taskRunId).toBeUndefined();
|
||||
});
|
||||
|
||||
it("normalizes the durable delete-dispatch boundary", () => {
|
||||
const valid = normalizeSubagentRunState(baseRun({ deleteCleanupDispatchedAt: 200 }));
|
||||
const malformed = normalizeSubagentRunState(baseRun({ deleteCleanupDispatchedAt: Number.NaN }));
|
||||
|
||||
expect(valid.deleteCleanupDispatchedAt).toBe(200);
|
||||
expect(malformed.deleteCleanupDispatchedAt).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves valid killed reconciliation ownership metadata", () => {
|
||||
const entry = normalizeSubagentRunState(
|
||||
baseRun({
|
||||
suppressCompletionDelivery: true,
|
||||
killReconciliation: {
|
||||
killedAt: 200,
|
||||
suppressTaskDelivery: true,
|
||||
supersededAt: 300,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(entry.killReconciliation).toEqual({
|
||||
killedAt: 200,
|
||||
suppressTaskDelivery: true,
|
||||
supersededAt: 300,
|
||||
});
|
||||
expect(entry.suppressCompletionDelivery).toBe(true);
|
||||
});
|
||||
|
||||
it("drops malformed killed reconciliation metadata", () => {
|
||||
const entry = normalizeSubagentRunState(
|
||||
baseRun({
|
||||
killReconciliation: { killedAt: Number.NaN },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(entry.killReconciliation).toBeUndefined();
|
||||
});
|
||||
|
||||
it("migrates legacy pending delivery fields into nested completion and delivery state", () => {
|
||||
// Restored runs may still carry flat pendingFinalDelivery fields from older
|
||||
// builds; normalization must preserve retry payloads before stripping them.
|
||||
|
|
|
|||
|
|
@ -48,6 +48,34 @@ type LegacyDeliveryState = SubagentCompletionDeliveryState & {
|
|||
/** Normalizes legacy subagent run fields into nested execution/completion/delivery state. */
|
||||
export function normalizeSubagentRunState(entry: SubagentRunRecord): SubagentRunRecord {
|
||||
const legacy = entry as LegacySubagentRunRecord;
|
||||
const taskRunId = typeof entry.taskRunId === "string" ? entry.taskRunId.trim() : "";
|
||||
entry.taskRunId = taskRunId || undefined;
|
||||
entry.generation =
|
||||
typeof entry.generation === "number" &&
|
||||
Number.isSafeInteger(entry.generation) &&
|
||||
entry.generation > 0
|
||||
? entry.generation
|
||||
: undefined;
|
||||
entry.deleteCleanupDispatchedAt = Number.isFinite(entry.deleteCleanupDispatchedAt)
|
||||
? entry.deleteCleanupDispatchedAt
|
||||
: undefined;
|
||||
entry.suppressCompletionDelivery = entry.suppressCompletionDelivery === true ? true : undefined;
|
||||
const killReconciliation = entry.killReconciliation;
|
||||
if (
|
||||
!killReconciliation ||
|
||||
typeof killReconciliation !== "object" ||
|
||||
!Number.isFinite(killReconciliation.killedAt)
|
||||
) {
|
||||
delete entry.killReconciliation;
|
||||
} else {
|
||||
entry.killReconciliation = {
|
||||
killedAt: killReconciliation.killedAt,
|
||||
suppressTaskDelivery: killReconciliation.suppressTaskDelivery === true ? true : undefined,
|
||||
supersededAt: Number.isFinite(killReconciliation.supersededAt)
|
||||
? killReconciliation.supersededAt
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
entry.execution = mergeExecutionState(entry.execution, buildExecutionState(entry));
|
||||
entry.completion = mergeCompletionState(entry.completion, buildCompletionState(entry, legacy));
|
||||
entry.delivery = mergeDeliveryState(entry, entry.delivery, buildDeliveryState(entry, legacy));
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import path from "node:path";
|
|||
import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import { updateSessionStore } from "../config/sessions/store.js";
|
||||
import { buildSubagentList } from "./subagent-list.js";
|
||||
import { buildLatestSubagentRunIndex, buildSubagentList } from "./subagent-list.js";
|
||||
import {
|
||||
addSubagentRunForTests,
|
||||
resetSubagentRegistryForTests,
|
||||
|
|
@ -34,6 +34,34 @@ beforeEach(() => {
|
|||
resetSubagentRegistryForTests();
|
||||
});
|
||||
|
||||
describe("buildLatestSubagentRunIndex", () => {
|
||||
it("prefers the newer generation when runs share a creation timestamp", () => {
|
||||
const childSessionKey = "agent:main:subagent:reused";
|
||||
const makeRun = (runId: string, generation: number): SubagentRunRecord => ({
|
||||
runId,
|
||||
generation,
|
||||
childSessionKey,
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: runId,
|
||||
cleanup: "keep",
|
||||
createdAt: 1_000,
|
||||
startedAt: 1_000,
|
||||
});
|
||||
const older = makeRun("run-older", 1);
|
||||
const newer = makeRun("run-newer", 2);
|
||||
|
||||
const index = buildLatestSubagentRunIndex(
|
||||
new Map([
|
||||
[older.runId, older],
|
||||
[newer.runId, newer],
|
||||
]),
|
||||
);
|
||||
|
||||
expect(index.latestByChildSessionKey.get(childSessionKey)).toBe(newer);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildSubagentList", () => {
|
||||
it("returns empty active and recent sections when no runs exist", () => {
|
||||
const cfg = {
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import {
|
|||
} from "./subagent-registry-read.js";
|
||||
import { getSubagentRunsSnapshotForRead } from "./subagent-registry-state.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
import { compareSubagentRunGeneration } from "./subagent-run-generation.js";
|
||||
import {
|
||||
hasSubagentRunEnded,
|
||||
isLiveUnendedSubagentRun,
|
||||
|
|
@ -103,7 +104,7 @@ export function buildLatestSubagentRunIndex(
|
|||
continue;
|
||||
}
|
||||
const existing = latestByChildSessionKey.get(childSessionKey);
|
||||
if (!existing || entry.createdAt > existing.createdAt) {
|
||||
if (!existing || compareSubagentRunGeneration(entry, existing) > 0) {
|
||||
latestByChildSessionKey.set(childSessionKey, entry);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,8 +5,14 @@
|
|||
*/
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { getGlobalHookRunner } from "../plugins/hook-runner-global.js";
|
||||
import {
|
||||
SUBAGENT_KILL_TASK_ERROR,
|
||||
type DetachedTaskTerminalState,
|
||||
} from "../tasks/detached-task-runtime-contract.js";
|
||||
import { resolveRequiredCompletionTerminalResult } from "../tasks/task-completion-contract.js";
|
||||
import type { SubagentRunOutcome } from "./subagent-announce-output.js";
|
||||
import {
|
||||
SUBAGENT_ENDED_REASON_KILLED,
|
||||
SUBAGENT_ENDED_OUTCOME_ERROR,
|
||||
SUBAGENT_ENDED_OUTCOME_OK,
|
||||
SUBAGENT_ENDED_OUTCOME_TIMEOUT,
|
||||
|
|
@ -62,6 +68,74 @@ export function shouldUpdateRunOutcome(
|
|||
);
|
||||
}
|
||||
|
||||
/** Returns the complete task projection only after completion capture has settled. */
|
||||
export function resolveFinalizedSubagentTaskState(
|
||||
entry: SubagentRunRecord,
|
||||
): DetachedTaskTerminalState | undefined {
|
||||
const endedAt = entry.endedAt;
|
||||
const outcome = entry.outcome;
|
||||
const completion = entry.completion;
|
||||
if (
|
||||
typeof endedAt !== "number" ||
|
||||
!outcome ||
|
||||
entry.pauseReason === "sessions_yield" ||
|
||||
(completion?.resultText === undefined && typeof completion?.capturedAt !== "number")
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
const progressSummary = completion.resultText ?? undefined;
|
||||
if (
|
||||
entry.endedReason === SUBAGENT_ENDED_REASON_KILLED &&
|
||||
entry.suppressAnnounceReason !== "steer-restart"
|
||||
) {
|
||||
return {
|
||||
status: "cancelled",
|
||||
endedAt,
|
||||
lastEventAt: endedAt,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
progressSummary,
|
||||
terminalSummary: null,
|
||||
};
|
||||
}
|
||||
if (outcome.status === "ok") {
|
||||
const terminal =
|
||||
entry.expectsCompletionMessage === true
|
||||
? resolveRequiredCompletionTerminalResult(completion.resultText)
|
||||
: {};
|
||||
return {
|
||||
status: "succeeded",
|
||||
endedAt,
|
||||
lastEventAt: endedAt,
|
||||
progressSummary,
|
||||
terminalSummary: terminal.terminalSummary ?? null,
|
||||
terminalOutcome: terminal.terminalOutcome,
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: outcome.status === "timeout" ? "timed_out" : "failed",
|
||||
endedAt,
|
||||
lastEventAt: endedAt,
|
||||
error: outcome.status === "error" ? outcome.error : undefined,
|
||||
progressSummary,
|
||||
terminalSummary: null,
|
||||
};
|
||||
}
|
||||
|
||||
/** Preserves execution end time, except when a paused run was killed after its yield. */
|
||||
export function resolveKilledSubagentTaskEndedAt(entry: SubagentRunRecord): number | undefined {
|
||||
if (entry.killReconciliation) {
|
||||
return entry.killReconciliation.killedAt;
|
||||
}
|
||||
const endedAt = entry.endedAt;
|
||||
const cleanupCompletedAt = entry.cleanupCompletedAt;
|
||||
return entry.suppressAnnounceReason === "killed" &&
|
||||
typeof endedAt === "number" &&
|
||||
typeof cleanupCompletedAt === "number" &&
|
||||
cleanupCompletedAt > endedAt
|
||||
? cleanupCompletedAt
|
||||
: endedAt;
|
||||
}
|
||||
|
||||
/** Maps registry run outcome to lifecycle event outcome. */
|
||||
export function resolveLifecycleOutcomeFromRunOutcome(
|
||||
outcome: SubagentRunOutcome | undefined,
|
||||
|
|
|
|||
|
|
@ -14,7 +14,10 @@ import { defaultRuntime } from "../runtime.js";
|
|||
import { truncateUtf8Prefix } from "../utils/utf8-truncate.js";
|
||||
import { withSubagentOutcomeTiming } from "./subagent-announce-output.js";
|
||||
import { getDeliveryAttemptCount, getDeliveryLastError } from "./subagent-delivery-state.js";
|
||||
import { SUBAGENT_ENDED_REASON_ERROR } from "./subagent-lifecycle-events.js";
|
||||
import {
|
||||
SUBAGENT_ENDED_REASON_ERROR,
|
||||
SUBAGENT_ENDED_REASON_KILLED,
|
||||
} from "./subagent-lifecycle-events.js";
|
||||
import { shouldUpdateRunOutcome } from "./subagent-registry-completion.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
import {
|
||||
|
|
@ -23,6 +26,7 @@ import {
|
|||
resolveSubagentSessionStatus,
|
||||
} from "./subagent-session-metrics.js";
|
||||
import {
|
||||
resolveCompletionFromSessionEntry,
|
||||
resolveSubagentRunOrphanReason,
|
||||
type SubagentRunOrphanReason,
|
||||
} from "./subagent-session-reconciliation.js";
|
||||
|
|
@ -33,6 +37,7 @@ export {
|
|||
resolveSubagentSessionStatus,
|
||||
} from "./subagent-session-metrics.js";
|
||||
|
||||
export const PROVISIONAL_KILL_RECONCILIATION_MS = 5 * 60_000;
|
||||
export const MIN_ANNOUNCE_RETRY_DELAY_MS = 1_000;
|
||||
const MAX_ANNOUNCE_RETRY_DELAY_MS = 8_000;
|
||||
export const MAX_ANNOUNCE_RETRY_COUNT = 3;
|
||||
|
|
@ -90,7 +95,10 @@ export function logAnnounceGiveUp(entry: SubagentRunRecord, reason: "retry-limit
|
|||
}
|
||||
|
||||
/** Persists child session timing/status derived from the subagent registry row. */
|
||||
export async function persistSubagentSessionTiming(entry: SubagentRunRecord) {
|
||||
export async function persistSubagentSessionTiming(
|
||||
entry: SubagentRunRecord,
|
||||
options?: { isCurrentGeneration?: () => boolean },
|
||||
) {
|
||||
const childSessionKey = entry.childSessionKey?.trim();
|
||||
if (!childSessionKey) {
|
||||
return;
|
||||
|
|
@ -111,6 +119,27 @@ export async function persistSubagentSessionTiming(entry: SubagentRunRecord) {
|
|||
await patchSessionEntry(
|
||||
{ storePath, sessionKey: childSessionKey },
|
||||
(sessionEntry) => {
|
||||
// Recheck under the session-store write lock. A completion may have
|
||||
// waited behind a steer/restart that transferred this session's ownership.
|
||||
if (options?.isCurrentGeneration && !options.isCurrentGeneration()) {
|
||||
return null;
|
||||
}
|
||||
if (status === "killed") {
|
||||
const existingCompletion = resolveCompletionFromSessionEntry(sessionEntry, Date.now(), {
|
||||
notBeforeMs: entry.startedAt ?? entry.createdAt,
|
||||
});
|
||||
if (existingCompletion && existingCompletion.reason !== SUBAGENT_ENDED_REASON_KILLED) {
|
||||
// A provider result already reached durable session state. The kill
|
||||
// marker is provisional and must not erase restart reconciliation evidence
|
||||
// or leave the session looking aborted after that completion won.
|
||||
if (sessionEntry.abortedLastRun !== true) {
|
||||
return null;
|
||||
}
|
||||
const completedEntry = { ...sessionEntry };
|
||||
delete completedEntry.abortedLastRun;
|
||||
return completedEntry;
|
||||
}
|
||||
}
|
||||
const next = { ...sessionEntry };
|
||||
|
||||
if (typeof startedAt === "number" && Number.isFinite(startedAt)) {
|
||||
|
|
@ -136,6 +165,9 @@ export async function persistSubagentSessionTiming(entry: SubagentRunRecord) {
|
|||
} else {
|
||||
delete next.status;
|
||||
}
|
||||
if (status && status !== "killed") {
|
||||
delete next.abortedLastRun;
|
||||
}
|
||||
return next;
|
||||
},
|
||||
{ replaceEntry: true },
|
||||
|
|
@ -286,6 +318,11 @@ export function reconcileOrphanedRestoredRuns(params: {
|
|||
const now = Date.now();
|
||||
let changed = false;
|
||||
for (const [runId, entry] of params.runs.entries()) {
|
||||
if (entry.killReconciliation) {
|
||||
// Provider completion may still repair this provisional kill. The
|
||||
// sweeper owns its bounded reconciliation even when the session vanished.
|
||||
continue;
|
||||
}
|
||||
const orphanReason = resolveSubagentRunOrphanReason({
|
||||
entry,
|
||||
includeStaleUnended: true,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
|
@ -26,6 +26,11 @@ function isAwaitingCompletionAnnounceForMaintenance(entry: SubagentRunRecord): b
|
|||
}
|
||||
|
||||
function shouldPreserveForMaintenance(entry: SubagentRunRecord): boolean {
|
||||
if (entry.killReconciliation) {
|
||||
// The killed row is a reconciliation tombstone. Its session owns the
|
||||
// provider result until the sweeper accepts completion or finalizes cancellation.
|
||||
return true;
|
||||
}
|
||||
if (isCleanupCompleteForMaintenance(entry)) {
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,6 +36,36 @@ function toRunMap(runs: SubagentRunRecord[]): Map<string, SubagentRunRecord> {
|
|||
}
|
||||
|
||||
describe("subagent registry query regressions", () => {
|
||||
it("selects the newer generation when child runs share a timestamp", () => {
|
||||
const childSessionKey = "agent:main:subagent:same-millisecond";
|
||||
const runs = toRunMap([
|
||||
makeRun({
|
||||
runId: "run-old-tombstone",
|
||||
childSessionKey,
|
||||
requesterSessionKey: "agent:main:old-parent",
|
||||
generation: 1,
|
||||
createdAt: 100,
|
||||
endedAt: 100,
|
||||
}),
|
||||
makeRun({
|
||||
runId: "run-live-successor",
|
||||
childSessionKey,
|
||||
requesterSessionKey: "agent:main:new-parent",
|
||||
generation: 2,
|
||||
createdAt: 100,
|
||||
startedAt: 100,
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(isSubagentSessionRunActiveFromRuns(runs, childSessionKey)).toBe(true);
|
||||
expect(resolveRequesterForChildSessionFromRuns(runs, childSessionKey)).toMatchObject({
|
||||
requesterSessionKey: "agent:main:new-parent",
|
||||
});
|
||||
expect(getSubagentRunByChildSessionKeyFromRuns(runs, childSessionKey)?.runId).toBe(
|
||||
"run-live-successor",
|
||||
);
|
||||
});
|
||||
|
||||
it("does not treat stale unended rows as active child-session liveness", () => {
|
||||
const now = Date.now();
|
||||
const childSessionKey = "agent:main:subagent:stale-live-check";
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
*/
|
||||
import type { DeliveryContext } from "../utils/delivery-context.types.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
import { compareSubagentRunGeneration } from "./subagent-run-generation.js";
|
||||
import { hasSubagentRunEnded, isLiveUnendedSubagentRun } from "./subagent-run-liveness.js";
|
||||
|
||||
function resolveControllerSessionKey(entry: SubagentRunRecord): string {
|
||||
|
|
@ -76,7 +77,7 @@ function rememberLatestRunEntry(
|
|||
entry: SubagentRunRecord,
|
||||
): void {
|
||||
const existing = map.get(key);
|
||||
if (!existing || entry.createdAt > existing.createdAt) {
|
||||
if (!existing || compareSubagentRunGeneration(entry, existing) > 0) {
|
||||
map.set(key, entry);
|
||||
}
|
||||
}
|
||||
|
|
@ -88,7 +89,7 @@ function rememberLatestRunPair(
|
|||
entry: SubagentRunRecord,
|
||||
): void {
|
||||
const existing = map.get(key);
|
||||
if (!existing || entry.createdAt > existing.entry.createdAt) {
|
||||
if (!existing || compareSubagentRunGeneration(entry, existing.entry) > 0) {
|
||||
map.set(key, { runId, entry });
|
||||
}
|
||||
}
|
||||
|
|
@ -126,12 +127,18 @@ export function buildSubagentRunReadIndexFromRuns(params: {
|
|||
inMemoryDisplayByChildSessionKey.set(childSessionKey, display);
|
||||
}
|
||||
if (hasSubagentRunEnded(entry)) {
|
||||
if (!display.latestInMemoryEnded || entry.createdAt > display.latestInMemoryEnded.createdAt) {
|
||||
if (
|
||||
!display.latestInMemoryEnded ||
|
||||
compareSubagentRunGeneration(entry, display.latestInMemoryEnded) > 0
|
||||
) {
|
||||
display.latestInMemoryEnded = entry;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!display.latestInMemoryActive || entry.createdAt > display.latestInMemoryActive.createdAt) {
|
||||
if (
|
||||
!display.latestInMemoryActive ||
|
||||
compareSubagentRunGeneration(entry, display.latestInMemoryActive) > 0
|
||||
) {
|
||||
display.latestInMemoryActive = entry;
|
||||
}
|
||||
}
|
||||
|
|
@ -181,7 +188,8 @@ export function buildSubagentRunReadIndexFromRuns(params: {
|
|||
if (latestInMemoryEnded || latestInMemoryActive) {
|
||||
if (
|
||||
latestInMemoryEnded &&
|
||||
(!latestInMemoryActive || latestInMemoryEnded.createdAt > latestInMemoryActive.createdAt)
|
||||
(!latestInMemoryActive ||
|
||||
compareSubagentRunGeneration(latestInMemoryEnded, latestInMemoryActive) > 0)
|
||||
) {
|
||||
return latestInMemoryEnded;
|
||||
}
|
||||
|
|
@ -258,7 +266,7 @@ function findLatestRunForChildSession(
|
|||
if (entry.childSessionKey !== key) {
|
||||
continue;
|
||||
}
|
||||
if (!latest || entry.createdAt > latest.createdAt) {
|
||||
if (!latest || compareSubagentRunGeneration(entry, latest) > 0) {
|
||||
latest = entry;
|
||||
}
|
||||
}
|
||||
|
|
@ -291,12 +299,12 @@ export function getSubagentRunByChildSessionKeyFromRuns(
|
|||
continue;
|
||||
}
|
||||
if (isLiveUnendedSubagentRun(entry)) {
|
||||
if (!latestActive || entry.createdAt > latestActive.createdAt) {
|
||||
if (!latestActive || compareSubagentRunGeneration(entry, latestActive) > 0) {
|
||||
latestActive = entry;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!latestEnded || entry.createdAt > latestEnded.createdAt) {
|
||||
if (!latestEnded || compareSubagentRunGeneration(entry, latestEnded) > 0) {
|
||||
latestEnded = entry;
|
||||
}
|
||||
}
|
||||
|
|
@ -363,7 +371,7 @@ export function countActiveRunsForSessionFromRuns(
|
|||
continue;
|
||||
}
|
||||
const existing = latestByChildSessionKey.get(entry.childSessionKey);
|
||||
if (!existing || entry.createdAt > existing.createdAt) {
|
||||
if (!existing || compareSubagentRunGeneration(entry, existing) > 0) {
|
||||
latestByChildSessionKey.set(entry.childSessionKey, entry);
|
||||
}
|
||||
}
|
||||
|
|
@ -403,7 +411,7 @@ function forEachDescendantRun(
|
|||
}
|
||||
const childKey = entry.childSessionKey.trim();
|
||||
const existing = latestByChildSessionKey.get(childKey);
|
||||
if (!existing || entry.createdAt > existing[1].createdAt) {
|
||||
if (!existing || compareSubagentRunGeneration(entry, existing[1]) > 0) {
|
||||
latestByChildSessionKey.set(childKey, [runId, entry]);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import {
|
|||
} from "./subagent-registry-queries.js";
|
||||
import { getSubagentRunsSnapshotForRead } from "./subagent-registry-state.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
import { compareSubagentRunGeneration } from "./subagent-run-generation.js";
|
||||
|
||||
export {
|
||||
getSubagentSessionRuntimeMs,
|
||||
|
|
@ -89,12 +90,12 @@ export function getSessionDisplaySubagentRunByChildSessionKey(
|
|||
continue;
|
||||
}
|
||||
if (typeof entry.endedAt === "number") {
|
||||
if (!latestInMemoryEnded || entry.createdAt > latestInMemoryEnded.createdAt) {
|
||||
if (!latestInMemoryEnded || compareSubagentRunGeneration(entry, latestInMemoryEnded) > 0) {
|
||||
latestInMemoryEnded = entry;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!latestInMemoryActive || entry.createdAt > latestInMemoryActive.createdAt) {
|
||||
if (!latestInMemoryActive || compareSubagentRunGeneration(entry, latestInMemoryActive) > 0) {
|
||||
latestInMemoryActive = entry;
|
||||
}
|
||||
}
|
||||
|
|
@ -103,7 +104,8 @@ export function getSessionDisplaySubagentRunByChildSessionKey(
|
|||
// Fresh in-memory terminal state is more accurate than an older active snapshot row.
|
||||
if (
|
||||
latestInMemoryEnded &&
|
||||
(!latestInMemoryActive || latestInMemoryEnded.createdAt > latestInMemoryActive.createdAt)
|
||||
(!latestInMemoryActive ||
|
||||
compareSubagentRunGeneration(latestInMemoryEnded, latestInMemoryActive) > 0)
|
||||
) {
|
||||
return latestInMemoryEnded;
|
||||
}
|
||||
|
|
@ -127,7 +129,7 @@ export function getLatestSubagentRunByChildSessionKey(
|
|||
if (entry.childSessionKey !== key) {
|
||||
continue;
|
||||
}
|
||||
if (!latest || entry.createdAt > latest.createdAt) {
|
||||
if (!latest || compareSubagentRunGeneration(entry, latest) > 0) {
|
||||
latest = entry;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,14 +7,16 @@ import { getRuntimeConfig } from "../config/config.js";
|
|||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { callGateway } from "../gateway/call.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { getGlobalHookRunner } from "../plugins/hook-runner-global.js";
|
||||
import { createRunningTaskRun } from "../tasks/detached-task-runtime.js";
|
||||
import {
|
||||
SUBAGENT_KILL_TASK_ERROR,
|
||||
type DetachedTaskFindResult,
|
||||
} from "../tasks/detached-task-runtime-contract.js";
|
||||
import { createRunningTaskRun, finalizeTaskRunByRunId } from "../tasks/detached-task-runtime.js";
|
||||
import { normalizeDeliveryContext } from "../utils/delivery-context.shared.js";
|
||||
import type { DeliveryContext } from "../utils/delivery-context.types.js";
|
||||
import { buildAgentRunTerminalOutcomeFromWaitResult } from "./agent-run-terminal-outcome.js";
|
||||
import { removeInternalSessionEffectsTranscript } from "./internal-session-effects.js";
|
||||
import { isRecoverableAgentWaitError, waitForAgentRun } from "./run-wait.js";
|
||||
import type { ensureRuntimePluginsLoaded as ensureRuntimePluginsLoadedFn } from "./runtime-plugins.js";
|
||||
import { type SubagentRunOutcome, withSubagentOutcomeTiming } from "./subagent-announce-output.js";
|
||||
import {
|
||||
clearDeliveryState,
|
||||
|
|
@ -22,15 +24,14 @@ import {
|
|||
normalizeSubagentRunState,
|
||||
} from "./subagent-delivery-state.js";
|
||||
import {
|
||||
SUBAGENT_ENDED_OUTCOME_KILLED,
|
||||
SUBAGENT_ENDED_REASON_COMPLETE,
|
||||
SUBAGENT_ENDED_REASON_ERROR,
|
||||
SUBAGENT_ENDED_REASON_KILLED,
|
||||
type SubagentLifecycleEndedReason,
|
||||
} from "./subagent-lifecycle-events.js";
|
||||
import {
|
||||
emitSubagentEndedHookOnce,
|
||||
shouldUpdateRunOutcome,
|
||||
resolveFinalizedSubagentTaskState,
|
||||
resolveKilledSubagentTaskEndedAt,
|
||||
} from "./subagent-registry-completion.js";
|
||||
import {
|
||||
getSubagentSessionRuntimeMs,
|
||||
|
|
@ -40,6 +41,10 @@ import {
|
|||
safeRemoveAttachmentsDir,
|
||||
} from "./subagent-registry-helpers.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
import {
|
||||
compareSubagentRunGeneration,
|
||||
nextSubagentRunGeneration,
|
||||
} from "./subagent-run-generation.js";
|
||||
import { resolveSubagentRunDeadlineMs } from "./subagent-run-timeout.js";
|
||||
import type { SubagentSessionCompletion } from "./subagent-session-reconciliation.js";
|
||||
|
||||
|
|
@ -100,6 +105,16 @@ export function markSubagentRunPausedAfterYield(params: {
|
|||
now?: number;
|
||||
}): boolean {
|
||||
const { entry } = params;
|
||||
if (
|
||||
entry.endedReason === SUBAGENT_ENDED_REASON_KILLED ||
|
||||
entry.suppressAnnounceReason === "killed" ||
|
||||
(entry.cleanup === "delete" && Number.isFinite(entry.deleteCleanupDispatchedAt))
|
||||
) {
|
||||
// agent.wait and lifecycle events can report the old yield after control
|
||||
// killed the run. Once delete dispatch starts, reviving the row would expose
|
||||
// a live run whose backing session may already be gone.
|
||||
return false;
|
||||
}
|
||||
let mutated = false;
|
||||
if (typeof params.startedAt === "number" && entry.startedAt !== params.startedAt) {
|
||||
entry.startedAt = params.startedAt;
|
||||
|
|
@ -129,6 +144,14 @@ export function markSubagentRunPausedAfterYield(params: {
|
|||
entry.cleanupHandled = false;
|
||||
mutated = true;
|
||||
}
|
||||
if (entry.cleanupCompletedAt !== undefined) {
|
||||
entry.cleanupCompletedAt = undefined;
|
||||
mutated = true;
|
||||
}
|
||||
if (entry.delivery !== undefined) {
|
||||
clearDeliveryState(entry);
|
||||
mutated = true;
|
||||
}
|
||||
const completion = ensureCompletionState(entry);
|
||||
if (completion.resultText !== undefined) {
|
||||
completion.resultText = undefined;
|
||||
|
|
@ -165,18 +188,10 @@ export type RegisterSubagentRunParams = {
|
|||
export function createSubagentRunManager(params: {
|
||||
runs: Map<string, SubagentRunRecord>;
|
||||
resumedRuns: Set<string>;
|
||||
endedHookInFlightRunIds: Set<string>;
|
||||
persist(): void;
|
||||
persistOrThrow(): void;
|
||||
callGateway: typeof callGateway;
|
||||
getRuntimeConfig: typeof getRuntimeConfig;
|
||||
ensureRuntimePluginsLoaded:
|
||||
| typeof ensureRuntimePluginsLoadedFn
|
||||
| ((args: {
|
||||
config: OpenClawConfig;
|
||||
workspaceDir?: string;
|
||||
allowGatewaySubagentBinding?: boolean;
|
||||
}) => void | Promise<void>);
|
||||
ensureListener(): void;
|
||||
startSweeper(): void;
|
||||
stopSweeper(): void;
|
||||
|
|
@ -205,6 +220,8 @@ export function createSubagentRunManager(params: {
|
|||
entry: SubagentRunRecord;
|
||||
cleanup: "delete" | "keep";
|
||||
completedAt: number;
|
||||
preserveTranscript?: boolean;
|
||||
provisionalKill?: boolean;
|
||||
}): void;
|
||||
completeSubagentRun(args: {
|
||||
runId: string;
|
||||
|
|
@ -216,7 +233,45 @@ export function createSubagentRunManager(params: {
|
|||
triggerCleanup: boolean;
|
||||
startedAt?: number;
|
||||
}): Promise<void>;
|
||||
resolveSubagentTask(entry: SubagentRunRecord): DetachedTaskFindResult;
|
||||
}) {
|
||||
const markOlderKillReconciliationsSuperseded = (next: SubagentRunRecord) => {
|
||||
const snapshots = new Map<SubagentRunRecord, SubagentRunRecord["killReconciliation"]>();
|
||||
for (const candidate of params.runs.values()) {
|
||||
if (
|
||||
candidate.runId === next.runId ||
|
||||
candidate.childSessionKey !== next.childSessionKey ||
|
||||
compareSubagentRunGeneration(candidate, next) >= 0 ||
|
||||
!candidate.killReconciliation
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
snapshots.set(candidate, structuredClone(candidate.killReconciliation));
|
||||
candidate.killReconciliation.supersededAt = Math.min(
|
||||
candidate.killReconciliation.supersededAt ?? next.createdAt,
|
||||
next.createdAt,
|
||||
);
|
||||
}
|
||||
return snapshots;
|
||||
};
|
||||
|
||||
const currentRunOwnsSession = (entry: SubagentRunRecord): boolean =>
|
||||
params.runs.get(entry.runId) === entry &&
|
||||
entry.killReconciliation?.supersededAt === undefined &&
|
||||
!Array.from(params.runs.values()).some(
|
||||
(candidate) =>
|
||||
candidate.childSessionKey === entry.childSessionKey &&
|
||||
compareSubagentRunGeneration(candidate, entry) > 0,
|
||||
);
|
||||
|
||||
const restoreKillReconciliationSnapshots = (
|
||||
snapshots: Map<SubagentRunRecord, SubagentRunRecord["killReconciliation"]>,
|
||||
) => {
|
||||
for (const [entry, snapshot] of snapshots) {
|
||||
entry.killReconciliation = snapshot;
|
||||
}
|
||||
};
|
||||
|
||||
const waitForSubagentCompletion = async (
|
||||
runId: string,
|
||||
waitTimeoutMs: number,
|
||||
|
|
@ -292,12 +347,6 @@ export function createSubagentRunManager(params: {
|
|||
notBeforeMs: entry.startedAt ?? entry.createdAt,
|
||||
});
|
||||
const completeAsRunTimeout = async (endedAt?: number, startedAt?: number) => {
|
||||
if (typeof startedAt === "number" && Number.isFinite(startedAt)) {
|
||||
entry.startedAt = startedAt;
|
||||
if (typeof entry.sessionStartedAt !== "number") {
|
||||
entry.sessionStartedAt = startedAt;
|
||||
}
|
||||
}
|
||||
const timeoutCompletion: Parameters<typeof params.completeSubagentRun>[0] = {
|
||||
runId,
|
||||
outcome: { status: "timeout" },
|
||||
|
|
@ -321,13 +370,6 @@ export function createSubagentRunManager(params: {
|
|||
typeof wait.stopReason === "string" ||
|
||||
typeof wait.livenessState === "string";
|
||||
const now = Date.now();
|
||||
if (observedStartedAt !== undefined && entry.startedAt !== observedStartedAt) {
|
||||
entry.startedAt = observedStartedAt;
|
||||
if (typeof entry.sessionStartedAt !== "number") {
|
||||
entry.sessionStartedAt = observedStartedAt;
|
||||
}
|
||||
params.persist();
|
||||
}
|
||||
// A plain agent.wait timeout has no terminal snapshot. For explicit
|
||||
// subagent run timeouts, the stored run deadline is the completion
|
||||
// contract so parent sessions are woken instead of retrying forever.
|
||||
|
|
@ -378,6 +420,13 @@ export function createSubagentRunManager(params: {
|
|||
await completeAsRunTimeout(timeoutEndedAt, observedStartedAt);
|
||||
return;
|
||||
}
|
||||
if (observedStartedAt !== undefined && entry.startedAt !== observedStartedAt) {
|
||||
entry.startedAt = observedStartedAt;
|
||||
if (typeof entry.sessionStartedAt !== "number") {
|
||||
entry.sessionStartedAt = observedStartedAt;
|
||||
}
|
||||
params.persist();
|
||||
}
|
||||
scheduleWaitRetry(
|
||||
entry,
|
||||
"subagent wait timed out; deferring terminal state until session reconciliation",
|
||||
|
|
@ -394,22 +443,7 @@ export function createSubagentRunManager(params: {
|
|||
await completeAsRunTimeout(completionAfterDeadline, observedStartedAt);
|
||||
return;
|
||||
}
|
||||
let mutated = false;
|
||||
if (typeof observedStartedAt === "number") {
|
||||
entry.startedAt = observedStartedAt;
|
||||
if (typeof entry.sessionStartedAt !== "number") {
|
||||
entry.sessionStartedAt = observedStartedAt;
|
||||
}
|
||||
mutated = true;
|
||||
}
|
||||
if (typeof wait.endedAt === "number") {
|
||||
entry.endedAt = wait.endedAt;
|
||||
mutated = true;
|
||||
}
|
||||
if (!entry.endedAt) {
|
||||
entry.endedAt = Date.now();
|
||||
mutated = true;
|
||||
}
|
||||
const endedAt = typeof wait.endedAt === "number" ? wait.endedAt : Date.now();
|
||||
const rawWaitError = typeof wait.error === "string" ? wait.error : undefined;
|
||||
const waitError = waitAborted
|
||||
? "subagent run terminated"
|
||||
|
|
@ -417,19 +451,12 @@ export function createSubagentRunManager(params: {
|
|||
const baseOutcome: SubagentRunOutcome =
|
||||
waitStatus === "error" ? { status: "error", error: waitError } : { status: "ok" };
|
||||
const outcome = withSubagentOutcomeTiming(baseOutcome, {
|
||||
startedAt: entry.startedAt,
|
||||
endedAt: entry.endedAt,
|
||||
startedAt: observedStartedAt ?? entry.startedAt,
|
||||
endedAt,
|
||||
});
|
||||
if (shouldUpdateRunOutcome(entry.outcome, outcome)) {
|
||||
entry.outcome = outcome;
|
||||
mutated = true;
|
||||
}
|
||||
if (mutated) {
|
||||
params.persist();
|
||||
}
|
||||
completionForRetry = {
|
||||
runId,
|
||||
endedAt: entry.endedAt,
|
||||
endedAt,
|
||||
outcome,
|
||||
reason: waitAborted
|
||||
? SUBAGENT_ENDED_REASON_KILLED
|
||||
|
|
@ -449,27 +476,31 @@ export function createSubagentRunManager(params: {
|
|||
childSessionKey: current?.childSessionKey ?? expectedEntry?.childSessionKey,
|
||||
error,
|
||||
});
|
||||
if (!current) {
|
||||
return;
|
||||
}
|
||||
if (completionForRetry) {
|
||||
try {
|
||||
await params.completeSubagentRun(completionForRetry);
|
||||
return;
|
||||
} catch (retryError) {
|
||||
log.warn("failed to complete subagent run after retry; retrying ended cleanup", {
|
||||
runId,
|
||||
childSessionKey: current.childSessionKey,
|
||||
error: retryError,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (
|
||||
current &&
|
||||
typeof current.endedAt === "number" &&
|
||||
!current.cleanupCompletedAt &&
|
||||
current.pauseReason !== "sessions_yield"
|
||||
) {
|
||||
if (completionForRetry) {
|
||||
try {
|
||||
await params.completeSubagentRun(completionForRetry);
|
||||
return;
|
||||
} catch (retryError) {
|
||||
log.warn("failed to complete subagent run after retry; retrying ended cleanup", {
|
||||
runId,
|
||||
childSessionKey: current.childSessionKey,
|
||||
error: retryError,
|
||||
});
|
||||
}
|
||||
}
|
||||
current.cleanupHandled = false;
|
||||
params.resumedRuns.delete(runId);
|
||||
params.resumeSubagentRun(runId);
|
||||
} else if (completionForRetry && typeof current.endedAt !== "number") {
|
||||
params.scheduleOrphanRecovery({ delayMs: 1_000 });
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
@ -503,6 +534,40 @@ export function createSubagentRunManager(params: {
|
|||
if (entry.suppressAnnounceReason !== "steer-restart") {
|
||||
return true;
|
||||
}
|
||||
if (typeof entry.endedAt === "number") {
|
||||
const taskResolution = params.resolveSubagentTask(entry);
|
||||
const task = taskResolution.lookup === "available" ? taskResolution.task : undefined;
|
||||
const terminal =
|
||||
entry.endedReason === SUBAGENT_ENDED_REASON_KILLED
|
||||
? {
|
||||
status: "cancelled" as const,
|
||||
endedAt: entry.endedAt,
|
||||
lastEventAt: entry.endedAt,
|
||||
error: "Subagent restart failed after the prior run was interrupted.",
|
||||
}
|
||||
: resolveFinalizedSubagentTaskState(entry);
|
||||
if (terminal) {
|
||||
const targetRunId = task?.runId ?? entry.taskRunId ?? entry.runId;
|
||||
const targetSessionKey = task?.childSessionKey ?? entry.childSessionKey;
|
||||
try {
|
||||
finalizeTaskRunByRunId({
|
||||
runId: targetRunId,
|
||||
runtime: "subagent",
|
||||
sessionKey: targetSessionKey,
|
||||
...terminal,
|
||||
suppressDelivery: true,
|
||||
});
|
||||
} catch (err) {
|
||||
// A task-runtime failure must not leave the interrupted run's
|
||||
// announcement and cleanup path permanently suppressed.
|
||||
log.warn("failed to finalize abandoned steer-restart task run", {
|
||||
err,
|
||||
runId: targetRunId,
|
||||
childSessionKey: targetSessionKey,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
entry.suppressAnnounceReason = undefined;
|
||||
params.persist();
|
||||
// If the interrupted run already finished while suppression was active, retry
|
||||
|
|
@ -535,22 +600,11 @@ export function createSubagentRunManager(params: {
|
|||
return false;
|
||||
}
|
||||
|
||||
if (previousRunId !== nextRunId) {
|
||||
params.clearPendingLifecycleError(previousRunId);
|
||||
if (shouldDeleteAttachments(source)) {
|
||||
void safeRemoveAttachmentsDir(source);
|
||||
}
|
||||
if (
|
||||
source.execution?.transcriptFile &&
|
||||
source.execution.transcriptFile !== replaceParams.transcriptFile
|
||||
) {
|
||||
void removeInternalSessionEffectsTranscript(source.execution.transcriptFile);
|
||||
}
|
||||
params.runs.delete(previousRunId);
|
||||
params.resumedRuns.delete(previousRunId);
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const generation = nextSubagentRunGeneration(
|
||||
[...params.runs.values(), source],
|
||||
source.childSessionKey,
|
||||
);
|
||||
const cfg = params.getRuntimeConfig();
|
||||
const archiveAfterMs = resolveArchiveAfterMs(cfg);
|
||||
const spawnMode = source.spawnMode === "session" ? "session" : "run";
|
||||
|
|
@ -588,7 +642,12 @@ export function createSubagentRunManager(params: {
|
|||
const next: SubagentRunRecord = normalizeSubagentRunState({
|
||||
...source,
|
||||
runId: nextRunId,
|
||||
// New rows carry an exact owner. Legacy replacement rows must retain an
|
||||
// unknown owner so their bounded session fallback can still find the
|
||||
// original detached task across another restart.
|
||||
taskRunId: source.taskRunId,
|
||||
task: nextTask,
|
||||
generation,
|
||||
createdAt: now,
|
||||
startedAt: now,
|
||||
sessionStartedAt,
|
||||
|
|
@ -598,6 +657,7 @@ export function createSubagentRunManager(params: {
|
|||
pauseReason: undefined,
|
||||
endedHookEmittedAt: undefined,
|
||||
browserCleanupDispatchedAt: undefined,
|
||||
deleteCleanupDispatchedAt: undefined,
|
||||
wakeOnDescendantSettle: undefined,
|
||||
outcome: undefined,
|
||||
execution: {
|
||||
|
|
@ -613,6 +673,8 @@ export function createSubagentRunManager(params: {
|
|||
cleanupCompletedAt: undefined,
|
||||
cleanupHandled: false,
|
||||
suppressAnnounceReason: undefined,
|
||||
killReconciliation: undefined,
|
||||
suppressCompletionDelivery: undefined,
|
||||
delivery: {
|
||||
status: source.expectsCompletionMessage === false ? "not_required" : "pending",
|
||||
},
|
||||
|
|
@ -622,9 +684,38 @@ export function createSubagentRunManager(params: {
|
|||
});
|
||||
clearDeliveryState(next);
|
||||
|
||||
if (previousRunId !== nextRunId) {
|
||||
params.runs.delete(previousRunId);
|
||||
}
|
||||
params.runs.set(nextRunId, next);
|
||||
markOlderKillReconciliationsSuperseded(next);
|
||||
try {
|
||||
params.persistOrThrow();
|
||||
} catch (error) {
|
||||
// The gateway has already started nextRunId. Keep its in-memory owner
|
||||
// authoritative and retry best-effort persistence; rolling back here
|
||||
// would orphan a live run that can still mutate the shared session.
|
||||
log.warn("failed to persist replacement subagent run; retaining live successor", {
|
||||
error,
|
||||
previousRunId,
|
||||
nextRunId,
|
||||
});
|
||||
params.persist();
|
||||
}
|
||||
if (previousRunId !== nextRunId) {
|
||||
params.clearPendingLifecycleError(previousRunId);
|
||||
params.resumedRuns.delete(previousRunId);
|
||||
if (shouldDeleteAttachments(source)) {
|
||||
void safeRemoveAttachmentsDir(source);
|
||||
}
|
||||
if (
|
||||
source.execution?.transcriptFile &&
|
||||
source.execution.transcriptFile !== replaceParams.transcriptFile
|
||||
) {
|
||||
void removeInternalSessionEffectsTranscript(source.execution.transcriptFile);
|
||||
}
|
||||
}
|
||||
params.ensureListener();
|
||||
params.persist();
|
||||
// Always start sweeper — session-mode runs (no archiveAtMs) also need TTL cleanup.
|
||||
params.startSweeper();
|
||||
void waitForSubagentCompletion(nextRunId, waitTimeoutMs, next);
|
||||
|
|
@ -640,6 +731,7 @@ export function createSubagentRunManager(params: {
|
|||
return;
|
||||
}
|
||||
const now = Date.now();
|
||||
const generation = nextSubagentRunGeneration(params.runs.values(), childSessionKey);
|
||||
const cfg = params.getRuntimeConfig();
|
||||
const archiveAfterMs = resolveArchiveAfterMs(cfg);
|
||||
const spawnMode = registerParams.spawnMode === "session" ? "session" : "run";
|
||||
|
|
@ -654,6 +746,7 @@ export function createSubagentRunManager(params: {
|
|||
const requesterOrigin = normalizeDeliveryContext(registerParams.requesterOrigin);
|
||||
const entry: SubagentRunRecord = normalizeSubagentRunState({
|
||||
runId,
|
||||
taskRunId: runId,
|
||||
childSessionKey,
|
||||
controllerSessionKey,
|
||||
requesterSessionKey,
|
||||
|
|
@ -669,6 +762,7 @@ export function createSubagentRunManager(params: {
|
|||
agentDir: registerParams.agentDir,
|
||||
workspaceDir: registerParams.workspaceDir,
|
||||
runTimeoutSeconds,
|
||||
generation,
|
||||
createdAt: now,
|
||||
startedAt: now,
|
||||
execution: {
|
||||
|
|
@ -691,10 +785,12 @@ export function createSubagentRunManager(params: {
|
|||
retainAttachmentsOnKeep: registerParams.retainAttachmentsOnKeep,
|
||||
});
|
||||
params.runs.set(runId, entry);
|
||||
const killReconciliationSnapshots = markOlderKillReconciliationsSuperseded(entry);
|
||||
try {
|
||||
params.persistOrThrow();
|
||||
} catch (error) {
|
||||
params.runs.delete(runId);
|
||||
restoreKillReconciliationSnapshots(killReconciliationSnapshots);
|
||||
throw error;
|
||||
}
|
||||
try {
|
||||
|
|
@ -762,6 +858,7 @@ export function createSubagentRunManager(params: {
|
|||
runId?: string;
|
||||
childSessionKey?: string;
|
||||
reason?: string;
|
||||
suppressTaskDelivery?: boolean;
|
||||
}): number => {
|
||||
const runIds = new Set<string>();
|
||||
if (typeof markParams.runId === "string" && markParams.runId.trim()) {
|
||||
|
|
@ -782,47 +879,120 @@ export function createSubagentRunManager(params: {
|
|||
const reason = markParams.reason?.trim() || "killed";
|
||||
let updated = 0;
|
||||
const entriesByChildSessionKey = new Map<string, SubagentRunRecord>();
|
||||
const entrySnapshots = new Map<SubagentRunRecord, SubagentRunRecord>();
|
||||
const pendingTaskFinalizations: Array<{ entry: SubagentRunRecord; endedAt: number }> = [];
|
||||
const finalizeKilledTask = (entry: SubagentRunRecord, endedAt: number) => {
|
||||
const taskResolution = params.resolveSubagentTask(entry);
|
||||
const task = taskResolution.lookup === "available" ? taskResolution.task : undefined;
|
||||
const targetRunId = task?.runId ?? entry.taskRunId ?? entry.runId;
|
||||
const targetSessionKey = task?.childSessionKey ?? entry.childSessionKey;
|
||||
try {
|
||||
finalizeTaskRunByRunId({
|
||||
runId: targetRunId,
|
||||
runtime: "subagent",
|
||||
sessionKey: targetSessionKey,
|
||||
status: "cancelled",
|
||||
endedAt,
|
||||
lastEventAt: endedAt,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
suppressDelivery: entry.killReconciliation?.suppressTaskDelivery === true,
|
||||
});
|
||||
} catch (err) {
|
||||
log.warn("failed to finalize killed subagent task run", {
|
||||
err,
|
||||
runId: targetRunId,
|
||||
childSessionKey: targetSessionKey,
|
||||
});
|
||||
}
|
||||
};
|
||||
for (const runId of runIds) {
|
||||
params.clearPendingLifecycleError(runId);
|
||||
params.clearPendingLifecycleTimeout(runId);
|
||||
const entry = params.runs.get(runId);
|
||||
if (!entry) {
|
||||
continue;
|
||||
}
|
||||
if (typeof entry.endedAt === "number") {
|
||||
const wasKilledLifecycle =
|
||||
entry.endedReason === SUBAGENT_ENDED_REASON_KILLED &&
|
||||
entry.killReconciliation !== undefined;
|
||||
const existingKillReconciliation = entry.killReconciliation;
|
||||
if (
|
||||
typeof entry.endedAt === "number" &&
|
||||
entry.pauseReason !== "sessions_yield" &&
|
||||
!wasKilledLifecycle
|
||||
) {
|
||||
// An abort lifecycle event can mark the run killed before this shared
|
||||
// termination path runs. Re-enter only for that provisional state so
|
||||
// it receives the same reconciliation tombstone as a direct kill.
|
||||
continue;
|
||||
}
|
||||
entry.endedAt = now;
|
||||
entrySnapshots.set(entry, structuredClone(entry));
|
||||
const wasYielded = entry.pauseReason === "sessions_yield";
|
||||
const endedAt =
|
||||
(wasYielded || wasKilledLifecycle) && typeof entry.endedAt === "number"
|
||||
? entry.endedAt
|
||||
: now;
|
||||
entry.endedAt = endedAt;
|
||||
entry.outcome = withSubagentOutcomeTiming(
|
||||
{ status: "error", error: reason },
|
||||
{
|
||||
startedAt: entry.startedAt,
|
||||
endedAt: now,
|
||||
endedAt,
|
||||
},
|
||||
);
|
||||
entry.endedReason = SUBAGENT_ENDED_REASON_KILLED;
|
||||
entry.cleanupHandled = true;
|
||||
entry.cleanupCompletedAt = now;
|
||||
entry.cleanupCompletedAt = existingKillReconciliation
|
||||
? (entry.cleanupCompletedAt ?? endedAt)
|
||||
: wasKilledLifecycle
|
||||
? endedAt
|
||||
: now;
|
||||
entry.suppressAnnounceReason = "killed";
|
||||
entry.pauseReason = undefined;
|
||||
// Setting endedAt above short-circuits the completion watcher, so the
|
||||
// lifecycle finalizer never reaches the detached task row for killed runs.
|
||||
const taskEndedAt = existingKillReconciliation
|
||||
? (resolveKilledSubagentTaskEndedAt(entry) ?? endedAt)
|
||||
: wasYielded
|
||||
? now
|
||||
: endedAt;
|
||||
entry.killReconciliation = {
|
||||
killedAt: existingKillReconciliation?.killedAt ?? taskEndedAt,
|
||||
suppressTaskDelivery:
|
||||
existingKillReconciliation?.suppressTaskDelivery === true ||
|
||||
markParams.suppressTaskDelivery === true
|
||||
? true
|
||||
: undefined,
|
||||
supersededAt: existingKillReconciliation?.supersededAt,
|
||||
};
|
||||
pendingTaskFinalizations.push({ entry, endedAt: taskEndedAt });
|
||||
if (!entriesByChildSessionKey.has(entry.childSessionKey)) {
|
||||
entriesByChildSessionKey.set(entry.childSessionKey, entry);
|
||||
}
|
||||
updated += 1;
|
||||
}
|
||||
if (updated > 0) {
|
||||
params.persist();
|
||||
try {
|
||||
// The registry tombstone is the recovery source for the provisional
|
||||
// task marker. It must commit first so the sweeper can always finish it.
|
||||
params.persistOrThrow();
|
||||
} catch (error) {
|
||||
for (const [entry, snapshot] of entrySnapshots) {
|
||||
const target = entry as unknown as Record<string, unknown>;
|
||||
for (const key of Object.keys(target)) {
|
||||
delete target[key];
|
||||
}
|
||||
Object.assign(target, snapshot);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
for (const pending of pendingTaskFinalizations) {
|
||||
finalizeKilledTask(pending.entry, pending.endedAt);
|
||||
}
|
||||
for (const entry of entriesByChildSessionKey.values()) {
|
||||
const emitEndedHook = () =>
|
||||
emitSubagentEndedHookOnce({
|
||||
entry,
|
||||
reason: SUBAGENT_ENDED_REASON_KILLED,
|
||||
sendFarewell: true,
|
||||
accountId: entry.requesterOrigin?.accountId,
|
||||
outcome: SUBAGENT_ENDED_OUTCOME_KILLED,
|
||||
error: reason,
|
||||
inFlightRunIds: params.endedHookInFlightRunIds,
|
||||
persist: () => params.persist(),
|
||||
});
|
||||
void persistSubagentSessionTiming(entry).catch((err: unknown) => {
|
||||
void persistSubagentSessionTiming(entry, {
|
||||
isCurrentGeneration: () => currentRunOwnsSession(entry),
|
||||
}).catch((err: unknown) => {
|
||||
log.warn("failed to persist killed subagent session timing", {
|
||||
err,
|
||||
runId: entry.runId,
|
||||
|
|
@ -835,27 +1005,13 @@ export function createSubagentRunManager(params: {
|
|||
params.completeCleanupBookkeeping({
|
||||
runId: entry.runId,
|
||||
entry,
|
||||
cleanup: entry.cleanup,
|
||||
// A direct kill is provisional until the runner reports its final
|
||||
// outcome. Keep delete-mode rows as reconciliation tombstones.
|
||||
cleanup: "keep",
|
||||
completedAt: now,
|
||||
preserveTranscript: true,
|
||||
provisionalKill: true,
|
||||
});
|
||||
if (getGlobalHookRunner()) {
|
||||
void emitEndedHook().catch(() => {
|
||||
// Hook failures should not break termination flow.
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const cfg = params.getRuntimeConfig();
|
||||
void Promise.resolve(
|
||||
params.ensureRuntimePluginsLoaded({
|
||||
config: cfg,
|
||||
workspaceDir: entry.workspaceDir,
|
||||
allowGatewaySubagentBinding: true,
|
||||
}),
|
||||
)
|
||||
.then(emitEndedHook)
|
||||
.catch(() => {
|
||||
// Hook failures should not break termination flow.
|
||||
});
|
||||
}
|
||||
}
|
||||
return updated;
|
||||
|
|
|
|||
|
|
@ -5,6 +5,20 @@ import os from "node:os";
|
|||
import path from "node:path";
|
||||
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { callGateway } from "../gateway/call.js";
|
||||
import { SUBAGENT_KILL_TASK_ERROR } from "../tasks/detached-task-runtime-contract.js";
|
||||
import {
|
||||
getDetachedTaskLifecycleRuntime,
|
||||
resetDetachedTaskLifecycleRuntimeForTests,
|
||||
setDetachedTaskLifecycleRuntime,
|
||||
} from "../tasks/detached-task-runtime.js";
|
||||
|
||||
const taskRuntimeMocks = vi.hoisted(() => ({
|
||||
finalizeTaskRunByRunId: vi.fn<(_params: unknown) => unknown[]>(() => [{}]),
|
||||
}));
|
||||
const taskStatusMocks = vi.hoisted(() => ({
|
||||
findTaskByRunIdForStatus: vi.fn(),
|
||||
listTasksForSessionKeyForStatus: vi.fn(() => [] as never[]),
|
||||
}));
|
||||
|
||||
const noop = () => {};
|
||||
let currentConfig = {
|
||||
|
|
@ -29,6 +43,19 @@ vi.mock("../gateway/call.js", () => ({
|
|||
}),
|
||||
}));
|
||||
|
||||
vi.mock("../tasks/detached-task-runtime.js", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("../tasks/detached-task-runtime.js")>();
|
||||
return {
|
||||
...actual,
|
||||
finalizeTaskRunByRunId: taskRuntimeMocks.finalizeTaskRunByRunId,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../tasks/task-status-access.js", () => ({
|
||||
findTaskByRunIdForStatus: taskStatusMocks.findTaskByRunIdForStatus,
|
||||
listTasksForSessionKeyForStatus: taskStatusMocks.listTasksForSessionKeyForStatus,
|
||||
}));
|
||||
|
||||
vi.mock("../infra/agent-events.js", () => ({
|
||||
getAgentRunContext: vi.fn(() => undefined),
|
||||
onAgentEvent: vi.fn((_handler: unknown) => noop),
|
||||
|
|
@ -68,11 +95,19 @@ describe("subagent registry archive behavior", () => {
|
|||
mod.testing.setDepsForTest({
|
||||
callGateway,
|
||||
getRuntimeConfig: loadConfigMock as typeof import("../config/config.js").getRuntimeConfig,
|
||||
ensureRuntimePluginsLoaded: vi.fn(),
|
||||
...overrides,
|
||||
});
|
||||
};
|
||||
|
||||
const waitForNoRequesterRuns = async () => {
|
||||
await vi.waitFor(() => {
|
||||
expect(mod.listSubagentRunsForRequester("agent:main:main")).toHaveLength(0);
|
||||
});
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
resetDetachedTaskLifecycleRuntimeForTests();
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(new Date("2026-01-01T00:00:00Z"));
|
||||
currentConfig = {
|
||||
|
|
@ -88,11 +123,32 @@ describe("subagent registry archive behavior", () => {
|
|||
return {};
|
||||
});
|
||||
loadConfigMock.mockClear();
|
||||
taskRuntimeMocks.finalizeTaskRunByRunId.mockClear();
|
||||
taskStatusMocks.findTaskByRunIdForStatus.mockReset();
|
||||
taskStatusMocks.listTasksForSessionKeyForStatus.mockReset();
|
||||
taskStatusMocks.listTasksForSessionKeyForStatus.mockReturnValue([]);
|
||||
taskStatusMocks.findTaskByRunIdForStatus.mockImplementation((runId: string) => {
|
||||
const entry = mod
|
||||
.listSubagentRunsForRequester("agent:main:main")
|
||||
.find((candidate) => candidate.runId === runId);
|
||||
return entry
|
||||
? ({
|
||||
taskId: `task-${runId}`,
|
||||
runId,
|
||||
runtime: "subagent",
|
||||
childSessionKey: entry.childSessionKey,
|
||||
createdAt: entry.createdAt,
|
||||
status: "cancelled",
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
} as never)
|
||||
: undefined;
|
||||
});
|
||||
setRegistryTestDeps();
|
||||
mod.resetSubagentRegistryForTests({ persist: false });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
resetDetachedTaskLifecycleRuntimeForTests();
|
||||
mod.testing.setDepsForTest();
|
||||
mod.resetSubagentRegistryForTests({ persist: false });
|
||||
vi.useRealTimers();
|
||||
|
|
@ -133,7 +189,7 @@ describe("subagent registry archive behavior", () => {
|
|||
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
|
||||
expect(mod.listSubagentRunsForRequester("agent:main:main")).toHaveLength(0);
|
||||
await waitForNoRequesterRuns();
|
||||
});
|
||||
|
||||
it("keeps archived delete-mode runs for retry when sessions.delete fails", async () => {
|
||||
|
|
@ -194,6 +250,438 @@ describe("subagent registry archive behavior", () => {
|
|||
expect(mod.listSubagentRunsForRequester("agent:main:main")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("stabilizes provisional killed tasks before deleting expired tombstones", async () => {
|
||||
const now = Date.now();
|
||||
mod.addSubagentRunForTests({
|
||||
runId: "run-killed-tombstone-expired",
|
||||
childSessionKey: "agent:main:subagent:killed-tombstone-expired",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "expire killed tombstone",
|
||||
cleanup: "delete",
|
||||
createdAt: now - 10 * 60_000,
|
||||
startedAt: now - 10 * 60_000,
|
||||
endedAt: now - 5 * 60_000,
|
||||
endedReason: "subagent-killed",
|
||||
outcome: { status: "error", error: "manual kill" },
|
||||
suppressAnnounceReason: "killed",
|
||||
killReconciliation: { killedAt: now - 5 * 60_000 },
|
||||
cleanupHandled: true,
|
||||
cleanupCompletedAt: now - 5 * 60_000,
|
||||
archiveAtMs: now,
|
||||
});
|
||||
|
||||
await mod.testing.sweepOnceForTests();
|
||||
await flushSweepMicrotasks();
|
||||
|
||||
expect(taskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalledWith({
|
||||
runId: "run-killed-tombstone-expired",
|
||||
runtime: "subagent",
|
||||
sessionKey: "agent:main:subagent:killed-tombstone-expired",
|
||||
status: "cancelled",
|
||||
endedAt: now - 5 * 60_000,
|
||||
lastEventAt: now - 5 * 60_000,
|
||||
error: "manual kill",
|
||||
suppressDelivery: true,
|
||||
});
|
||||
await waitForNoRequesterRuns();
|
||||
});
|
||||
|
||||
it("retains expired tombstones when provisional task finalization is rejected", async () => {
|
||||
const now = Date.now();
|
||||
taskRuntimeMocks.finalizeTaskRunByRunId.mockReturnValueOnce([]);
|
||||
mod.addSubagentRunForTests({
|
||||
runId: "run-killed-tombstone-retry",
|
||||
childSessionKey: "agent:main:subagent:killed-tombstone-retry",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "retry killed tombstone",
|
||||
cleanup: "delete",
|
||||
createdAt: now - 10 * 60_000,
|
||||
endedAt: now - 5 * 60_000,
|
||||
endedReason: "subagent-killed",
|
||||
outcome: { status: "error", error: "manual kill" },
|
||||
suppressAnnounceReason: "killed",
|
||||
killReconciliation: { killedAt: now - 5 * 60_000 },
|
||||
cleanupHandled: true,
|
||||
cleanupCompletedAt: now - 5 * 60_000,
|
||||
archiveAtMs: now,
|
||||
});
|
||||
|
||||
await mod.testing.sweepOnceForTests();
|
||||
|
||||
expect(mod.listSubagentRunsForRequester("agent:main:main")).toHaveLength(1);
|
||||
expect(
|
||||
vi
|
||||
.mocked(callGateway)
|
||||
.mock.calls.some(
|
||||
([request]) => (request as { method?: string } | undefined)?.method === "sessions.delete",
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("retires expired tombstones when their task row is already gone", async () => {
|
||||
const now = Date.now();
|
||||
taskStatusMocks.findTaskByRunIdForStatus.mockReturnValue(undefined);
|
||||
mod.addSubagentRunForTests({
|
||||
runId: "run-killed-task-missing",
|
||||
childSessionKey: "agent:main:subagent:killed-task-missing",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "retire missing task tombstone",
|
||||
cleanup: "keep",
|
||||
createdAt: now - 10 * 60_000,
|
||||
endedAt: now - 5 * 60_000,
|
||||
endedReason: "subagent-killed",
|
||||
outcome: { status: "error", error: "manual kill" },
|
||||
suppressAnnounceReason: "killed",
|
||||
killReconciliation: { killedAt: now - 5 * 60_000 },
|
||||
cleanupHandled: true,
|
||||
cleanupCompletedAt: now - 5 * 60_000,
|
||||
archiveAtMs: now,
|
||||
});
|
||||
|
||||
await mod.testing.sweepOnceForTests();
|
||||
await flushSweepMicrotasks();
|
||||
|
||||
expect(taskRuntimeMocks.finalizeTaskRunByRunId).not.toHaveBeenCalled();
|
||||
await waitForNoRequesterRuns();
|
||||
});
|
||||
|
||||
it("preserves stable operator cancellation when retiring expired tombstones", async () => {
|
||||
const now = Date.now();
|
||||
taskStatusMocks.findTaskByRunIdForStatus.mockReturnValue({
|
||||
taskId: "task-killed-operator-cancelled",
|
||||
runId: "run-killed-operator-cancelled",
|
||||
runtime: "subagent",
|
||||
childSessionKey: "agent:main:subagent:killed-operator-cancelled",
|
||||
createdAt: now - 10 * 60_000,
|
||||
status: "cancelled",
|
||||
error: "Cancelled by operator.",
|
||||
} as never);
|
||||
mod.addSubagentRunForTests({
|
||||
runId: "run-killed-operator-cancelled",
|
||||
childSessionKey: "agent:main:subagent:killed-operator-cancelled",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "preserve operator cancellation",
|
||||
cleanup: "keep",
|
||||
createdAt: now - 10 * 60_000,
|
||||
endedAt: now - 5 * 60_000,
|
||||
endedReason: "subagent-killed",
|
||||
outcome: { status: "error", error: "manual kill" },
|
||||
suppressAnnounceReason: "killed",
|
||||
killReconciliation: { killedAt: now - 5 * 60_000 },
|
||||
cleanupHandled: true,
|
||||
cleanupCompletedAt: now - 5 * 60_000,
|
||||
archiveAtMs: now,
|
||||
});
|
||||
|
||||
await mod.testing.sweepOnceForTests();
|
||||
await flushSweepMicrotasks();
|
||||
|
||||
expect(taskRuntimeMocks.finalizeTaskRunByRunId).not.toHaveBeenCalled();
|
||||
await waitForNoRequesterRuns();
|
||||
});
|
||||
|
||||
it("keeps stable cancellation tombstones through the completion grace window", async () => {
|
||||
const now = Date.now();
|
||||
taskStatusMocks.findTaskByRunIdForStatus.mockReturnValue({
|
||||
taskId: "task-killed-grace",
|
||||
runId: "run-killed-grace",
|
||||
runtime: "subagent",
|
||||
childSessionKey: "agent:main:subagent:killed-grace",
|
||||
createdAt: now - 2 * 60_000,
|
||||
status: "cancelled",
|
||||
error: "Cancelled by operator.",
|
||||
} as never);
|
||||
mod.addSubagentRunForTests({
|
||||
runId: "run-killed-grace",
|
||||
childSessionKey: "agent:main:subagent:killed-grace",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "retain cancellation evidence",
|
||||
cleanup: "keep",
|
||||
createdAt: now - 2 * 60_000,
|
||||
endedAt: now - 60_000,
|
||||
endedReason: "subagent-killed",
|
||||
outcome: { status: "error", error: "manual kill" },
|
||||
suppressAnnounceReason: "killed",
|
||||
killReconciliation: { killedAt: now - 60_000 },
|
||||
cleanupHandled: true,
|
||||
cleanupCompletedAt: now - 60_000,
|
||||
archiveAtMs: now,
|
||||
});
|
||||
|
||||
await mod.testing.sweepOnceForTests();
|
||||
|
||||
expect(mod.listSubagentRunsForRequester("agent:main:main")).toHaveLength(1);
|
||||
expect(taskRuntimeMocks.finalizeTaskRunByRunId).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retires expired tombstones after an opaque legacy runtime finalizer misses", async () => {
|
||||
const legacyRuntime = { ...getDetachedTaskLifecycleRuntime() };
|
||||
delete legacyRuntime.findTaskRun;
|
||||
setDetachedTaskLifecycleRuntime(legacyRuntime);
|
||||
taskStatusMocks.findTaskByRunIdForStatus.mockReturnValue(undefined);
|
||||
taskStatusMocks.listTasksForSessionKeyForStatus.mockReturnValue([]);
|
||||
taskRuntimeMocks.finalizeTaskRunByRunId.mockReturnValueOnce([]);
|
||||
const now = Date.now();
|
||||
mod.addSubagentRunForTests({
|
||||
runId: "run-killed-opaque-runtime",
|
||||
childSessionKey: "agent:main:subagent:killed-opaque-runtime",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "bound opaque runtime reconciliation",
|
||||
cleanup: "keep",
|
||||
createdAt: now - 10 * 60_000,
|
||||
endedAt: now - 5 * 60_000,
|
||||
endedReason: "subagent-killed",
|
||||
outcome: { status: "error", error: "manual kill" },
|
||||
suppressAnnounceReason: "killed",
|
||||
killReconciliation: { killedAt: now - 5 * 60_000 },
|
||||
cleanupHandled: true,
|
||||
cleanupCompletedAt: now - 5 * 60_000,
|
||||
archiveAtMs: now,
|
||||
});
|
||||
|
||||
await mod.testing.sweepOnceForTests();
|
||||
await flushSweepMicrotasks();
|
||||
|
||||
expect(taskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalled();
|
||||
await waitForNoRequesterRuns();
|
||||
});
|
||||
|
||||
it("stabilizes replacement runs through their durable task session scope", async () => {
|
||||
const now = Date.now();
|
||||
taskStatusMocks.findTaskByRunIdForStatus.mockReturnValue(undefined);
|
||||
taskStatusMocks.listTasksForSessionKeyForStatus.mockReturnValue([
|
||||
{
|
||||
taskId: "task-before-replacement",
|
||||
runId: "run-before-replacement",
|
||||
runtime: "subagent",
|
||||
childSessionKey: "agent:main:subagent:replacement",
|
||||
status: "running",
|
||||
createdAt: now - 11 * 60_000,
|
||||
},
|
||||
] as never);
|
||||
mod.addSubagentRunForTests({
|
||||
runId: "run-after-replacement",
|
||||
childSessionKey: "agent:main:subagent:replacement",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "stabilize replacement task",
|
||||
cleanup: "keep",
|
||||
createdAt: now - 10 * 60_000,
|
||||
sessionStartedAt: now - 11 * 60_000,
|
||||
endedAt: now - 5 * 60_000,
|
||||
endedReason: "subagent-killed",
|
||||
outcome: { status: "error", error: "manual kill" },
|
||||
suppressAnnounceReason: "killed",
|
||||
killReconciliation: { killedAt: now - 5 * 60_000 },
|
||||
cleanupHandled: true,
|
||||
cleanupCompletedAt: now - 5 * 60_000,
|
||||
archiveAtMs: now,
|
||||
});
|
||||
|
||||
await mod.testing.sweepOnceForTests();
|
||||
await flushSweepMicrotasks();
|
||||
|
||||
expect(taskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
runId: "run-before-replacement",
|
||||
sessionKey: "agent:main:subagent:replacement",
|
||||
}),
|
||||
);
|
||||
await waitForNoRequesterRuns();
|
||||
});
|
||||
|
||||
it("directly kills a replacement run through its durable task ID", () => {
|
||||
const now = Date.now();
|
||||
const childSessionKey = "agent:main:subagent:replacement-direct-kill";
|
||||
taskStatusMocks.findTaskByRunIdForStatus.mockReturnValue(undefined);
|
||||
taskStatusMocks.listTasksForSessionKeyForStatus.mockReturnValue([
|
||||
{
|
||||
taskId: "task-before-replacement-direct-kill",
|
||||
runId: "run-before-replacement-direct-kill",
|
||||
runtime: "subagent",
|
||||
childSessionKey,
|
||||
status: "running",
|
||||
createdAt: now - 11 * 60_000,
|
||||
},
|
||||
] as never);
|
||||
mod.addSubagentRunForTests({
|
||||
runId: "run-after-replacement-direct-kill",
|
||||
childSessionKey,
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "kill replacement task",
|
||||
cleanup: "keep",
|
||||
createdAt: now - 10 * 60_000,
|
||||
sessionStartedAt: now - 11 * 60_000,
|
||||
});
|
||||
|
||||
expect(
|
||||
mod.markSubagentRunTerminated({
|
||||
runId: "run-after-replacement-direct-kill",
|
||||
reason: "manual kill",
|
||||
}),
|
||||
).toBe(1);
|
||||
|
||||
expect(taskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
runId: "run-before-replacement-direct-kill",
|
||||
sessionKey: childSessionKey,
|
||||
status: "cancelled",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not reconcile an older tombstone through a newer session task", async () => {
|
||||
const now = Date.now();
|
||||
taskStatusMocks.findTaskByRunIdForStatus.mockReturnValue(undefined);
|
||||
taskStatusMocks.listTasksForSessionKeyForStatus.mockReturnValue([
|
||||
{
|
||||
taskId: "task-new-generation",
|
||||
runId: "run-new-generation",
|
||||
runtime: "subagent",
|
||||
childSessionKey: "agent:main:subagent:reused-session",
|
||||
status: "running",
|
||||
createdAt: now - 60_000,
|
||||
},
|
||||
] as never);
|
||||
mod.addSubagentRunForTests({
|
||||
runId: "run-old-generation",
|
||||
childSessionKey: "agent:main:subagent:reused-session",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "expire old generation",
|
||||
cleanup: "keep",
|
||||
createdAt: now - 10 * 60_000,
|
||||
sessionStartedAt: now - 10 * 60_000,
|
||||
endedAt: now - 5 * 60_000,
|
||||
endedReason: "subagent-killed",
|
||||
outcome: { status: "error", error: "manual kill" },
|
||||
suppressAnnounceReason: "killed",
|
||||
killReconciliation: {
|
||||
killedAt: now - 5 * 60_000,
|
||||
supersededAt: now - 60_000,
|
||||
},
|
||||
cleanupHandled: true,
|
||||
cleanupCompletedAt: now - 5 * 60_000,
|
||||
});
|
||||
|
||||
await mod.testing.sweepOnceForTests();
|
||||
await flushSweepMicrotasks();
|
||||
|
||||
expect(taskRuntimeMocks.finalizeTaskRunByRunId).not.toHaveBeenCalled();
|
||||
await waitForNoRequesterRuns();
|
||||
});
|
||||
|
||||
it("retires expired keep-mode reconciliation rows without deleting their sessions", async () => {
|
||||
const now = Date.now();
|
||||
mod.addSubagentRunForTests({
|
||||
runId: "run-killed-keep-expired",
|
||||
childSessionKey: "agent:main:subagent:killed-keep-expired",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "stabilize retained session kill",
|
||||
cleanup: "keep",
|
||||
createdAt: now - 10 * 60_000,
|
||||
endedAt: now - 5 * 60_000,
|
||||
endedReason: "subagent-killed",
|
||||
outcome: { status: "error", error: "manual kill" },
|
||||
suppressAnnounceReason: "killed",
|
||||
killReconciliation: { killedAt: now - 5 * 60_000 },
|
||||
cleanupHandled: true,
|
||||
cleanupCompletedAt: now - 5 * 60_000,
|
||||
archiveAtMs: now,
|
||||
});
|
||||
|
||||
await mod.testing.sweepOnceForTests();
|
||||
await flushSweepMicrotasks();
|
||||
|
||||
expect(taskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalled();
|
||||
await waitForNoRequesterRuns();
|
||||
expect(
|
||||
vi
|
||||
.mocked(callGateway)
|
||||
.mock.calls.some(
|
||||
([request]) => (request as { method?: string } | undefined)?.method === "sessions.delete",
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("stabilizes killed tasks before their configured session archive deadline", async () => {
|
||||
const now = Date.now();
|
||||
const archiveAtMs = now + 55 * 60_000;
|
||||
mod.addSubagentRunForTests({
|
||||
runId: "run-killed-retained-session",
|
||||
childSessionKey: "agent:main:subagent:killed-retained-session",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "stabilize before archive",
|
||||
cleanup: "delete",
|
||||
createdAt: now - 10 * 60_000,
|
||||
endedAt: now - 5 * 60_000,
|
||||
endedReason: "subagent-killed",
|
||||
outcome: { status: "error", error: "manual kill" },
|
||||
suppressAnnounceReason: "killed",
|
||||
killReconciliation: { killedAt: now - 5 * 60_000 },
|
||||
cleanupHandled: true,
|
||||
cleanupCompletedAt: now - 5 * 60_000,
|
||||
archiveAtMs,
|
||||
});
|
||||
|
||||
await mod.testing.sweepOnceForTests();
|
||||
|
||||
expect(taskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalled();
|
||||
expect(mod.listSubagentRunsForRequester("agent:main:main")).toEqual([
|
||||
expect.objectContaining({
|
||||
runId: "run-killed-retained-session",
|
||||
archiveAtMs,
|
||||
suppressAnnounceReason: undefined,
|
||||
}),
|
||||
]);
|
||||
expect(
|
||||
vi
|
||||
.mocked(callGateway)
|
||||
.mock.calls.some(
|
||||
([request]) => (request as { method?: string } | undefined)?.method === "sessions.delete",
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("continues killed cleanup when ended hook loading fails", async () => {
|
||||
const now = Date.now();
|
||||
setRegistryTestDeps({
|
||||
ensureRuntimePluginsLoaded: vi.fn(() => {
|
||||
throw new Error("plugin load failed");
|
||||
}),
|
||||
});
|
||||
mod.addSubagentRunForTests({
|
||||
runId: "run-killed-hook-load-failure",
|
||||
childSessionKey: "agent:main:subagent:killed-hook-load-failure",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "cleanup despite hook failure",
|
||||
cleanup: "keep",
|
||||
createdAt: now - 10 * 60_000,
|
||||
endedAt: now - 5 * 60_000,
|
||||
endedReason: "subagent-killed",
|
||||
outcome: { status: "error", error: "manual kill" },
|
||||
suppressAnnounceReason: "killed",
|
||||
killReconciliation: { killedAt: now - 5 * 60_000 },
|
||||
cleanupHandled: true,
|
||||
cleanupCompletedAt: now - 5 * 60_000,
|
||||
});
|
||||
|
||||
await expect(mod.testing.sweepOnceForTests()).resolves.toBeUndefined();
|
||||
await flushSweepMicrotasks();
|
||||
|
||||
await waitForNoRequesterRuns();
|
||||
});
|
||||
|
||||
it("does not overlap archive sweep retries while sessions.delete is still in flight", async () => {
|
||||
currentConfig = {
|
||||
agents: { defaults: { subagents: { archiveAfterMinutes: 1 } } },
|
||||
|
|
|
|||
|
|
@ -271,6 +271,90 @@ describe("subagent registry persistence", () => {
|
|||
expect(persisted?.startedAt).toBeLessThanOrEqual(endedAt);
|
||||
});
|
||||
|
||||
it("rejects a stale timing write after session ownership changes", async () => {
|
||||
tempStateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-subagent-"));
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", tempStateDir);
|
||||
|
||||
const startedAt = Date.now();
|
||||
const storePath = await writeChildSessionEntry({
|
||||
sessionKey: "agent:main:subagent:stale-timing",
|
||||
sessionId: "sess-stale-timing",
|
||||
updatedAt: startedAt - 1,
|
||||
});
|
||||
await persistSubagentSessionTiming(
|
||||
{
|
||||
runId: "run-stale-timing",
|
||||
childSessionKey: "agent:main:subagent:stale-timing",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "do not persist stale timing",
|
||||
cleanup: "keep",
|
||||
createdAt: startedAt,
|
||||
startedAt,
|
||||
endedAt: startedAt + 500,
|
||||
outcome: { status: "ok" },
|
||||
} as never,
|
||||
{ isCurrentGeneration: () => false },
|
||||
);
|
||||
|
||||
const persisted = (await readSubagentSessionStore(storePath))[
|
||||
"agent:main:subagent:stale-timing"
|
||||
];
|
||||
expect(persisted).toMatchObject({
|
||||
sessionId: "sess-stale-timing",
|
||||
updatedAt: startedAt - 1,
|
||||
});
|
||||
expect(persisted?.startedAt).toBeUndefined();
|
||||
expect(persisted?.endedAt).toBeUndefined();
|
||||
expect(persisted?.status).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not overwrite durable completion with a provisional killed status", async () => {
|
||||
tempStateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-subagent-"));
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", tempStateDir);
|
||||
|
||||
const startedAt = Date.now();
|
||||
const completedAt = startedAt + 500;
|
||||
const storePath = await writeChildSessionEntry({
|
||||
sessionKey: "agent:main:subagent:kill-race",
|
||||
sessionId: "sess-kill-race",
|
||||
updatedAt: completedAt,
|
||||
});
|
||||
const store = await readSubagentSessionStore(storePath);
|
||||
store["agent:main:subagent:kill-race"] = {
|
||||
...store["agent:main:subagent:kill-race"],
|
||||
status: "done",
|
||||
startedAt,
|
||||
endedAt: completedAt,
|
||||
runtimeMs: 500,
|
||||
abortedLastRun: true,
|
||||
};
|
||||
await fs.writeFile(storePath, `${JSON.stringify(store)}\n`, "utf8");
|
||||
|
||||
await persistSubagentSessionTiming({
|
||||
runId: "run-kill-race",
|
||||
childSessionKey: "agent:main:subagent:kill-race",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "preserve completion",
|
||||
cleanup: "keep",
|
||||
createdAt: startedAt,
|
||||
startedAt,
|
||||
endedAt: completedAt + 1,
|
||||
endedReason: "subagent-killed",
|
||||
outcome: { status: "error", error: "manual kill" },
|
||||
} as never);
|
||||
|
||||
const persisted = (await readSubagentSessionStore(storePath))["agent:main:subagent:kill-race"];
|
||||
expect(persisted).toMatchObject({
|
||||
status: "done",
|
||||
startedAt,
|
||||
endedAt: completedAt,
|
||||
runtimeMs: 500,
|
||||
});
|
||||
expect(persisted?.abortedLastRun).toBeUndefined();
|
||||
});
|
||||
|
||||
it("skips cleanup when cleanupHandled was persisted", async () => {
|
||||
tempStateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-subagent-"));
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", tempStateDir);
|
||||
|
|
@ -691,6 +775,48 @@ describe("subagent registry persistence", () => {
|
|||
expect(listSubagentRunsForRequester("agent:main:main")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("preserves restored killed tombstones until bounded reconciliation", async () => {
|
||||
const now = Date.now();
|
||||
const runId = "run-killed-restore-tombstone";
|
||||
await writePersistedRegistry(
|
||||
{
|
||||
version: 2,
|
||||
runs: {
|
||||
[runId]: {
|
||||
runId,
|
||||
childSessionKey: "agent:main:subagent:killed-restore-tombstone",
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "restore killed tombstone",
|
||||
cleanup: "keep",
|
||||
createdAt: now - 100,
|
||||
startedAt: now - 50,
|
||||
endedAt: now,
|
||||
endedReason: "subagent-killed",
|
||||
outcome: { status: "error", error: "manual kill" },
|
||||
suppressAnnounceReason: "killed",
|
||||
killReconciliation: { killedAt: now },
|
||||
cleanupHandled: true,
|
||||
cleanupCompletedAt: now,
|
||||
},
|
||||
},
|
||||
},
|
||||
{ seedChildSessions: false },
|
||||
);
|
||||
|
||||
restartRegistry();
|
||||
await flushQueuedRegistryWork();
|
||||
|
||||
expect(announceSpy).not.toHaveBeenCalled();
|
||||
expect(listSubagentRunsForRequester("agent:main:main")).toEqual([
|
||||
expect.objectContaining({
|
||||
runId,
|
||||
endedReason: "subagent-killed",
|
||||
suppressAnnounceReason: "killed",
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("reconciles stale unended restored runs that are not restart-recoverable", async () => {
|
||||
const now = Date.now();
|
||||
const runId = "run-stale-unended-restore";
|
||||
|
|
|
|||
|
|
@ -2,6 +2,11 @@
|
|||
// commands while preserving lifecycle hooks and completion delivery.
|
||||
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { ContextEngine } from "../context-engine/types.js";
|
||||
import {
|
||||
getDetachedTaskLifecycleRuntime,
|
||||
setDetachedTaskLifecycleRuntime,
|
||||
} from "../tasks/detached-task-runtime.js";
|
||||
import { findTaskByRunIdForStatus } from "../tasks/task-status-access.js";
|
||||
|
||||
const noop = () => {};
|
||||
let lifecycleHandler:
|
||||
|
|
@ -14,6 +19,7 @@ let lifecycleHandler:
|
|||
endedAt?: number;
|
||||
aborted?: boolean;
|
||||
error?: string;
|
||||
stopReason?: string;
|
||||
};
|
||||
}) => void)
|
||||
| undefined;
|
||||
|
|
@ -275,6 +281,7 @@ describe("subagent registry steer restarts", () => {
|
|||
endedAt?: number;
|
||||
aborted?: boolean;
|
||||
error?: string;
|
||||
stopReason?: string;
|
||||
} = {},
|
||||
) => {
|
||||
lifecycleHandler?.({
|
||||
|
|
@ -565,6 +572,8 @@ describe("subagent registry steer restarts", () => {
|
|||
|
||||
const previous = listMainRuns()[0];
|
||||
expect(previous?.runId).toBe("run-steer-task-old");
|
||||
expect(previous?.taskRunId).toBe("run-steer-task-old");
|
||||
expect(previous?.generation).toBe(1);
|
||||
|
||||
const run = replaceRunAfterSteer({
|
||||
previousRunId: "run-steer-task-old",
|
||||
|
|
@ -574,6 +583,31 @@ describe("subagent registry steer restarts", () => {
|
|||
});
|
||||
|
||||
expect(run.task).toBe("new steer instruction from user");
|
||||
expect(run.taskRunId).toBe("run-steer-task-old");
|
||||
expect(run.generation).toBe(2);
|
||||
});
|
||||
|
||||
it("advances the generation from a fallback outside the live registry", () => {
|
||||
registerRun({
|
||||
runId: "run-fallback-generation-old",
|
||||
childSessionKey: "agent:main:subagent:fallback-generation",
|
||||
task: "restored replacement source",
|
||||
});
|
||||
const fallback = listMainRuns()[0];
|
||||
expect(fallback?.runId).toBe("run-fallback-generation-old");
|
||||
if (!fallback) {
|
||||
throw new Error("expected fallback run");
|
||||
}
|
||||
fallback.generation = 2;
|
||||
mod.releaseSubagentRun(fallback.runId);
|
||||
|
||||
const run = replaceRunAfterSteer({
|
||||
previousRunId: fallback.runId,
|
||||
nextRunId: "run-fallback-generation-new",
|
||||
fallback,
|
||||
});
|
||||
|
||||
expect(run.generation).toBe(3);
|
||||
});
|
||||
|
||||
it("preserves the previous task when no replacement is provided", () => {
|
||||
|
|
@ -598,6 +632,29 @@ describe("subagent registry steer restarts", () => {
|
|||
expect(run.task).toBe("preserve me verbatim");
|
||||
});
|
||||
|
||||
it("retains a legacy task owner fallback across another restart", () => {
|
||||
registerRun({
|
||||
runId: "run-legacy-owner-original",
|
||||
childSessionKey: "agent:main:subagent:legacy-owner",
|
||||
task: "legacy owner task",
|
||||
});
|
||||
const first = replaceRunAfterSteer({
|
||||
previousRunId: "run-legacy-owner-original",
|
||||
nextRunId: "run-legacy-owner-restored",
|
||||
});
|
||||
// Pre-change persisted replacement rows did not record taskRunId.
|
||||
first.taskRunId = undefined;
|
||||
first.sessionStartedAt = first.createdAt - 1;
|
||||
|
||||
const second = replaceRunAfterSteer({
|
||||
previousRunId: "run-legacy-owner-restored",
|
||||
nextRunId: "run-legacy-owner-next",
|
||||
fallback: first,
|
||||
});
|
||||
expect(second.taskRunId).toBeUndefined();
|
||||
expect(second.generation).toBe(3);
|
||||
});
|
||||
|
||||
it("preserves cumulative session timing across steer replacement runs", () => {
|
||||
registerRun({
|
||||
runId: "run-runtime-old",
|
||||
|
|
@ -731,7 +788,107 @@ describe("subagent registry steer restarts", () => {
|
|||
expect(announce.childRunId).toBe("run-failed-restart");
|
||||
});
|
||||
|
||||
it("marks killed runs terminated and inactive", async () => {
|
||||
it("restores announce when abandoned steer task finalization fails", async () => {
|
||||
registerRun({
|
||||
runId: "run-failed-task-finalize",
|
||||
childSessionKey: "agent:main:subagent:failed-task-finalize",
|
||||
task: "recover despite task runtime failure",
|
||||
});
|
||||
expect(mod.markSubagentRunForSteerRestart("run-failed-task-finalize")).toBe(true);
|
||||
emitLifecycleEnd("run-failed-task-finalize");
|
||||
await flushAnnounce();
|
||||
expect(announceSpy).not.toHaveBeenCalled();
|
||||
|
||||
const runtime = getDetachedTaskLifecycleRuntime();
|
||||
setDetachedTaskLifecycleRuntime({
|
||||
...runtime,
|
||||
finalizeTaskRunByRunId: () => {
|
||||
throw new Error("task store unavailable");
|
||||
},
|
||||
});
|
||||
try {
|
||||
expect(mod.clearSubagentRunSteerRestart("run-failed-task-finalize")).toBe(true);
|
||||
} finally {
|
||||
setDetachedTaskLifecycleRuntime(runtime);
|
||||
}
|
||||
|
||||
await flushAnnounce();
|
||||
expect(announceSpy).toHaveBeenCalledTimes(1);
|
||||
expect(listMainRuns()[0]?.suppressAnnounceReason).toBeUndefined();
|
||||
});
|
||||
|
||||
it("terminalizes the shared task when an interrupted steer restart is abandoned", async () => {
|
||||
registerRun({
|
||||
runId: "run-abandoned-restart",
|
||||
childSessionKey: "agent:main:subagent:abandoned-restart",
|
||||
task: "restart me or settle me",
|
||||
});
|
||||
expect(mod.markSubagentRunForSteerRestart("run-abandoned-restart")).toBe(true);
|
||||
|
||||
emitLifecycleEnd("run-abandoned-restart", {
|
||||
endedAt: Date.now(),
|
||||
aborted: true,
|
||||
stopReason: "aborted",
|
||||
});
|
||||
await waitForRegistrySideEffect(() => {
|
||||
expect(listMainRuns()[0]).toMatchObject({
|
||||
endedReason: "subagent-killed",
|
||||
suppressAnnounceReason: "steer-restart",
|
||||
});
|
||||
});
|
||||
expect(findTaskByRunIdForStatus("run-abandoned-restart")).toMatchObject({
|
||||
status: "running",
|
||||
});
|
||||
|
||||
expect(mod.clearSubagentRunSteerRestart("run-abandoned-restart")).toBe(true);
|
||||
expect(findTaskByRunIdForStatus("run-abandoned-restart")).toMatchObject({
|
||||
status: "cancelled",
|
||||
error: "Subagent restart failed after the prior run was interrupted.",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
});
|
||||
|
||||
it("terminalizes a deadline-normalized timeout when its steer restart is abandoned", async () => {
|
||||
const endedAt = Date.now();
|
||||
const startedAt = endedAt - 2_000;
|
||||
mod.registerSubagentRun({
|
||||
runId: "run-abandoned-timeout-restart",
|
||||
childSessionKey: "agent:main:subagent:abandoned-timeout-restart",
|
||||
requesterSessionKey: MAIN_REQUESTER_SESSION_KEY,
|
||||
requesterDisplayKey: MAIN_REQUESTER_DISPLAY_KEY,
|
||||
task: "restart before the deadline or time out",
|
||||
cleanup: "keep",
|
||||
runTimeoutSeconds: 1,
|
||||
});
|
||||
expect(mod.markSubagentRunForSteerRestart("run-abandoned-timeout-restart")).toBe(true);
|
||||
|
||||
emitLifecycleEnd("run-abandoned-timeout-restart", {
|
||||
startedAt,
|
||||
endedAt,
|
||||
aborted: true,
|
||||
stopReason: "aborted",
|
||||
});
|
||||
await waitForRegistrySideEffect(() => {
|
||||
expect(listMainRuns()[0]).toMatchObject({
|
||||
endedAt: startedAt + 1_000,
|
||||
endedReason: "subagent-complete",
|
||||
outcome: { status: "timeout" },
|
||||
suppressAnnounceReason: "steer-restart",
|
||||
});
|
||||
});
|
||||
await flushAnnounce();
|
||||
expect(findTaskByRunIdForStatus("run-abandoned-timeout-restart")).toMatchObject({
|
||||
status: "running",
|
||||
});
|
||||
|
||||
expect(mod.clearSubagentRunSteerRestart("run-abandoned-timeout-restart")).toBe(true);
|
||||
expect(findTaskByRunIdForStatus("run-abandoned-timeout-restart")).toMatchObject({
|
||||
status: "timed_out",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
});
|
||||
|
||||
it("marks killed runs terminated and inactive while reconciliation is pending", async () => {
|
||||
const childSessionKey = "agent:main:subagent:killed";
|
||||
|
||||
registerRun({
|
||||
|
|
@ -739,6 +896,15 @@ describe("subagent registry steer restarts", () => {
|
|||
childSessionKey,
|
||||
task: "kill me",
|
||||
});
|
||||
const activeRun = listMainRuns()[0];
|
||||
if (!activeRun) {
|
||||
throw new Error("expected active run");
|
||||
}
|
||||
activeRun.execution = {
|
||||
...activeRun.execution,
|
||||
status: activeRun.execution?.status ?? "running",
|
||||
transcriptFile: "/tmp/recovered-subagent.jsonl",
|
||||
};
|
||||
|
||||
expect(mod.isSubagentSessionRunActive(childSessionKey)).toBe(true);
|
||||
const updated = mod.markSubagentRunTerminated({
|
||||
|
|
@ -759,19 +925,8 @@ describe("subagent registry steer restarts", () => {
|
|||
expect(run?.cleanupHandled).toBe(true);
|
||||
expect(typeof run?.cleanupCompletedAt).toBe("number");
|
||||
await flushAnnounce();
|
||||
const hookCall = requireSubagentEndedHookCall("run-killed");
|
||||
expect(hookCall.event.targetSessionKey).toBe(childSessionKey);
|
||||
expect(hookCall.event.targetKind).toBe("subagent");
|
||||
expect(hookCall.event.reason).toBe("subagent-killed");
|
||||
expect(hookCall.event.sendFarewell).toBe(true);
|
||||
expect(hookCall.event.accountId).toBeUndefined();
|
||||
expect(hookCall.event.runId).toBe("run-killed");
|
||||
expect(typeof hookCall.event.endedAt).toBe("number");
|
||||
expect(hookCall.event.outcome).toBe("killed");
|
||||
expect(hookCall.event.error).toBe("manual kill");
|
||||
expect(hookCall.ctx.runId).toBe("run-killed");
|
||||
expect(hookCall.ctx.childSessionKey).toBe(childSessionKey);
|
||||
expect(hookCall.ctx.requesterSessionKey).toBe(MAIN_REQUESTER_SESSION_KEY);
|
||||
expect(runSubagentEndedHookMock).not.toHaveBeenCalled();
|
||||
expect(removeInternalSessionEffectsTranscriptMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("treats a child session as inactive when only a stale older row is still unended", () => {
|
||||
|
|
@ -832,7 +987,10 @@ describe("subagent registry steer restarts", () => {
|
|||
expect(run?.suppressAnnounceReason).toBeUndefined();
|
||||
expect(run?.cleanupHandled).toBe(true);
|
||||
expect(typeof run?.cleanupCompletedAt).toBe("number");
|
||||
expect(runSubagentEndedHookMock).toHaveBeenCalledTimes(1);
|
||||
const hookCall = requireSubagentEndedHookCall("run-kill-race");
|
||||
expect(hookCall.event.reason).toBe("subagent-complete");
|
||||
expect(hookCall.event.outcome).toBe("ok");
|
||||
expect(hookCall.event.error).toBeUndefined();
|
||||
});
|
||||
|
||||
it("retries deferred parent cleanup after a descendant announces", async () => {
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -14,6 +14,10 @@ import { createSubsystemLogger } from "../logging/subsystem.js";
|
|||
import { formatBlockedLivenessError, isBlockedLivenessState } from "../shared/agent-liveness.js";
|
||||
import { createLazyImportLoader, createLazyPromiseLoader } from "../shared/lazy-promise.js";
|
||||
import { importRuntimeModule } from "../shared/runtime-import.js";
|
||||
import { SUBAGENT_KILL_TASK_ERROR } from "../tasks/detached-task-runtime-contract.js";
|
||||
import { finalizeTaskRunByRunId, findDetachedTaskRun } from "../tasks/detached-task-runtime.js";
|
||||
import { isProvisionalSubagentKillTask } from "../tasks/task-cancellation-state.js";
|
||||
import type { TaskRecord } from "../tasks/task-registry.types.js";
|
||||
import { normalizeDeliveryContext } from "../utils/delivery-context.shared.js";
|
||||
import type { DeliveryContext } from "../utils/delivery-context.types.js";
|
||||
import {
|
||||
|
|
@ -35,6 +39,7 @@ import {
|
|||
isDeliverySuspended,
|
||||
} from "./subagent-delivery-state.js";
|
||||
import {
|
||||
SUBAGENT_ENDED_OUTCOME_KILLED,
|
||||
SUBAGENT_ENDED_REASON_COMPLETE,
|
||||
SUBAGENT_ENDED_REASON_ERROR,
|
||||
SUBAGENT_ENDED_REASON_KILLED,
|
||||
|
|
@ -47,6 +52,7 @@ import {
|
|||
import {
|
||||
ANNOUNCE_EXPIRY_MS,
|
||||
MAX_ANNOUNCE_RETRY_COUNT,
|
||||
PROVISIONAL_KILL_RECONCILIATION_MS,
|
||||
reconcileOrphanedRestoredRuns,
|
||||
reconcileOrphanedRun,
|
||||
resolveAnnounceRetryDelayMs,
|
||||
|
|
@ -81,6 +87,11 @@ import {
|
|||
} from "./subagent-registry-state.js";
|
||||
import { configureSubagentRegistrySteerRuntime } from "./subagent-registry-steer-runtime.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
import { compareSubagentRunGeneration } from "./subagent-run-generation.js";
|
||||
import {
|
||||
resolveSubagentRunDeadlineMs,
|
||||
resolveSubagentRunEffectiveEndedAt,
|
||||
} from "./subagent-run-timeout.js";
|
||||
import {
|
||||
loadSubagentSessionEntry,
|
||||
resolveCompletionFromSessionEntry,
|
||||
|
|
@ -121,7 +132,9 @@ type SubagentRegistryDeps = {
|
|||
restoreSubagentRunsFromDisk: typeof restoreSubagentRunsFromDisk;
|
||||
runSubagentAnnounceFlow: SubagentAnnounceModule["runSubagentAnnounceFlow"];
|
||||
ensureContextEnginesInitialized?: () => void;
|
||||
ensureRuntimePluginsLoaded?: typeof ensureRuntimePluginsLoadedFn;
|
||||
ensureRuntimePluginsLoaded?: (
|
||||
params: Parameters<typeof ensureRuntimePluginsLoadedFn>[0],
|
||||
) => void | Promise<void>;
|
||||
resolveContextEngine?: (
|
||||
cfg?: OpenClawConfig,
|
||||
options?: ResolveContextEngineOptions,
|
||||
|
|
@ -252,7 +265,7 @@ async function ensureSubagentRegistryPluginRuntimeLoaded(params: {
|
|||
}) {
|
||||
const ensureRuntimePluginsLoaded = subagentRegistryDeps.ensureRuntimePluginsLoaded;
|
||||
if (ensureRuntimePluginsLoaded) {
|
||||
ensureRuntimePluginsLoaded(params);
|
||||
await ensureRuntimePluginsLoaded(params);
|
||||
return;
|
||||
}
|
||||
(await loadRuntimePluginsModule()).ensureRuntimePluginsLoaded(params);
|
||||
|
|
@ -281,6 +294,78 @@ function persistSubagentRunsOrThrow() {
|
|||
subagentRegistryDeps.persistSubagentRunsToDiskOrThrow(subagentRuns);
|
||||
}
|
||||
|
||||
function findSubagentTaskForRun(entry: SubagentRunRecord) {
|
||||
const nextRunCreatedAt = findNextSubagentRunCreatedAt(entry);
|
||||
const generationStartedAt = entry.sessionStartedAt ?? entry.createdAt;
|
||||
return findDetachedTaskRun({
|
||||
runId: entry.taskRunId ?? entry.runId,
|
||||
runtime: "subagent",
|
||||
sessionKey: entry.childSessionKey,
|
||||
createdAtOrAfter: generationStartedAt,
|
||||
createdBefore: nextRunCreatedAt,
|
||||
// Steer/wake replaces the registry run ID while retaining the original
|
||||
// task row. Only those continuations may adopt a session-scoped task.
|
||||
allowSessionFallback:
|
||||
entry.taskRunId === undefined &&
|
||||
typeof entry.sessionStartedAt === "number" &&
|
||||
entry.sessionStartedAt < entry.createdAt,
|
||||
});
|
||||
}
|
||||
|
||||
function findNextSubagentRunCreatedAt(entry: SubagentRunRecord): number | undefined {
|
||||
let nextCreatedAt = entry.killReconciliation?.supersededAt;
|
||||
for (const candidate of subagentRuns.values()) {
|
||||
if (
|
||||
candidate.runId === entry.runId ||
|
||||
candidate.childSessionKey !== entry.childSessionKey ||
|
||||
compareSubagentRunGeneration(candidate, entry) <= 0
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
nextCreatedAt = Math.min(nextCreatedAt ?? candidate.createdAt, candidate.createdAt);
|
||||
}
|
||||
return nextCreatedAt;
|
||||
}
|
||||
|
||||
function resolveCompletionFromTerminalTask(
|
||||
task: TaskRecord | undefined,
|
||||
entry: SubagentRunRecord,
|
||||
):
|
||||
| {
|
||||
startedAt?: number;
|
||||
endedAt: number;
|
||||
outcome: SubagentRunOutcome;
|
||||
reason: SubagentLifecycleEndedReason;
|
||||
completionSnapshot: { resultText: string | null; capturedAt: number };
|
||||
}
|
||||
| undefined {
|
||||
if (
|
||||
!task ||
|
||||
typeof task.endedAt !== "number" ||
|
||||
(task.status !== "succeeded" && task.status !== "failed" && task.status !== "timed_out")
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
const outcome: SubagentRunOutcome =
|
||||
task.status === "succeeded"
|
||||
? { status: "ok" }
|
||||
: task.status === "timed_out"
|
||||
? { status: "timeout" }
|
||||
: { status: "error", error: task.error };
|
||||
return {
|
||||
// A steer continuation keeps the original task row but owns a new timeout
|
||||
// window. Replay against the current registry generation, not task history.
|
||||
startedAt: entry.startedAt ?? task.startedAt,
|
||||
endedAt: task.endedAt,
|
||||
outcome,
|
||||
reason: task.status === "failed" ? SUBAGENT_ENDED_REASON_ERROR : SUBAGENT_ENDED_REASON_COMPLETE,
|
||||
completionSnapshot: {
|
||||
resultText: task.progressSummary ?? task.terminalSummary ?? null,
|
||||
capturedAt: task.endedAt,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function scheduleSubagentOrphanRecovery(params?: { delayMs?: number; maxRetries?: number }) {
|
||||
const now = Date.now();
|
||||
if (now - lastOrphanRecoveryScheduleAt < ORPHAN_RECOVERY_DEBOUNCE_MS) {
|
||||
|
|
@ -362,6 +447,7 @@ type CompleteSubagentRunParams = {
|
|||
accountId?: string;
|
||||
triggerCleanup: boolean;
|
||||
startedAt?: number;
|
||||
suppressSessionEffects?: boolean;
|
||||
};
|
||||
|
||||
async function completeSubagentRunWithRecovery(params: CompleteSubagentRunParams, source: string) {
|
||||
|
|
@ -379,12 +465,7 @@ async function completeSubagentRunWithRecovery(params: CompleteSubagentRunParams
|
|||
}
|
||||
|
||||
const current = subagentRuns.get(params.runId);
|
||||
if (
|
||||
!current ||
|
||||
typeof current.endedAt !== "number" ||
|
||||
typeof current.cleanupCompletedAt === "number" ||
|
||||
current.pauseReason === "sessions_yield"
|
||||
) {
|
||||
if (!current) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -401,6 +482,12 @@ async function completeSubagentRunWithRecovery(params: CompleteSubagentRunParams
|
|||
}
|
||||
|
||||
const latest = subagentRuns.get(params.runId);
|
||||
if (latest && typeof latest.endedAt !== "number") {
|
||||
// The durable write rolled the in-memory entry back. Preserve the original
|
||||
// completion through the normal persisted-session recovery path.
|
||||
scheduleSubagentOrphanRecovery({ delayMs: 1_000 });
|
||||
return;
|
||||
}
|
||||
if (
|
||||
!latest ||
|
||||
typeof latest.endedAt !== "number" ||
|
||||
|
|
@ -557,6 +644,7 @@ async function emitSubagentEndedHookForRun(params: {
|
|||
reason?: SubagentLifecycleEndedReason;
|
||||
sendFarewell?: boolean;
|
||||
accountId?: string;
|
||||
isCurrent?: () => boolean;
|
||||
}) {
|
||||
if (params.entry.endedHookEmittedAt) {
|
||||
return;
|
||||
|
|
@ -567,8 +655,17 @@ async function emitSubagentEndedHookForRun(params: {
|
|||
workspaceDir: params.entry.workspaceDir,
|
||||
allowGatewaySubagentBinding: true,
|
||||
});
|
||||
const reason = params.reason ?? params.entry.endedReason ?? SUBAGENT_ENDED_REASON_COMPLETE;
|
||||
const outcome = resolveLifecycleOutcomeFromRunOutcome(params.entry.outcome);
|
||||
if (params.entry.endedHookEmittedAt || params.isCurrent?.() === false) {
|
||||
return;
|
||||
}
|
||||
// Plugin loading yields after the terminal lock is released. Resolve the
|
||||
// event from the canonical row only after that boundary so an older callback
|
||||
// cannot claim the exactly-once hook with a superseded timeout or error.
|
||||
const reason = params.entry.endedReason ?? params.reason ?? SUBAGENT_ENDED_REASON_COMPLETE;
|
||||
const outcome =
|
||||
reason === SUBAGENT_ENDED_REASON_KILLED
|
||||
? SUBAGENT_ENDED_OUTCOME_KILLED
|
||||
: resolveLifecycleOutcomeFromRunOutcome(params.entry.outcome);
|
||||
const error = params.entry.outcome?.status === "error" ? params.entry.outcome.error : undefined;
|
||||
await emitSubagentEndedHookOnce({
|
||||
entry: params.entry,
|
||||
|
|
@ -587,12 +684,15 @@ const subagentLifecycleController = createSubagentRegistryLifecycleController({
|
|||
resumedRuns,
|
||||
subagentAnnounceTimeoutMs: SUBAGENT_ANNOUNCE_TIMEOUT_MS,
|
||||
persist: persistSubagentRuns,
|
||||
persistOrThrow: persistSubagentRunsOrThrow,
|
||||
clearPendingLifecycleError,
|
||||
countPendingDescendantRuns,
|
||||
suppressAnnounceForSteerRestart,
|
||||
resolveSubagentTask: findSubagentTaskForRun,
|
||||
shouldEmitEndedHookForRun,
|
||||
emitSubagentEndedHookForRun,
|
||||
notifyContextEngineSubagentEnded,
|
||||
retireSupersededRun: retireSupersededSubagentRun,
|
||||
resumeSubagentRun,
|
||||
callGateway: (request) => subagentRegistryDeps.callGateway(request),
|
||||
captureSubagentCompletionReply: (sessionKey, options) =>
|
||||
|
|
@ -675,6 +775,12 @@ function resumeSubagentRun(runId: string) {
|
|||
}
|
||||
|
||||
if (typeof entry.endedAt === "number" && entry.endedAt > 0) {
|
||||
if (entry.killReconciliation) {
|
||||
// Restored kills remain reconciliation tombstones; only the sweeper may
|
||||
// accept late provider completion or stabilize their task cancellation.
|
||||
resumedRuns.add(runId);
|
||||
return;
|
||||
}
|
||||
const orphanReason = resolveSubagentRunOrphanReason({ entry });
|
||||
if (orphanReason) {
|
||||
if (
|
||||
|
|
@ -862,6 +968,21 @@ async function discardSuspendedPendingFinalDelivery(
|
|||
}
|
||||
}
|
||||
|
||||
async function retireSupersededSubagentRun(runId: string, entry: SubagentRunRecord): Promise<void> {
|
||||
const transcriptFile = entry.execution?.transcriptFile;
|
||||
clearPendingLifecycleError(runId);
|
||||
subagentRuns.delete(runId);
|
||||
const transcriptStillOwned = Array.from(subagentRuns.values()).some(
|
||||
(candidate) => candidate.execution?.transcriptFile === transcriptFile,
|
||||
);
|
||||
if (transcriptFile && !transcriptStillOwned) {
|
||||
await removeInternalSessionEffectsTranscript(transcriptFile);
|
||||
}
|
||||
if (entry.cleanup === "delete" || !entry.retainAttachmentsOnKeep) {
|
||||
await safeRemoveAttachmentsDir(entry);
|
||||
}
|
||||
}
|
||||
|
||||
async function sweepSubagentRuns() {
|
||||
if (sweepInProgress) {
|
||||
return;
|
||||
|
|
@ -979,6 +1100,226 @@ async function sweepSubagentRuns() {
|
|||
}
|
||||
}
|
||||
|
||||
if (entry.killReconciliation) {
|
||||
const killReconciliation = entry.killReconciliation;
|
||||
const taskResolutionBeforeReconciliation = findSubagentTaskForRun(entry);
|
||||
const taskBeforeReconciliation = taskResolutionBeforeReconciliation.task;
|
||||
const nextRunCreatedAt = findNextSubagentRunCreatedAt(entry);
|
||||
const hasStableTaskCancellation =
|
||||
taskBeforeReconciliation?.status === "cancelled" &&
|
||||
!isProvisionalSubagentKillTask(taskBeforeReconciliation);
|
||||
const killedAt = killReconciliation.killedAt;
|
||||
const taskCompletion =
|
||||
nextRunCreatedAt === undefined
|
||||
? resolveCompletionFromTerminalTask(taskBeforeReconciliation, entry)
|
||||
: undefined;
|
||||
if (taskCompletion) {
|
||||
// Provider reconciliation commits the non-publishing task ledger first.
|
||||
// If the registry write was interrupted, replay that durable projection
|
||||
// before the provisional kill can age into a contradictory cancellation.
|
||||
await completeSubagentRunWithRecovery(
|
||||
{
|
||||
runId,
|
||||
...taskCompletion,
|
||||
sendFarewell: true,
|
||||
accountId: entry.requesterOrigin?.accountId,
|
||||
triggerCleanup: true,
|
||||
},
|
||||
"sweeper-provisional-kill-task-completion",
|
||||
);
|
||||
const current = subagentRuns.get(runId);
|
||||
if (current !== entry || current.killReconciliation !== killReconciliation) {
|
||||
continue;
|
||||
}
|
||||
// A failed registry retry must preserve the replayable task evidence.
|
||||
continue;
|
||||
}
|
||||
const reconcileAtMs = killedAt + PROVISIONAL_KILL_RECONCILIATION_MS;
|
||||
if (reconcileAtMs > now) {
|
||||
// Even durable cancellation keeps the evidence window open: a
|
||||
// provider result persisted before killedAt remains canonical.
|
||||
continue;
|
||||
}
|
||||
const sessionEntry = loadSubagentSessionEntry({
|
||||
childSessionKey: entry.childSessionKey,
|
||||
storeCache,
|
||||
});
|
||||
const completion = resolveCompletionFromSessionEntry(sessionEntry, now, {
|
||||
notBeforeMs: entry.startedAt ?? entry.createdAt,
|
||||
});
|
||||
const completionEndedAt = completion
|
||||
? resolveSubagentRunEffectiveEndedAt(entry, completion.endedAt, completion.startedAt)
|
||||
: undefined;
|
||||
const completionDeadline = completion
|
||||
? resolveSubagentRunDeadlineMs(entry, completion.startedAt)
|
||||
: undefined;
|
||||
const killedSnapshotExpiredDeadline =
|
||||
completion?.reason === SUBAGENT_ENDED_REASON_KILLED &&
|
||||
completionDeadline !== undefined &&
|
||||
completion.endedAt > completionDeadline
|
||||
? completionDeadline
|
||||
: undefined;
|
||||
const completionCanOverrideCancellation =
|
||||
!hasStableTaskCancellation || (completionEndedAt ?? Number.POSITIVE_INFINITY) < killedAt;
|
||||
const completionBelongsToGeneration =
|
||||
nextRunCreatedAt === undefined ||
|
||||
(completion != null && completion.endedAt < nextRunCreatedAt);
|
||||
if (
|
||||
completion &&
|
||||
completionEndedAt !== undefined &&
|
||||
completionCanOverrideCancellation &&
|
||||
completionBelongsToGeneration &&
|
||||
(completion.reason !== SUBAGENT_ENDED_REASON_KILLED ||
|
||||
killedSnapshotExpiredDeadline !== undefined)
|
||||
) {
|
||||
const hasNewerGeneration = nextRunCreatedAt !== undefined;
|
||||
await completeSubagentRunWithRecovery(
|
||||
{
|
||||
runId,
|
||||
startedAt: completion.startedAt,
|
||||
endedAt: killedSnapshotExpiredDeadline ?? completion.endedAt,
|
||||
outcome:
|
||||
killedSnapshotExpiredDeadline !== undefined
|
||||
? { status: "timeout" }
|
||||
: completion.outcome,
|
||||
reason:
|
||||
killedSnapshotExpiredDeadline !== undefined
|
||||
? SUBAGENT_ENDED_REASON_COMPLETE
|
||||
: completion.reason,
|
||||
sendFarewell: true,
|
||||
accountId: entry.requesterOrigin?.accountId,
|
||||
triggerCleanup: !hasNewerGeneration,
|
||||
suppressSessionEffects: hasNewerGeneration,
|
||||
},
|
||||
"sweeper-provisional-kill-completion",
|
||||
);
|
||||
if (
|
||||
hasNewerGeneration &&
|
||||
subagentRuns.get(runId) === entry &&
|
||||
entry.endedReason !== SUBAGENT_ENDED_REASON_KILLED
|
||||
) {
|
||||
await retireSupersededSubagentRun(runId, entry);
|
||||
mutated = true;
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
subagentRuns.get(runId) !== entry ||
|
||||
entry.endedReason !== SUBAGENT_ENDED_REASON_KILLED ||
|
||||
entry.killReconciliation !== killReconciliation
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const taskResolutionAfterCompletion = findSubagentTaskForRun(entry);
|
||||
const taskAfterCompletion = taskResolutionAfterCompletion.task;
|
||||
const stableCancellationWonDuringCompletion =
|
||||
taskAfterCompletion?.status === "cancelled" &&
|
||||
!isProvisionalSubagentKillTask(taskAfterCompletion) &&
|
||||
completionEndedAt >= killedAt;
|
||||
if (
|
||||
!stableCancellationWonDuringCompletion &&
|
||||
taskResolutionAfterCompletion.lookup !== "unavailable"
|
||||
) {
|
||||
// The attempted completion did not commit. Keep both durable
|
||||
// sources unless a newer stable cancellation won during capture.
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// Completion capture yields. Revalidate both owners before promoting a
|
||||
// provisional marker into a sticky operator cancellation.
|
||||
if (
|
||||
subagentRuns.get(runId) !== entry ||
|
||||
entry.endedReason !== SUBAGENT_ENDED_REASON_KILLED ||
|
||||
entry.killReconciliation !== killReconciliation
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const taskResolutionBefore = findSubagentTaskForRun(entry);
|
||||
const taskBefore = taskResolutionBefore.task;
|
||||
const stableTaskCancellationAfterReconciliation =
|
||||
taskBefore?.status === "cancelled" && !isProvisionalSubagentKillTask(taskBefore);
|
||||
const taskNeedsStabilization =
|
||||
taskResolutionBefore.lookup === "unavailable" ||
|
||||
(taskBefore !== undefined &&
|
||||
(taskBefore.status === "queued" ||
|
||||
taskBefore.status === "running" ||
|
||||
isProvisionalSubagentKillTask(taskBefore)));
|
||||
if (taskNeedsStabilization) {
|
||||
const observedError =
|
||||
entry.outcome?.status === "error" ? entry.outcome.error?.trim() : undefined;
|
||||
try {
|
||||
// The live callback may be lost across restart. Make the provisional
|
||||
// task state stable before its last reconciliation record is deleted.
|
||||
const finalizedTasks = finalizeTaskRunByRunId({
|
||||
runId: taskBefore?.runId ?? entry.taskRunId ?? runId,
|
||||
runtime: "subagent",
|
||||
sessionKey: taskBefore?.childSessionKey ?? entry.childSessionKey,
|
||||
status: "cancelled",
|
||||
endedAt: killedAt,
|
||||
lastEventAt: killedAt,
|
||||
error:
|
||||
observedError && observedError !== SUBAGENT_KILL_TASK_ERROR
|
||||
? observedError
|
||||
: "Subagent run cancellation finalized.",
|
||||
suppressDelivery: true,
|
||||
});
|
||||
if (finalizedTasks.length === 0) {
|
||||
const taskAfterResolution = findSubagentTaskForRun(entry);
|
||||
const taskAfter = taskAfterResolution.task;
|
||||
if (
|
||||
taskAfterResolution.lookup === "available" &&
|
||||
taskAfter !== undefined &&
|
||||
(taskAfter.status === "queued" ||
|
||||
taskAfter.status === "running" ||
|
||||
isProvisionalSubagentKillTask(taskAfter))
|
||||
) {
|
||||
log.warn("killed task was not stabilized during sweep", {
|
||||
runId,
|
||||
childSessionKey: entry.childSessionKey,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (taskAfterResolution.lookup === "unavailable") {
|
||||
// Legacy custom runtimes cannot distinguish missing from
|
||||
// opaque task state. After the bounded window and one finalizer
|
||||
// attempt, do not leak the registry/session tombstone forever.
|
||||
log.warn("retiring killed tombstone after opaque task finalization", {
|
||||
runId,
|
||||
childSessionKey: entry.childSessionKey,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
log.warn("failed to finalize provisional killed task during sweep", {
|
||||
error,
|
||||
runId,
|
||||
childSessionKey: entry.childSessionKey,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (findNextSubagentRunCreatedAt(entry) !== undefined) {
|
||||
// A newer generation owns this session key. Retire only the old run;
|
||||
// session-scoped hooks or context cleanup would tear down the live owner.
|
||||
await retireSupersededSubagentRun(runId, entry);
|
||||
mutated = true;
|
||||
continue;
|
||||
}
|
||||
// Re-enter the normal cleanup owner only after cancellation is canonical.
|
||||
// It publishes the final failure once, then applies keep/delete semantics.
|
||||
entry.suppressCompletionDelivery =
|
||||
killReconciliation.suppressTaskDelivery === true ||
|
||||
hasStableTaskCancellation ||
|
||||
stableTaskCancellationAfterReconciliation
|
||||
? true
|
||||
: undefined;
|
||||
entry.suppressAnnounceReason = undefined;
|
||||
entry.killReconciliation = undefined;
|
||||
entry.cleanupHandled = false;
|
||||
entry.cleanupCompletedAt = undefined;
|
||||
mutated = true;
|
||||
startSubagentAnnounceCleanupFlow(runId, entry);
|
||||
continue;
|
||||
}
|
||||
if (!entry.archiveAtMs && entry.cleanup === "keep" && entry.spawnMode !== "session") {
|
||||
continue;
|
||||
}
|
||||
|
|
@ -1196,16 +1537,10 @@ function ensureListener() {
|
|||
const subagentRunManager = createSubagentRunManager({
|
||||
runs: subagentRuns,
|
||||
resumedRuns,
|
||||
endedHookInFlightRunIds,
|
||||
persist: persistSubagentRuns,
|
||||
persistOrThrow: persistSubagentRunsOrThrow,
|
||||
callGateway: (request) => subagentRegistryDeps.callGateway(request),
|
||||
getRuntimeConfig: () => subagentRegistryDeps.getRuntimeConfig(),
|
||||
ensureRuntimePluginsLoaded: (args: {
|
||||
config: OpenClawConfig;
|
||||
workspaceDir?: string;
|
||||
allowGatewaySubagentBinding?: boolean;
|
||||
}) => ensureSubagentRegistryPluginRuntimeLoaded(args),
|
||||
ensureListener,
|
||||
startSweeper,
|
||||
stopSweeper,
|
||||
|
|
@ -1219,6 +1554,7 @@ const subagentRunManager = createSubagentRunManager({
|
|||
notifyContextEngineSubagentEnded,
|
||||
completeCleanupBookkeeping,
|
||||
completeSubagentRun,
|
||||
resolveSubagentTask: findSubagentTaskForRun,
|
||||
});
|
||||
|
||||
configureSubagentRegistrySteerRuntime({
|
||||
|
|
@ -1387,6 +1723,7 @@ export function markSubagentRunTerminated(params: {
|
|||
runId?: string;
|
||||
childSessionKey?: string;
|
||||
reason?: string;
|
||||
suppressTaskDelivery?: boolean;
|
||||
}): number {
|
||||
return subagentRunManager.markSubagentRunTerminated(params);
|
||||
}
|
||||
|
|
@ -1526,7 +1863,7 @@ export function getLatestSubagentRunByChildSessionKey(
|
|||
if (entry.childSessionKey !== key) {
|
||||
continue;
|
||||
}
|
||||
if (!latest || entry.createdAt > latest.createdAt) {
|
||||
if (!latest || compareSubagentRunGeneration(entry, latest) > 0) {
|
||||
latest = entry;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -84,8 +84,19 @@ export type SubagentCompletionDeliveryState = {
|
|||
| "waiting_for_requester_turn";
|
||||
};
|
||||
|
||||
export type SubagentKillReconciliationState = {
|
||||
/** Actual cancellation time; a yielded run may have an older execution end. */
|
||||
killedAt: number;
|
||||
/** Requester aborts must not re-inject a delayed completion after queues are cleared. */
|
||||
suppressTaskDelivery?: boolean;
|
||||
/** Durable ownership boundary even after the newer registry row is released. */
|
||||
supersededAt?: number;
|
||||
};
|
||||
|
||||
export type SubagentRunRecord = {
|
||||
runId: string;
|
||||
/** Detached task owner; steer/restart changes runId but continues the same task. */
|
||||
taskRunId?: string;
|
||||
childSessionKey: string;
|
||||
controllerSessionKey?: string;
|
||||
requesterSessionKey: string;
|
||||
|
|
@ -100,6 +111,8 @@ export type SubagentRunRecord = {
|
|||
workspaceDir?: string;
|
||||
runTimeoutSeconds?: number;
|
||||
spawnMode?: SpawnSubagentMode;
|
||||
/** Monotonic ownership generation within one child session. */
|
||||
generation?: number;
|
||||
createdAt: number;
|
||||
startedAt?: number;
|
||||
sessionStartedAt?: number;
|
||||
|
|
@ -110,6 +123,10 @@ export type SubagentRunRecord = {
|
|||
cleanupCompletedAt?: number;
|
||||
cleanupHandled?: boolean;
|
||||
suppressAnnounceReason?: "steer-restart" | "killed";
|
||||
/** Present only while a current-version killed run awaits bounded reconciliation. */
|
||||
killReconciliation?: SubagentKillReconciliationState;
|
||||
/** Durable requester-stop policy until silent completion cleanup finishes. */
|
||||
suppressCompletionDelivery?: boolean;
|
||||
expectsCompletionMessage?: boolean;
|
||||
endedReason?: SubagentLifecycleEndedReason;
|
||||
pauseReason?: "sessions_yield";
|
||||
|
|
@ -120,6 +137,8 @@ export type SubagentRunRecord = {
|
|||
endedHookEmittedAt?: number;
|
||||
/** Set after cleanupBrowserSessionsForLifecycleEnd has been dispatched once. */
|
||||
browserCleanupDispatchedAt?: number;
|
||||
/** Set immediately before irreversible sessions.delete cleanup is dispatched. */
|
||||
deleteCleanupDispatchedAt?: number;
|
||||
/** Durable outbox marker for parent/external completion delivery. */
|
||||
delivery?: SubagentCompletionDeliveryState;
|
||||
attachmentsDir?: string;
|
||||
|
|
|
|||
45
src/agents/subagent-run-generation.ts
Normal file
45
src/agents/subagent-run-generation.ts
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
type ComparableSubagentRun = {
|
||||
runId: string;
|
||||
createdAt: number;
|
||||
generation?: number;
|
||||
};
|
||||
|
||||
type GenerationalSubagentRun = ComparableSubagentRun & {
|
||||
childSessionKey?: string;
|
||||
};
|
||||
|
||||
function normalizeGeneration(entry: ComparableSubagentRun): number {
|
||||
return typeof entry.generation === "number" && Number.isFinite(entry.generation)
|
||||
? entry.generation
|
||||
: 0;
|
||||
}
|
||||
|
||||
/** Orders runs that share a child session, including legacy rows without a generation. */
|
||||
export function compareSubagentRunGeneration(
|
||||
left: ComparableSubagentRun,
|
||||
right: ComparableSubagentRun,
|
||||
): number {
|
||||
const generationDelta = normalizeGeneration(left) - normalizeGeneration(right);
|
||||
if (generationDelta !== 0) {
|
||||
return generationDelta;
|
||||
}
|
||||
const createdAtDelta = left.createdAt - right.createdAt;
|
||||
if (createdAtDelta !== 0) {
|
||||
return createdAtDelta;
|
||||
}
|
||||
return left.runId.localeCompare(right.runId);
|
||||
}
|
||||
|
||||
/** Allocates a durable monotonic generation within one child session. */
|
||||
export function nextSubagentRunGeneration(
|
||||
runs: Iterable<GenerationalSubagentRun>,
|
||||
childSessionKey: string,
|
||||
): number {
|
||||
let generation = 0;
|
||||
for (const entry of runs) {
|
||||
if (entry.childSessionKey === childSessionKey) {
|
||||
generation = Math.max(generation, normalizeGeneration(entry));
|
||||
}
|
||||
}
|
||||
return generation + 1;
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@ import { MAX_TIMER_TIMEOUT_MS } from "../shared/number-coercion.js";
|
|||
import {
|
||||
resolveSubagentRunDeadlineMs,
|
||||
resolveSubagentRunDurationMs,
|
||||
resolveSubagentRunEffectiveEndedAt,
|
||||
resolveSubagentRunTimerDelayMs,
|
||||
} from "./subagent-run-timeout.js";
|
||||
|
||||
|
|
@ -30,6 +31,15 @@ describe("subagent run timeout helpers", () => {
|
|||
expect(resolveSubagentRunDurationMs(thirtyDaysSeconds)).toBeGreaterThan(MAX_TIMER_TIMEOUT_MS);
|
||||
});
|
||||
|
||||
it("clamps delayed terminal observations to the explicit deadline", () => {
|
||||
expect(
|
||||
resolveSubagentRunEffectiveEndedAt(
|
||||
{ createdAt: 1_000, startedAt: 2_000, runTimeoutSeconds: 3 },
|
||||
6_000,
|
||||
),
|
||||
).toBe(5_000);
|
||||
});
|
||||
|
||||
it("ignores invalid timeout seconds and invalid start timestamps", () => {
|
||||
expect(resolveSubagentRunDurationMs(Number.NaN)).toBeUndefined();
|
||||
expect(resolveSubagentRunDurationMs(0)).toBeUndefined();
|
||||
|
|
|
|||
|
|
@ -51,3 +51,13 @@ export function resolveSubagentRunDeadlineMs(
|
|||
? deadlineMs
|
||||
: undefined;
|
||||
}
|
||||
|
||||
/** Clamp a reported terminal time to the run's explicit timeout deadline. */
|
||||
export function resolveSubagentRunEffectiveEndedAt(
|
||||
entry: Pick<SubagentRunRecord, "createdAt" | "startedAt" | "runTimeoutSeconds">,
|
||||
endedAt: number,
|
||||
observedStartedAt?: number,
|
||||
): number {
|
||||
const deadlineMs = resolveSubagentRunDeadlineMs(entry, observedStartedAt);
|
||||
return deadlineMs !== undefined && endedAt > deadlineMs ? deadlineMs : endedAt;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -681,6 +681,7 @@ describe("abort detection", () => {
|
|||
childSessionKey: childKey,
|
||||
reason: "killed",
|
||||
runId: "slow-child-run",
|
||||
suppressTaskDelivery: true,
|
||||
});
|
||||
expect(getFollowupQueueDepth(sessionKey)).toBe(0);
|
||||
expectSessionLaneCleared(sessionKey);
|
||||
|
|
@ -1205,6 +1206,43 @@ describe("abort detection", () => {
|
|||
expectSessionLaneCleared(childKey);
|
||||
});
|
||||
|
||||
it("continues stopping siblings when one termination persistence write fails", () => {
|
||||
subagentRegistryMocks.markSubagentRunTerminated.mockClear();
|
||||
const sessionKey = "telegram:persistence-failure-parent";
|
||||
const firstChildKey = "agent:main:subagent:persistence-failure-first";
|
||||
const secondChildKey = "agent:main:subagent:persistence-failure-second";
|
||||
const run = (runId: string, childSessionKey: string): SubagentRunRecord => ({
|
||||
runId,
|
||||
childSessionKey,
|
||||
requesterSessionKey: sessionKey,
|
||||
requesterDisplayKey: sessionKey,
|
||||
task: "stop despite persistence failure",
|
||||
cleanup: "keep",
|
||||
createdAt: Date.now(),
|
||||
});
|
||||
subagentRegistryMocks.listSubagentRunsForRequester
|
||||
.mockReturnValueOnce([
|
||||
run("run-persistence-failure-first", firstChildKey),
|
||||
run("run-persistence-failure-second", secondChildKey),
|
||||
])
|
||||
.mockReturnValue([]);
|
||||
subagentRegistryMocks.markSubagentRunTerminated
|
||||
.mockImplementationOnce(() => {
|
||||
throw new Error("sqlite busy");
|
||||
})
|
||||
.mockReturnValue(1);
|
||||
|
||||
expect(
|
||||
stopSubagentsForRequester({
|
||||
cfg: {} as OpenClawConfig,
|
||||
requesterSessionKey: sessionKey,
|
||||
}),
|
||||
).toEqual({ stopped: 2 });
|
||||
expect(subagentRegistryMocks.markSubagentRunTerminated).toHaveBeenCalledTimes(2);
|
||||
expectSessionLaneCleared(firstChildKey);
|
||||
expectSessionLaneCleared(secondChildKey);
|
||||
});
|
||||
|
||||
it("cascade stop kills depth-2 children when stopping depth-1 agent", async () => {
|
||||
const sessionKey = "telegram:parent";
|
||||
const depth1Key = "agent:main:subagent:child-1";
|
||||
|
|
@ -1261,6 +1299,43 @@ describe("abort detection", () => {
|
|||
expectSessionLaneCleared(depth2Key);
|
||||
});
|
||||
|
||||
it("stops a subagent that is paused after yielding", () => {
|
||||
subagentRegistryMocks.listSubagentRunsForRequester.mockClear();
|
||||
subagentRegistryMocks.markSubagentRunTerminated.mockClear();
|
||||
const sessionKey = "telegram:yield-parent";
|
||||
const childKey = "agent:main:subagent:yield-child";
|
||||
const now = Date.now();
|
||||
subagentRegistryMocks.listSubagentRunsForRequester
|
||||
.mockReturnValueOnce([
|
||||
{
|
||||
runId: "run-yield-child",
|
||||
childSessionKey: childKey,
|
||||
requesterSessionKey: sessionKey,
|
||||
requesterDisplayKey: sessionKey,
|
||||
task: "paused worker",
|
||||
cleanup: "keep",
|
||||
createdAt: now - 1_000,
|
||||
endedAt: now - 500,
|
||||
pauseReason: "sessions_yield",
|
||||
},
|
||||
])
|
||||
.mockReturnValueOnce([]);
|
||||
|
||||
const result = stopSubagentsForRequester({
|
||||
cfg: {} as OpenClawConfig,
|
||||
requesterSessionKey: sessionKey,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ stopped: 1 });
|
||||
expectSessionLaneCleared(childKey);
|
||||
expect(subagentRegistryMocks.markSubagentRunTerminated).toHaveBeenCalledWith({
|
||||
runId: "run-yield-child",
|
||||
childSessionKey: childKey,
|
||||
reason: "killed",
|
||||
suppressTaskDelivery: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("cascade stop traverses ended depth-1 parents to stop active depth-2 children", async () => {
|
||||
subagentRegistryMocks.listSubagentRunsForRequester.mockClear();
|
||||
subagentRegistryMocks.markSubagentRunTerminated.mockClear();
|
||||
|
|
|
|||
|
|
@ -218,6 +218,21 @@ function normalizeRequesterSessionKey(
|
|||
return resolveInternalSessionKey({ key: cleaned, alias, mainKey });
|
||||
}
|
||||
|
||||
function markSubagentRunTerminatedBestEffort(
|
||||
params: Parameters<typeof markSubagentRunTerminated>[0],
|
||||
): number {
|
||||
try {
|
||||
return abortDeps.markSubagentRunTerminated(params);
|
||||
} catch (error) {
|
||||
// The runtime abort already happened. Keep stopping siblings and descendants;
|
||||
// durable reconciliation can retry the rolled-back registry transition later.
|
||||
logVerbose(
|
||||
`abort: failed to persist killed subagent ${params.runId ?? params.childSessionKey ?? "unknown"}: ${formatErrorMessage(error)}`,
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
export function stopSubagentsForRequester(params: {
|
||||
cfg: OpenClawConfig;
|
||||
requesterSessionKey?: string;
|
||||
|
|
@ -266,7 +281,7 @@ export function stopSubagentsForRequester(params: {
|
|||
}
|
||||
seenChildKeys.add(childKey);
|
||||
|
||||
if (!run.endedAt) {
|
||||
if (!run.endedAt || run.pauseReason === "sessions_yield") {
|
||||
const cleared = clearSessionQueues([childKey]);
|
||||
const parsed = parseAgentSessionKey(childKey);
|
||||
const storePath = resolveStorePath(params.cfg.session?.store, { agentId: parsed?.agentId });
|
||||
|
|
@ -282,10 +297,11 @@ export function stopSubagentsForRequester(params: {
|
|||
const abortRejected = abortOutcome.active && !abortOutcome.aborted;
|
||||
const markedTerminated = abortRejected
|
||||
? false
|
||||
: abortDeps.markSubagentRunTerminated({
|
||||
: markSubagentRunTerminatedBestEffort({
|
||||
runId: run.runId,
|
||||
childSessionKey: childKey,
|
||||
reason: "killed",
|
||||
suppressTaskDelivery: true,
|
||||
}) > 0;
|
||||
|
||||
if (
|
||||
|
|
|
|||
|
|
@ -11,6 +11,10 @@ import type {
|
|||
TaskTerminalOutcome,
|
||||
} from "./task-registry.types.js";
|
||||
|
||||
// A killed subagent can still report a completion that raced the kill marker.
|
||||
// Task cancellation replaces this marker once the operator request is accepted.
|
||||
export const SUBAGENT_KILL_TASK_ERROR = "Subagent run killed.";
|
||||
|
||||
export type DetachedTaskCreateParams = {
|
||||
runtime: TaskRuntime;
|
||||
taskKind?: string;
|
||||
|
|
@ -66,6 +70,7 @@ export type DetachedTaskCompleteParams = {
|
|||
progressSummary?: string | null;
|
||||
terminalSummary?: string | null;
|
||||
terminalOutcome?: TaskTerminalOutcome | null;
|
||||
suppressDelivery?: boolean;
|
||||
};
|
||||
|
||||
export type DetachedTaskFailParams = {
|
||||
|
|
@ -78,6 +83,7 @@ export type DetachedTaskFailParams = {
|
|||
error?: string;
|
||||
progressSummary?: string | null;
|
||||
terminalSummary?: string | null;
|
||||
suppressDelivery?: boolean;
|
||||
};
|
||||
|
||||
export type DetachedTaskFinalizeParams = {
|
||||
|
|
@ -91,8 +97,14 @@ export type DetachedTaskFinalizeParams = {
|
|||
progressSummary?: string | null;
|
||||
terminalSummary?: string | null;
|
||||
terminalOutcome?: TaskTerminalOutcome | null;
|
||||
suppressDelivery?: boolean;
|
||||
};
|
||||
|
||||
export type DetachedTaskTerminalState = Omit<
|
||||
DetachedTaskFinalizeParams,
|
||||
"runId" | "runtime" | "sessionKey"
|
||||
>;
|
||||
|
||||
export type DetachedTaskDeliveryStatusParams = {
|
||||
runId: string;
|
||||
runtime?: TaskRuntime;
|
||||
|
|
@ -125,6 +137,19 @@ export type DetachedTaskRecoveryAttemptResult = {
|
|||
recovered: boolean;
|
||||
};
|
||||
|
||||
export type DetachedTaskFindParams = {
|
||||
runId: string;
|
||||
runtime: TaskRuntime;
|
||||
sessionKey: string;
|
||||
createdAtOrAfter: number;
|
||||
createdBefore?: number;
|
||||
allowSessionFallback?: boolean;
|
||||
};
|
||||
|
||||
export type DetachedTaskFindResult =
|
||||
| { lookup: "available"; task?: TaskRecord }
|
||||
| { lookup: "unavailable"; task?: undefined };
|
||||
|
||||
export type DetachedTaskLifecycleRuntime = {
|
||||
createQueuedTaskRun: (params: DetachedTaskCreateParams) => TaskRecord | null;
|
||||
createRunningTaskRun: (params: DetachedRunningTaskCreateParams) => TaskRecord | null;
|
||||
|
|
@ -134,6 +159,11 @@ export type DetachedTaskLifecycleRuntime = {
|
|||
completeTaskRunByRunId: (params: DetachedTaskCompleteParams) => TaskRecord[];
|
||||
failTaskRunByRunId: (params: DetachedTaskFailParams) => TaskRecord[];
|
||||
setDetachedTaskDeliveryStatusByRunId: (params: DetachedTaskDeliveryStatusParams) => TaskRecord[];
|
||||
/**
|
||||
* Resolve the task owned by one run generation. Custom runtimes should
|
||||
* implement this when their records are not mirrored into core task state.
|
||||
*/
|
||||
findTaskRun?: (params: DetachedTaskFindParams) => TaskRecord | undefined;
|
||||
/**
|
||||
* Return `found: false` when this runtime does not own the task so core can
|
||||
* fall back to the legacy detached-task cancel path.
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import {
|
|||
createQueuedTaskRun,
|
||||
createRunningTaskRun,
|
||||
failTaskRunByRunId,
|
||||
findDetachedTaskRun,
|
||||
finalizeTaskRunByRunId,
|
||||
getDetachedTaskLifecycleRuntime,
|
||||
getDetachedTaskLifecycleRuntimeRegistration,
|
||||
|
|
@ -19,9 +20,12 @@ import {
|
|||
} from "./detached-task-runtime.js";
|
||||
import type { TaskRecord } from "./task-registry.types.js";
|
||||
|
||||
const { mockLogWarn } = vi.hoisted(() => ({
|
||||
mockLogWarn: vi.fn(),
|
||||
}));
|
||||
const { mockFindTaskByRunIdForStatus, mockListTasksForSessionKeyForStatus, mockLogWarn } =
|
||||
vi.hoisted(() => ({
|
||||
mockFindTaskByRunIdForStatus: vi.fn(),
|
||||
mockListTasksForSessionKeyForStatus: vi.fn(() => [] as TaskRecord[]),
|
||||
mockLogWarn: vi.fn(),
|
||||
}));
|
||||
vi.mock("../logging/subsystem.js", () => ({
|
||||
createSubsystemLogger: () => ({
|
||||
subsystem: "tasks/detached-runtime",
|
||||
|
|
@ -37,6 +41,11 @@ vi.mock("../logging/subsystem.js", () => ({
|
|||
}),
|
||||
}));
|
||||
|
||||
vi.mock("./task-status-access.js", () => ({
|
||||
findTaskByRunIdForStatus: mockFindTaskByRunIdForStatus,
|
||||
listTasksForSessionKeyForStatus: mockListTasksForSessionKeyForStatus,
|
||||
}));
|
||||
|
||||
function createFakeTaskRecord(overrides?: Partial<TaskRecord>): TaskRecord {
|
||||
return {
|
||||
taskId: "task-fake",
|
||||
|
|
@ -77,9 +86,123 @@ function requireFirstCallArg(
|
|||
describe("detached-task-runtime", () => {
|
||||
afterEach(() => {
|
||||
resetDetachedTaskLifecycleRuntimeForTests();
|
||||
mockFindTaskByRunIdForStatus.mockReset();
|
||||
mockListTasksForSessionKeyForStatus.mockReset();
|
||||
mockListTasksForSessionKeyForStatus.mockReturnValue([]);
|
||||
mockLogWarn.mockClear();
|
||||
});
|
||||
|
||||
it("finds a replacement task within the requested session generation", () => {
|
||||
const expected = createFakeTaskRecord({
|
||||
taskId: "task-expected",
|
||||
runtime: "subagent",
|
||||
runId: "run-shared",
|
||||
childSessionKey: "agent:main:subagent:expected",
|
||||
createdAt: 30,
|
||||
});
|
||||
mockFindTaskByRunIdForStatus.mockReturnValue(
|
||||
createFakeTaskRecord({
|
||||
taskId: "task-other-generation",
|
||||
runtime: "subagent",
|
||||
runId: "run-shared",
|
||||
childSessionKey: "agent:main:subagent:other",
|
||||
createdAt: 10,
|
||||
}),
|
||||
);
|
||||
mockListTasksForSessionKeyForStatus.mockReturnValue([
|
||||
createFakeTaskRecord({
|
||||
taskId: "task-next-generation",
|
||||
runtime: "subagent",
|
||||
runId: "run-next",
|
||||
childSessionKey: "agent:main:subagent:expected",
|
||||
createdAt: 40,
|
||||
}),
|
||||
expected,
|
||||
]);
|
||||
|
||||
expect(
|
||||
findDetachedTaskRun({
|
||||
runId: "run-shared",
|
||||
runtime: "subagent",
|
||||
sessionKey: "agent:main:subagent:expected",
|
||||
createdAtOrAfter: 15,
|
||||
createdBefore: 40,
|
||||
allowSessionFallback: true,
|
||||
}),
|
||||
).toEqual({ lookup: "available", task: expected });
|
||||
});
|
||||
|
||||
it("uses an exact task owner even when its timestamps predate the current run", () => {
|
||||
const expected = createFakeTaskRecord({
|
||||
taskId: "task-original-owner",
|
||||
runtime: "subagent",
|
||||
runId: "run-original-owner",
|
||||
childSessionKey: "agent:main:subagent:steered",
|
||||
createdAt: 10,
|
||||
});
|
||||
mockFindTaskByRunIdForStatus.mockReturnValue(expected);
|
||||
|
||||
expect(
|
||||
findDetachedTaskRun({
|
||||
runId: "run-original-owner",
|
||||
runtime: "subagent",
|
||||
sessionKey: "agent:main:subagent:steered",
|
||||
createdAtOrAfter: 20,
|
||||
createdBefore: 20,
|
||||
}),
|
||||
).toEqual({ lookup: "available", task: expected });
|
||||
expect(mockListTasksForSessionKeyForStatus).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not adopt a session task for an unchanged run ID", () => {
|
||||
mockListTasksForSessionKeyForStatus.mockReturnValue([
|
||||
createFakeTaskRecord({
|
||||
taskId: "task-unrelated",
|
||||
runtime: "subagent",
|
||||
runId: "run-unrelated",
|
||||
childSessionKey: "agent:main:subagent:expected",
|
||||
createdAt: 30,
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(
|
||||
findDetachedTaskRun({
|
||||
runId: "run-current",
|
||||
runtime: "subagent",
|
||||
sessionKey: "agent:main:subagent:expected",
|
||||
createdAtOrAfter: 15,
|
||||
createdBefore: 40,
|
||||
}),
|
||||
).toEqual({ lookup: "available", task: undefined });
|
||||
expect(mockListTasksForSessionKeyForStatus).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("contains failures from custom task lookup hooks", () => {
|
||||
setDetachedTaskLifecycleRuntime({
|
||||
...getDetachedTaskLifecycleRuntime(),
|
||||
findTaskRun: () => {
|
||||
throw new Error("lookup unavailable");
|
||||
},
|
||||
});
|
||||
|
||||
expect(
|
||||
findDetachedTaskRun({
|
||||
runId: "run-lookup-failure",
|
||||
runtime: "subagent",
|
||||
sessionKey: "agent:main:subagent:lookup-failure",
|
||||
createdAtOrAfter: 1,
|
||||
}),
|
||||
).toEqual({ lookup: "unavailable" });
|
||||
expect(mockLogWarn).toHaveBeenCalledWith(
|
||||
"Detached task lookup failed",
|
||||
expect.objectContaining({
|
||||
runtime: "subagent",
|
||||
runId: "run-lookup-failure",
|
||||
error: expect.any(Error),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("dispatches lifecycle operations through the installed runtime", async () => {
|
||||
const defaultRuntime = getDetachedTaskLifecycleRuntime();
|
||||
const queuedTask = createFakeTaskRecord({
|
||||
|
|
@ -102,6 +225,7 @@ describe("detached-task-runtime", () => {
|
|||
completeTaskRunByRunId: vi.fn(() => updatedTasks),
|
||||
failTaskRunByRunId: vi.fn(() => updatedTasks),
|
||||
setDetachedTaskDeliveryStatusByRunId: vi.fn(() => updatedTasks),
|
||||
findTaskRun: vi.fn(() => runningTask),
|
||||
cancelDetachedTaskRunById: vi.fn(async () => ({
|
||||
found: true,
|
||||
cancelled: true,
|
||||
|
|
@ -141,6 +265,14 @@ describe("detached-task-runtime", () => {
|
|||
runId: "run-running",
|
||||
deliveryStatus: "delivered",
|
||||
});
|
||||
expect(
|
||||
findDetachedTaskRun({
|
||||
runId: "run-running",
|
||||
runtime: "cli",
|
||||
sessionKey: "agent:main:main",
|
||||
createdAtOrAfter: 1,
|
||||
}),
|
||||
).toEqual({ lookup: "available", task: runningTask });
|
||||
await cancelDetachedTaskRunById({
|
||||
cfg: {} as never,
|
||||
taskId: runningTask.taskId,
|
||||
|
|
@ -182,6 +314,12 @@ describe("detached-task-runtime", () => {
|
|||
.calls[0]?.[0];
|
||||
expect(deliveryArgs?.runId).toBe("run-running");
|
||||
expect(deliveryArgs?.deliveryStatus).toBe("delivered");
|
||||
expect(fakeRuntime.findTaskRun).toHaveBeenCalledWith({
|
||||
runId: "run-running",
|
||||
runtime: "cli",
|
||||
sessionKey: "agent:main:main",
|
||||
createdAtOrAfter: 1,
|
||||
});
|
||||
expect(fakeRuntime.cancelDetachedTaskRunById).toHaveBeenCalledWith({
|
||||
cfg: {} as never,
|
||||
taskId: runningTask.taskId,
|
||||
|
|
@ -232,6 +370,21 @@ describe("detached-task-runtime", () => {
|
|||
expect(failArgs.endedAt).toBe(20);
|
||||
});
|
||||
|
||||
it("reports unavailable lookup for an opaque legacy runtime", () => {
|
||||
const legacyRuntime = { ...getDetachedTaskLifecycleRuntime() };
|
||||
delete legacyRuntime.findTaskRun;
|
||||
setDetachedTaskLifecycleRuntime(legacyRuntime);
|
||||
|
||||
expect(
|
||||
findDetachedTaskRun({
|
||||
runId: "run-not-mirrored",
|
||||
runtime: "subagent",
|
||||
sessionKey: "agent:main:subagent:not-mirrored",
|
||||
createdAtOrAfter: 1,
|
||||
}),
|
||||
).toEqual({ lookup: "unavailable" });
|
||||
});
|
||||
|
||||
describe("tryRecoverTaskBeforeMarkLost", () => {
|
||||
it("returns recovered when hook returns recovered true", async () => {
|
||||
const task = createFakeTaskRecord({ taskId: "task-recover", runtime: "subagent" });
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ import { createSubsystemLogger } from "../logging/subsystem.js";
|
|||
import type {
|
||||
DetachedTaskRecoveryAttemptParams,
|
||||
DetachedTaskRecoveryAttemptResult,
|
||||
DetachedTaskFindParams,
|
||||
DetachedTaskFindResult,
|
||||
DetachedTaskFinalizeParams,
|
||||
DetachedTaskLifecycleRuntime,
|
||||
DetachedTaskLifecycleRuntimeRegistration,
|
||||
|
|
@ -25,10 +27,37 @@ import {
|
|||
startTaskRunByRunId as startTaskRunByRunIdFromExecutor,
|
||||
} from "./task-executor.js";
|
||||
import type { TaskRecord } from "./task-registry.types.js";
|
||||
import { findTaskByRunIdForStatus, listTasksForSessionKeyForStatus } from "./task-status-access.js";
|
||||
|
||||
const log = createSubsystemLogger("tasks/detached-runtime");
|
||||
const DETACHED_TASK_RECOVERY_WARN_MS = 5_000;
|
||||
|
||||
function taskMatchesFindScope(task: TaskRecord, params: DetachedTaskFindParams): boolean {
|
||||
return (
|
||||
task.runtime === params.runtime &&
|
||||
task.childSessionKey === params.sessionKey &&
|
||||
task.createdAt >= params.createdAtOrAfter &&
|
||||
(params.createdBefore === undefined || task.createdAt < params.createdBefore)
|
||||
);
|
||||
}
|
||||
|
||||
function taskMatchesFindIdentity(task: TaskRecord, params: DetachedTaskFindParams): boolean {
|
||||
return task.runtime === params.runtime && task.childSessionKey === params.sessionKey;
|
||||
}
|
||||
|
||||
function findCoreTaskRun(params: DetachedTaskFindParams): TaskRecord | undefined {
|
||||
const direct = findTaskByRunIdForStatus(params.runId);
|
||||
if (direct && taskMatchesFindIdentity(direct, params)) {
|
||||
return direct;
|
||||
}
|
||||
if (params.allowSessionFallback !== true) {
|
||||
return undefined;
|
||||
}
|
||||
return listTasksForSessionKeyForStatus(params.sessionKey).find((task) =>
|
||||
taskMatchesFindScope(task, params),
|
||||
);
|
||||
}
|
||||
|
||||
export type { DetachedTaskLifecycleRuntime, DetachedTaskLifecycleRuntimeRegistration };
|
||||
|
||||
// Default runtime keeps detached task APIs usable before plugins install custom lifecycle hooks.
|
||||
|
|
@ -41,6 +70,7 @@ const DEFAULT_DETACHED_TASK_LIFECYCLE_RUNTIME: DetachedTaskLifecycleRuntime = {
|
|||
completeTaskRunByRunId: completeTaskRunByRunIdFromExecutor,
|
||||
failTaskRunByRunId: failTaskRunByRunIdFromExecutor,
|
||||
setDetachedTaskDeliveryStatusByRunId: setDetachedTaskDeliveryStatusByRunIdFromExecutor,
|
||||
findTaskRun: findCoreTaskRun,
|
||||
cancelDetachedTaskRunById: cancelDetachedTaskRunByIdInCore,
|
||||
};
|
||||
|
||||
|
|
@ -125,6 +155,26 @@ export function setDetachedTaskDeliveryStatusByRunId(
|
|||
return getDetachedTaskLifecycleRuntime().setDetachedTaskDeliveryStatusByRunId(...args);
|
||||
}
|
||||
|
||||
export function findDetachedTaskRun(params: DetachedTaskFindParams): DetachedTaskFindResult {
|
||||
const runtime = getDetachedTaskLifecycleRuntime();
|
||||
if (runtime.findTaskRun) {
|
||||
try {
|
||||
return { lookup: "available", task: runtime.findTaskRun(params) };
|
||||
} catch (error) {
|
||||
log.warn("Detached task lookup failed", {
|
||||
runtime: params.runtime,
|
||||
runId: params.runId,
|
||||
error,
|
||||
});
|
||||
return { lookup: "unavailable" };
|
||||
}
|
||||
}
|
||||
const coreTask = findCoreTaskRun(params);
|
||||
// Older custom runtimes may mirror records into core. When they do not, an
|
||||
// empty fallback cannot prove that the runtime-owned task is absent.
|
||||
return coreTask ? { lookup: "available", task: coreTask } : { lookup: "unavailable" };
|
||||
}
|
||||
|
||||
export function cancelDetachedTaskRunById(
|
||||
...args: Parameters<DetachedTaskLifecycleRuntime["cancelDetachedTaskRunById"]>
|
||||
): ReturnType<DetachedTaskLifecycleRuntime["cancelDetachedTaskRunById"]> {
|
||||
|
|
|
|||
21
src/tasks/task-cancellation-state.ts
Normal file
21
src/tasks/task-cancellation-state.ts
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
// Defines task states that still participate in flow cancellation.
|
||||
import { SUBAGENT_KILL_TASK_ERROR } from "./detached-task-runtime-contract.js";
|
||||
import type { TaskRecord } from "./task-registry.types.js";
|
||||
|
||||
export function isProvisionalSubagentKillTask(
|
||||
task: Pick<TaskRecord, "runtime" | "status" | "error">,
|
||||
): boolean {
|
||||
return (
|
||||
task.runtime === "subagent" &&
|
||||
task.status === "cancelled" &&
|
||||
task.error === SUBAGENT_KILL_TASK_ERROR
|
||||
);
|
||||
}
|
||||
|
||||
export function isTaskFlowCancellationPending(
|
||||
task: Pick<TaskRecord, "runtime" | "status" | "error">,
|
||||
): boolean {
|
||||
return (
|
||||
task.status === "queued" || task.status === "running" || isProvisionalSubagentKillTask(task)
|
||||
);
|
||||
}
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
// Verifies task executor delivery policy and terminal message formatting.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { SUBAGENT_KILL_TASK_ERROR } from "./detached-task-runtime-contract.js";
|
||||
import {
|
||||
formatTaskBlockedFollowupMessage,
|
||||
formatTaskStateChangeMessage,
|
||||
|
|
@ -71,6 +72,11 @@ describe("task-executor-policy", () => {
|
|||
expect(formatTaskBlockedFollowupMessage(blockedTask)).toBe(
|
||||
"Task needs follow-up: ACP import (run run-1234). Needs login.",
|
||||
);
|
||||
expect(
|
||||
formatTaskTerminalMessage(
|
||||
createTask({ runtime: "subagent", status: "cancelled", runId: "run-34567890" }),
|
||||
),
|
||||
).toBe("Background task cancellation requested: Subagent task (run run-3456).");
|
||||
expect(formatTaskStateChangeMessage(blockedTask, progressEvent)).toBe(
|
||||
"Background task update: ACP import. No output for 60s.",
|
||||
);
|
||||
|
|
@ -152,6 +158,26 @@ describe("task-executor-policy", () => {
|
|||
}),
|
||||
),
|
||||
).toBe(false);
|
||||
expect(
|
||||
shouldAutoDeliverTaskTerminalUpdate(
|
||||
createTask({
|
||||
runtime: "subagent",
|
||||
status: "cancelled",
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
deliveryStatus: "pending",
|
||||
}),
|
||||
),
|
||||
).toBe(false);
|
||||
expect(
|
||||
shouldAutoDeliverTaskTerminalUpdate(
|
||||
createTask({
|
||||
runtime: "subagent",
|
||||
status: "cancelled",
|
||||
error: "Cancelled by operator.",
|
||||
deliveryStatus: "pending",
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
shouldAutoDeliverTaskStateChange(
|
||||
createTask({
|
||||
|
|
@ -179,6 +205,17 @@ describe("task-executor-policy", () => {
|
|||
preferredTaskId: "task-2",
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
shouldSuppressDuplicateTerminalDelivery({
|
||||
task: createTask({
|
||||
runtime: "subagent",
|
||||
status: "cancelled",
|
||||
runId: "run-duplicate",
|
||||
}),
|
||||
preferredTaskId: "task-1",
|
||||
peerDeliveryCovered: true,
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
shouldSuppressDuplicateTerminalDelivery({
|
||||
task: createTask({
|
||||
|
|
@ -197,6 +234,16 @@ describe("task-executor-policy", () => {
|
|||
preferredTaskId: undefined,
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
shouldSuppressDuplicateTerminalDelivery({
|
||||
task: createTask({
|
||||
runtime: "subagent",
|
||||
status: "cancelled",
|
||||
runId: "run-duplicate",
|
||||
}),
|
||||
preferredTaskId: "task-2",
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
shouldUseParentReviewTaskTerminalMessage(
|
||||
createTask({
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
// Decides task executor delivery, terminal update, and follow-up message policy.
|
||||
import { SUBAGENT_KILL_TASK_ERROR } from "./detached-task-runtime-contract.js";
|
||||
import type { TaskEventRecord, TaskRecord, TaskStatus } from "./task-registry.types.js";
|
||||
import { formatTaskStatusTitleText, sanitizeTaskStatusText } from "./task-status.js";
|
||||
|
||||
|
|
@ -62,6 +63,11 @@ export function formatTaskTerminalMessage(
|
|||
return `Background task lost: ${title}${runLabel}. ${error || fallbackSummary || "Backing session disappeared."}`;
|
||||
}
|
||||
if (task.status === "cancelled") {
|
||||
if (task.runtime === "subagent") {
|
||||
// A final reply can win the kill race and reconcile this row to success.
|
||||
// Report the operator action without claiming an irreversible outcome.
|
||||
return `Background task cancellation requested: ${title}${runLabel}.`;
|
||||
}
|
||||
return `Background task cancelled: ${title}${runLabel}.`;
|
||||
}
|
||||
const error = sanitizeTaskStatusText(task.error, { errorContext: true });
|
||||
|
|
@ -114,6 +120,15 @@ export function shouldAutoDeliverTaskTerminalUpdate(task: TaskRecord): boolean {
|
|||
return false;
|
||||
}
|
||||
if (task.runtime === "subagent" && task.status !== "cancelled") {
|
||||
// Subagent lifecycle owns provider-result publication.
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
task.runtime === "subagent" &&
|
||||
task.status === "cancelled" &&
|
||||
task.error === SUBAGENT_KILL_TASK_ERROR
|
||||
) {
|
||||
// A direct kill is provisional until lifecycle reconciliation settles.
|
||||
return false;
|
||||
}
|
||||
if (!isTerminalTaskStatus(task.status)) {
|
||||
|
|
@ -133,9 +148,19 @@ export function shouldAutoDeliverTaskStateChange(task: TaskRecord): boolean {
|
|||
export function shouldSuppressDuplicateTerminalDelivery(params: {
|
||||
task: TaskRecord;
|
||||
preferredTaskId?: string;
|
||||
peerDeliveryCovered?: boolean;
|
||||
}): boolean {
|
||||
if (params.task.runtime !== "acp" || !params.task.runId?.trim()) {
|
||||
if (!params.task.runId?.trim()) {
|
||||
return false;
|
||||
}
|
||||
const sharesRunDelivery =
|
||||
params.task.runtime === "acp" ||
|
||||
(params.task.runtime === "subagent" && params.task.status === "cancelled");
|
||||
if (!sharesRunDelivery) {
|
||||
return false;
|
||||
}
|
||||
if (params.task.runtime === "subagent" && params.peerDeliveryCovered) {
|
||||
return true;
|
||||
}
|
||||
return Boolean(params.preferredTaskId && params.preferredTaskId !== params.task.taskId);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import { resetHeartbeatWakeStateForTests } from "../infra/heartbeat-wake.js";
|
|||
import { resetSystemEventsForTest } from "../infra/system-events.js";
|
||||
import { withStateDirEnv } from "../test-helpers/state-dir-env.js";
|
||||
import { captureEnv } from "../test-utils/env.js";
|
||||
import { SUBAGENT_KILL_TASK_ERROR } from "./detached-task-runtime-contract.js";
|
||||
import {
|
||||
getDetachedTaskLifecycleRuntime,
|
||||
resetDetachedTaskLifecycleRuntimeForTests,
|
||||
|
|
@ -362,6 +363,58 @@ describe("task-executor", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("promotes a provisional kill in an already-cancelled one-task flow", async () => {
|
||||
await withTaskExecutorStateDir(async () => {
|
||||
const child = createRunningTaskRun({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:worker:subagent:mirrored-kill",
|
||||
runId: "run-mirrored-provisional-kill",
|
||||
task: "Stop mirrored child",
|
||||
startedAt: 10,
|
||||
deliveryStatus: "pending",
|
||||
});
|
||||
const flowId = expectParentFlowId(child);
|
||||
failTaskRunByRunId({
|
||||
runId: child.runId!,
|
||||
runtime: "subagent",
|
||||
sessionKey: child.childSessionKey,
|
||||
status: "cancelled",
|
||||
endedAt: 20,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
});
|
||||
hoisted.killSubagentRunAdminMock.mockResolvedValueOnce({
|
||||
found: true,
|
||||
killed: false,
|
||||
runId: child.runId!,
|
||||
sessionKey: child.childSessionKey!,
|
||||
targetState: {
|
||||
state: "terminal",
|
||||
task: { status: "cancelled", endedAt: 20, error: SUBAGENT_KILL_TASK_ERROR },
|
||||
},
|
||||
});
|
||||
expect(getTaskFlowById(flowId)?.status).toBe("cancelled");
|
||||
|
||||
const cancelled = await cancelFlowById({ cfg: {} as never, flowId });
|
||||
completeTaskRunByRunId({
|
||||
runId: child.runId!,
|
||||
runtime: "subagent",
|
||||
sessionKey: child.childSessionKey,
|
||||
endedAt: 30,
|
||||
terminalSummary: "completed too late",
|
||||
});
|
||||
|
||||
expect(cancelled).toMatchObject({ found: true, cancelled: true });
|
||||
expect(getTaskFlowById(flowId)?.status).toBe("cancelled");
|
||||
expect(getTaskById(child.taskId)).toMatchObject({
|
||||
status: "cancelled",
|
||||
endedAt: 20,
|
||||
error: "Cancelled by operator.",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("does not auto-create one-task flows for non-returning bookkeeping runs", async () => {
|
||||
await withTaskExecutorStateDir(async () => {
|
||||
const created = createRunningTaskRun({
|
||||
|
|
@ -421,6 +474,61 @@ describe("task-executor", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("promotes provisional subagent kills before cancelling a managed TaskFlow", async () => {
|
||||
await withTaskExecutorStateDir(async () => {
|
||||
const flow = createManagedTaskFlow({
|
||||
ownerKey: "agent:main:main",
|
||||
controllerId: "tests/managed-flow",
|
||||
goal: "Cancel a killed child",
|
||||
});
|
||||
const created = runTaskInFlow({
|
||||
flowId: flow.flowId,
|
||||
runtime: "subagent",
|
||||
childSessionKey: "agent:worker:subagent:flow-killed",
|
||||
runId: "run-flow-provisional-kill",
|
||||
task: "Stop the child",
|
||||
status: "running",
|
||||
startedAt: 10,
|
||||
});
|
||||
const child = requireCreatedFlowTask(created);
|
||||
failTaskRunByRunId({
|
||||
runId: child.runId!,
|
||||
runtime: "subagent",
|
||||
sessionKey: child.childSessionKey,
|
||||
status: "cancelled",
|
||||
endedAt: 20,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
});
|
||||
hoisted.killSubagentRunAdminMock.mockResolvedValueOnce({
|
||||
found: true,
|
||||
killed: false,
|
||||
runId: child.runId!,
|
||||
sessionKey: child.childSessionKey!,
|
||||
targetState: {
|
||||
state: "terminal",
|
||||
task: { status: "cancelled", endedAt: 20, error: SUBAGENT_KILL_TASK_ERROR },
|
||||
},
|
||||
});
|
||||
|
||||
const cancelled = await cancelFlowById({ cfg: {} as never, flowId: flow.flowId });
|
||||
completeTaskRunByRunId({
|
||||
runId: child.runId!,
|
||||
runtime: "subagent",
|
||||
sessionKey: child.childSessionKey,
|
||||
endedAt: 30,
|
||||
terminalSummary: "completed too late",
|
||||
});
|
||||
|
||||
expect(cancelled).toMatchObject({ found: true, cancelled: true });
|
||||
expect(getTaskFlowById(flow.flowId)?.status).toBe("cancelled");
|
||||
expect(getTaskById(child.taskId)).toMatchObject({
|
||||
status: "cancelled",
|
||||
endedAt: 20,
|
||||
error: "Cancelled by operator.",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("runs child tasks under managed TaskFlows", async () => {
|
||||
await withTaskExecutorStateDir(async () => {
|
||||
const flow = createManagedTaskFlow({
|
||||
|
|
|
|||
|
|
@ -19,6 +19,10 @@ import {
|
|||
recordTaskProgressByRunId,
|
||||
setTaskRunDeliveryStatusByRunId,
|
||||
} from "./runtime-internal.js";
|
||||
import {
|
||||
isProvisionalSubagentKillTask,
|
||||
isTaskFlowCancellationPending,
|
||||
} from "./task-cancellation-state.js";
|
||||
import { getTaskFlowByIdForOwner } from "./task-flow-owner-access.js";
|
||||
import type { TaskFlowRecord } from "./task-flow-registry.types.js";
|
||||
import {
|
||||
|
|
@ -174,6 +178,7 @@ export function completeTaskRunByRunId(params: {
|
|||
progressSummary?: string | null;
|
||||
terminalSummary?: string | null;
|
||||
terminalOutcome?: TaskTerminalOutcome | null;
|
||||
suppressDelivery?: boolean;
|
||||
}) {
|
||||
return finalizeTaskRunByRunId({
|
||||
...params,
|
||||
|
|
@ -195,6 +200,7 @@ export function failTaskRunByRunId(params: {
|
|||
error?: string;
|
||||
progressSummary?: string | null;
|
||||
terminalSummary?: string | null;
|
||||
suppressDelivery?: boolean;
|
||||
}) {
|
||||
return finalizeTaskRunByRunId({
|
||||
...params,
|
||||
|
|
@ -228,10 +234,6 @@ type RunTaskInFlowResult = {
|
|||
task?: TaskRecord;
|
||||
};
|
||||
|
||||
function isActiveTaskStatus(status: TaskStatus): boolean {
|
||||
return status === "queued" || status === "running";
|
||||
}
|
||||
|
||||
function isTerminalFlowStatus(status: TaskFlowRecord["status"]): boolean {
|
||||
return (
|
||||
status === "succeeded" || status === "failed" || status === "cancelled" || status === "lost"
|
||||
|
|
@ -475,6 +477,33 @@ export async function cancelFlowById(params: {
|
|||
};
|
||||
}
|
||||
if (isTerminalFlowStatus(flow.status)) {
|
||||
const provisionalTasks = listTasksForFlowId(flow.flowId).filter(isProvisionalSubagentKillTask);
|
||||
if (flow.status === "cancelled" && provisionalTasks.length > 0) {
|
||||
for (const task of provisionalTasks) {
|
||||
await cancelDetachedTaskRunById({ cfg: params.cfg, taskId: task.taskId });
|
||||
}
|
||||
const tasks = listTasksForFlowId(flow.flowId);
|
||||
if (tasks.some(isProvisionalSubagentKillTask)) {
|
||||
return {
|
||||
found: true,
|
||||
cancelled: false,
|
||||
reason: "One or more child tasks remain provisionally cancelled.",
|
||||
flow: getTaskFlowById(flow.flowId) ?? flow,
|
||||
tasks,
|
||||
};
|
||||
}
|
||||
const refreshedFlow = getTaskFlowById(flow.flowId) ?? flow;
|
||||
return {
|
||||
found: true,
|
||||
cancelled: refreshedFlow.status === "cancelled",
|
||||
reason:
|
||||
refreshedFlow.status === "cancelled"
|
||||
? undefined
|
||||
: `Flow is already ${refreshedFlow.status}.`,
|
||||
flow: refreshedFlow,
|
||||
tasks,
|
||||
};
|
||||
}
|
||||
return {
|
||||
found: true,
|
||||
cancelled: false,
|
||||
|
|
@ -494,7 +523,7 @@ export async function cancelFlowById(params: {
|
|||
};
|
||||
}
|
||||
const linkedTasks = listTasksForFlowId(flow.flowId);
|
||||
const activeTasks = linkedTasks.filter((task) => isActiveTaskStatus(task.status));
|
||||
const activeTasks = linkedTasks.filter(isTaskFlowCancellationPending);
|
||||
for (const task of activeTasks) {
|
||||
await cancelDetachedTaskRunById({
|
||||
cfg: params.cfg,
|
||||
|
|
@ -502,7 +531,7 @@ export async function cancelFlowById(params: {
|
|||
});
|
||||
}
|
||||
const refreshedTasks = listTasksForFlowId(flow.flowId);
|
||||
const remainingActive = refreshedTasks.filter((task) => isActiveTaskStatus(task.status));
|
||||
const remainingActive = refreshedTasks.filter(isTaskFlowCancellationPending);
|
||||
if (remainingActive.length > 0) {
|
||||
return {
|
||||
found: true,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,11 @@
|
|||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { captureEnv } from "../test-utils/env.js";
|
||||
import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js";
|
||||
import { createRunningTaskRun as createRunningTaskRunOrNull } from "./task-executor.js";
|
||||
import { SUBAGENT_KILL_TASK_ERROR } from "./detached-task-runtime-contract.js";
|
||||
import {
|
||||
createRunningTaskRun as createRunningTaskRunOrNull,
|
||||
finalizeTaskRunByRunId,
|
||||
} from "./task-executor.js";
|
||||
import {
|
||||
listTaskFlowAuditFindings,
|
||||
type TaskFlowAuditCode,
|
||||
|
|
@ -10,6 +14,7 @@ import {
|
|||
} from "./task-flow-registry.audit.js";
|
||||
import {
|
||||
createManagedTaskFlow as createManagedTaskFlowOrNull,
|
||||
requestFlowCancel,
|
||||
resetTaskFlowRegistryForTests,
|
||||
setFlowWaiting,
|
||||
} from "./task-flow-registry.js";
|
||||
|
|
@ -242,4 +247,50 @@ describe("task-flow-registry audit", () => {
|
|||
expect(requireFinding(findings, "cancel_stuck", flow.flowId).flow?.flowId).toBe(flow.flowId);
|
||||
});
|
||||
});
|
||||
|
||||
it("counts provisional subagent cancellation as active during audit", async () => {
|
||||
await withTaskFlowAuditStateDir(async () => {
|
||||
const now = Date.now();
|
||||
const flow = createManagedTaskFlow({
|
||||
ownerKey: "agent:main:main",
|
||||
controllerId: "tests/task-flow-audit",
|
||||
goal: "Cancel subagent work",
|
||||
status: "running",
|
||||
createdAt: now - 6 * 60_000,
|
||||
updatedAt: now - 6 * 60_000,
|
||||
});
|
||||
const runId = "run-provisional-cancel-audit";
|
||||
const task = createRunningTaskRun({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
parentFlowId: flow.flowId,
|
||||
childSessionKey: "agent:main:subagent:provisional-cancel",
|
||||
runId,
|
||||
task: "Wait for kill reconciliation",
|
||||
startedAt: now - 6 * 60_000,
|
||||
lastEventAt: now - 6 * 60_000,
|
||||
});
|
||||
expect(task.runId).toBe(runId);
|
||||
requestFlowCancel({
|
||||
flowId: flow.flowId,
|
||||
expectedRevision: flow.revision,
|
||||
cancelRequestedAt: now - 6 * 60_000,
|
||||
updatedAt: now - 6 * 60_000,
|
||||
});
|
||||
finalizeTaskRunByRunId({
|
||||
runId,
|
||||
runtime: "subagent",
|
||||
status: "cancelled",
|
||||
endedAt: now - 6 * 60_000,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
});
|
||||
|
||||
expect(
|
||||
listTaskFlowAuditFindings({ now }).find(
|
||||
(finding) => finding.code === "cancel_stuck" && finding.flow?.flowId === flow.flowId,
|
||||
),
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
// Produces task-flow registry audit summaries for diagnostics and maintenance.
|
||||
import { listTasksForFlowId } from "./runtime-internal.js";
|
||||
import { isTaskFlowCancellationPending } from "./task-cancellation-state.js";
|
||||
import { getTaskFlowRegistryRestoreFailure, listTaskFlowRecords } from "./task-flow-registry.js";
|
||||
import type { TaskFlowRecord } from "./task-flow-registry.types.js";
|
||||
import type { TaskRecord } from "./task-registry.types.js";
|
||||
|
|
@ -164,9 +165,7 @@ export function listTaskFlowAuditFindings(
|
|||
const referenceAt = getReferenceAt(flow);
|
||||
const ageMs = Math.max(0, now - referenceAt);
|
||||
const linkedTasks = getLinkedTasks(flow.flowId);
|
||||
const activeTasks = linkedTasks.filter(
|
||||
(task) => task.status === "queued" || task.status === "running",
|
||||
);
|
||||
const activeTasks = linkedTasks.filter((task) => isTaskFlowCancellationPending(task));
|
||||
|
||||
if (flow.status === "running" && ageMs >= staleRunningMs) {
|
||||
findings.push(
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { captureEnv } from "../test-utils/env.js";
|
||||
import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js";
|
||||
import { SUBAGENT_KILL_TASK_ERROR } from "./detached-task-runtime-contract.js";
|
||||
import { createRunningTaskRun as createRunningTaskRunOrNull } from "./task-executor.js";
|
||||
import {
|
||||
createFlowRecord as createFlowRecordOrNull,
|
||||
|
|
@ -18,6 +19,7 @@ import {
|
|||
} from "./task-flow-registry.maintenance.js";
|
||||
import type { TaskFlowRecord } from "./task-flow-registry.types.js";
|
||||
import {
|
||||
finalizeTaskRunByRunId,
|
||||
resetTaskRegistryDeliveryRuntimeForTests,
|
||||
resetTaskRegistryForTests,
|
||||
} from "./task-registry.js";
|
||||
|
|
@ -229,6 +231,56 @@ describe("task-flow-registry maintenance", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("does not finalize cancel-requested flows while a child kill is provisional", async () => {
|
||||
await withTaskFlowMaintenanceStateDir(async () => {
|
||||
const flow = createManagedTaskFlow({
|
||||
ownerKey: "agent:main:main",
|
||||
controllerId: "tests/task-flow-maintenance",
|
||||
goal: "Wait for child kill reconciliation",
|
||||
status: "running",
|
||||
createdAt: 1,
|
||||
updatedAt: 100,
|
||||
});
|
||||
const child = createRunningTaskRun({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
parentFlowId: flow.flowId,
|
||||
childSessionKey: "agent:main:subagent:provisional-kill",
|
||||
runId: "run-provisional-kill",
|
||||
task: "Finish while cancellation races",
|
||||
startedAt: 100,
|
||||
lastEventAt: 100,
|
||||
});
|
||||
finalizeTaskRunByRunId({
|
||||
runId: child.runId!,
|
||||
runtime: "subagent",
|
||||
sessionKey: child.childSessionKey,
|
||||
status: "cancelled",
|
||||
endedAt: 110,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
});
|
||||
const currentFlow = getTaskFlowById(flow.flowId);
|
||||
if (!currentFlow) {
|
||||
throw new Error("Expected provisional child flow to remain registered");
|
||||
}
|
||||
const cancelResult = requestFlowCancel({
|
||||
flowId: currentFlow.flowId,
|
||||
expectedRevision: currentFlow.revision,
|
||||
cancelRequestedAt: 120,
|
||||
updatedAt: 120,
|
||||
});
|
||||
expect(cancelResult.applied).toBe(true);
|
||||
|
||||
expect(previewTaskFlowRegistryMaintenance()).toEqual({ reconciled: 0, pruned: 0 });
|
||||
expect(await runTaskFlowRegistryMaintenance()).toEqual({ reconciled: 0, pruned: 0 });
|
||||
expect(getTaskFlowById(flow.flowId)).toMatchObject({
|
||||
status: "running",
|
||||
cancelRequestedAt: 120,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("prunes many old terminal flows while keeping fresh and active ones", async () => {
|
||||
await withTaskFlowMaintenanceStateDir(async () => {
|
||||
const now = Date.now();
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
// Reconciles stale task-flow records with their child task state.
|
||||
import { listTasksForFlowId } from "./runtime-internal.js";
|
||||
import { isTaskFlowCancellationPending } from "./task-cancellation-state.js";
|
||||
import {
|
||||
listTaskFlowAuditFindings,
|
||||
summarizeTaskFlowAuditFindings,
|
||||
|
|
@ -32,9 +33,7 @@ function isTerminalFlow(flow: TaskFlowRecord): boolean {
|
|||
}
|
||||
|
||||
function hasActiveLinkedTasks(flowId: string): boolean {
|
||||
return listTasksForFlowId(flowId).some(
|
||||
(task) => task.status === "queued" || task.status === "running",
|
||||
);
|
||||
return listTasksForFlowId(flowId).some(isTaskFlowCancellationPending);
|
||||
}
|
||||
|
||||
function resolveTerminalAt(flow: TaskFlowRecord): number {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,10 @@
|
|||
// Defines task control runtime contracts exposed to command surfaces.
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import type { DetachedTaskTerminalState } from "./detached-task-runtime-contract.js";
|
||||
|
||||
type KillSubagentTargetState =
|
||||
| { state: "finalizing" }
|
||||
| { state: "terminal"; task: DetachedTaskTerminalState };
|
||||
|
||||
/** Admin cancellation hook for ACP sessions owned by task records. */
|
||||
export type CancelAcpSessionAdmin = (params: {
|
||||
|
|
@ -8,14 +13,17 @@ export type CancelAcpSessionAdmin = (params: {
|
|||
reason: string;
|
||||
}) => Promise<void>;
|
||||
|
||||
export type KillSubagentRunAdminResult = {
|
||||
found: boolean;
|
||||
killed: boolean;
|
||||
runId?: string;
|
||||
sessionKey?: string;
|
||||
cascadeKilled?: number;
|
||||
cascadeLabels?: string[];
|
||||
};
|
||||
export type KillSubagentRunAdminResult =
|
||||
| { found: false; killed: false }
|
||||
| {
|
||||
found: true;
|
||||
killed: boolean;
|
||||
targetState?: KillSubagentTargetState;
|
||||
runId: string;
|
||||
sessionKey: string;
|
||||
cascadeKilled: number;
|
||||
cascadeLabels?: string[];
|
||||
};
|
||||
|
||||
export type KillSubagentRunAdmin = (params: {
|
||||
cfg: OpenClawConfig;
|
||||
|
|
|
|||
|
|
@ -18,10 +18,12 @@ import type { ParsedAgentSessionKey } from "../routing/session-key.js";
|
|||
import { withTempDir } from "../test-helpers/temp-dir.js";
|
||||
import { withEnvAsync } from "../test-utils/env.js";
|
||||
import { registerActiveCronTaskRun, resetActiveCronTaskRunsForTests } from "./cron-task-cancel.js";
|
||||
import { SUBAGENT_KILL_TASK_ERROR } from "./detached-task-runtime-contract.js";
|
||||
import {
|
||||
createTaskFlowForTask as createTaskFlowForTaskOrNull,
|
||||
createManagedTaskFlow as createManagedTaskFlowOrNull,
|
||||
getTaskFlowById,
|
||||
requestFlowCancel,
|
||||
resetTaskFlowRegistryForTests,
|
||||
} from "./task-flow-registry.js";
|
||||
import { configureTaskFlowRegistryRuntime } from "./task-flow-registry.store.js";
|
||||
|
|
@ -524,6 +526,47 @@ describe("task-registry", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("keeps subagent abort lifecycle projections provisional", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
resetTaskRegistryMemoryForTest();
|
||||
createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:main:subagent:abort-race",
|
||||
runId: "run-subagent-abort-race",
|
||||
task: "Finish while aborting",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
startedAt: 100,
|
||||
});
|
||||
|
||||
emitAgentEvent({
|
||||
runId: "run-subagent-abort-race",
|
||||
stream: "lifecycle",
|
||||
data: { phase: "end", stopReason: "aborted", endedAt: 200 },
|
||||
});
|
||||
expectRecordFields(requireTaskByRunId("run-subagent-abort-race"), {
|
||||
status: "cancelled",
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
});
|
||||
|
||||
finalizeTaskRunByRunId({
|
||||
runId: "run-subagent-abort-race",
|
||||
runtime: "subagent",
|
||||
status: "succeeded",
|
||||
endedAt: 201,
|
||||
terminalSummary: "finished",
|
||||
});
|
||||
expectRecordFields(requireTaskByRunId("run-subagent-abort-race"), {
|
||||
status: "succeeded",
|
||||
endedAt: 201,
|
||||
error: undefined,
|
||||
terminalSummary: "finished",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("reuses an ACP run task when a derived flow id is linked before a duplicate create", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
resetTaskRegistryMemoryForTest({ persist: false });
|
||||
|
|
@ -654,6 +697,79 @@ describe("task-registry", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("recovers only direct subagent kill markers when completion wins the race", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
for (const entry of [
|
||||
{
|
||||
runId: "run-subagent-late-success",
|
||||
runtime: "subagent" as const,
|
||||
childSessionKey: "agent:main:subagent:late-success",
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
terminalStatus: "succeeded" as const,
|
||||
terminalError: undefined,
|
||||
expectedError: undefined,
|
||||
},
|
||||
{
|
||||
runId: "run-subagent-late-failure",
|
||||
runtime: "subagent" as const,
|
||||
childSessionKey: "agent:main:subagent:late-failure",
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
terminalStatus: "failed" as const,
|
||||
terminalError: "provider failed",
|
||||
expectedError: "provider failed",
|
||||
},
|
||||
{
|
||||
runId: "run-subagent-late-timeout",
|
||||
runtime: "subagent" as const,
|
||||
childSessionKey: "agent:main:subagent:late-timeout",
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
terminalStatus: "timed_out" as const,
|
||||
terminalError: undefined,
|
||||
expectedError: undefined,
|
||||
},
|
||||
{
|
||||
runId: "run-acp-late-success",
|
||||
runtime: "acp" as const,
|
||||
childSessionKey: "agent:main:acp:late-success",
|
||||
error: "Task cancellation requested.",
|
||||
terminalStatus: "succeeded" as const,
|
||||
terminalError: undefined,
|
||||
expectedError: "Task cancellation requested.",
|
||||
},
|
||||
]) {
|
||||
createTaskRecord({
|
||||
runtime: entry.runtime,
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: entry.childSessionKey,
|
||||
runId: entry.runId,
|
||||
task: entry.runId,
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
finalizeTaskRunByRunId({
|
||||
runId: entry.runId,
|
||||
runtime: entry.runtime,
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error: entry.error,
|
||||
});
|
||||
finalizeTaskRunByRunId({
|
||||
runId: entry.runId,
|
||||
runtime: entry.runtime,
|
||||
status: entry.terminalStatus,
|
||||
endedAt: 201,
|
||||
error: entry.terminalError,
|
||||
terminalSummary: "completed",
|
||||
});
|
||||
|
||||
const task = requireTaskByRunId(entry.runId);
|
||||
expect(task.status).toBe(entry.runtime === "acp" ? "cancelled" : entry.terminalStatus);
|
||||
expect(task.error).toBe(entry.expectedError);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps stronger run-scoped terminal states when a late success arrives", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
resetTaskRegistryMemoryForTest();
|
||||
|
|
@ -1223,6 +1339,47 @@ describe("task-registry", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("keeps managed cancellation pending while a child kill is provisional", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
const flow = createManagedTaskFlow({
|
||||
ownerKey: "agent:main:main",
|
||||
controllerId: "tests/task-registry",
|
||||
goal: "Wait for canonical child state",
|
||||
});
|
||||
const task = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: flow.ownerKey,
|
||||
scopeKind: "session",
|
||||
parentFlowId: flow.flowId,
|
||||
childSessionKey: "agent:worker:subagent:provisional-flow",
|
||||
runId: "run-provisional-managed-flow",
|
||||
task: "Resolve cancellation race",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
expect(
|
||||
requestFlowCancel({
|
||||
flowId: flow.flowId,
|
||||
expectedRevision: getTaskFlowById(flow.flowId)!.revision,
|
||||
cancelRequestedAt: 100,
|
||||
}).applied,
|
||||
).toBe(true);
|
||||
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
});
|
||||
|
||||
expect(getTaskFlowById(flow.flowId)).toMatchObject({
|
||||
status: "queued",
|
||||
cancelRequestedAt: 100,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects parent flow links for terminal flows", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
resetTaskRegistryMemoryForTest({ persist: false });
|
||||
|
|
@ -3733,11 +3890,17 @@ describe("task-registry", () => {
|
|||
|
||||
it("cancels subagent-backed tasks through subagent control", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
hoisted.killSubagentRunAdminMock.mockResolvedValue({
|
||||
found: true,
|
||||
killed: true,
|
||||
const silentTask = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:worker:subagent:child",
|
||||
runId: "run-cancel-subagent",
|
||||
task: "Silent projection",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
notifyPolicy: "silent",
|
||||
});
|
||||
|
||||
const task = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
|
|
@ -3752,6 +3915,30 @@ describe("task-registry", () => {
|
|||
status: "running",
|
||||
deliveryStatus: "pending",
|
||||
});
|
||||
const peerTask = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
requesterOrigin: {
|
||||
channel: "notifychat",
|
||||
to: "notifychat:123",
|
||||
},
|
||||
childSessionKey: "agent:worker:subagent:child",
|
||||
runId: "run-cancel-subagent",
|
||||
task: "Peer projection",
|
||||
status: "running",
|
||||
deliveryStatus: "pending",
|
||||
});
|
||||
hoisted.killSubagentRunAdminMock.mockImplementationOnce(async () => {
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
});
|
||||
return { found: true, killed: true };
|
||||
});
|
||||
|
||||
const result = await cancelTaskById({
|
||||
cfg: {} as never,
|
||||
|
|
@ -3772,13 +3959,755 @@ describe("task-registry", () => {
|
|||
status: "cancelled",
|
||||
error: "Cancelled by operator.",
|
||||
});
|
||||
expectRecordFields(getTaskById(peerTask.taskId), {
|
||||
status: "cancelled",
|
||||
error: "Cancelled by operator.",
|
||||
});
|
||||
expectRecordFields(getTaskById(silentTask.taskId), {
|
||||
status: "cancelled",
|
||||
deliveryStatus: "not_applicable",
|
||||
error: "Cancelled by operator.",
|
||||
});
|
||||
await waitForAssertion(() =>
|
||||
expectRecordFields(sentMessageCall(), {
|
||||
channel: "notifychat",
|
||||
to: "notifychat:123",
|
||||
content: "Background task cancelled: Subagent task (run run-canc).",
|
||||
content: "Background task cancellation requested: Subagent task (run run-canc).",
|
||||
}),
|
||||
);
|
||||
expect(hoisted.sendMessageMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
it("promotes a provisional subagent kill that races task cancellation", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
const task = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:worker:subagent:concurrent-kill",
|
||||
runId: "run-subagent-concurrent-kill",
|
||||
task: "Cancel during teardown",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
hoisted.killSubagentRunAdminMock.mockImplementationOnce(async () => {
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
});
|
||||
return { found: true, killed: false };
|
||||
});
|
||||
|
||||
const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId });
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "succeeded",
|
||||
endedAt: 201,
|
||||
terminalSummary: "completed too late",
|
||||
});
|
||||
|
||||
expectRecordFields(result, { found: true, cancelled: true });
|
||||
expectRecordFields(getTaskById(task.taskId), {
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error: "Cancelled by operator.",
|
||||
terminalSummary: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("reconciles an already-provisional kill before making cancellation sticky", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
const task = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:worker:subagent:provisional-completion",
|
||||
runId: "run-subagent-provisional-completion",
|
||||
task: "Finish before explicit cancellation",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
});
|
||||
hoisted.killSubagentRunAdminMock.mockResolvedValueOnce({
|
||||
found: true,
|
||||
killed: false,
|
||||
runId: task.runId!,
|
||||
sessionKey: task.childSessionKey!,
|
||||
cascadeKilled: 0,
|
||||
targetState: {
|
||||
state: "terminal",
|
||||
task: {
|
||||
status: "succeeded",
|
||||
endedAt: 201,
|
||||
terminalSummary: "completed",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId });
|
||||
|
||||
expect(hoisted.killSubagentRunAdminMock).toHaveBeenCalledOnce();
|
||||
expectRecordFields(result, {
|
||||
found: true,
|
||||
cancelled: false,
|
||||
reason: "Subagent completed while cancellation was in progress.",
|
||||
});
|
||||
expectRecordFields(getTaskById(task.taskId), {
|
||||
status: "succeeded",
|
||||
endedAt: 201,
|
||||
error: undefined,
|
||||
terminalSummary: "completed",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves subagent success that completes during cancellation", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
const task = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:worker:subagent:cancel-race",
|
||||
runId: "run-subagent-cancel-race",
|
||||
task: "Finish during cancellation",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
const peerTask = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:worker:subagent:cancel-race",
|
||||
runId: "run-subagent-cancel-race",
|
||||
task: "Peer projection",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
hoisted.killSubagentRunAdminMock.mockImplementationOnce(async () => {
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
});
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "succeeded",
|
||||
endedAt: 201,
|
||||
terminalSummary: "completed",
|
||||
});
|
||||
return { found: true, killed: true };
|
||||
});
|
||||
|
||||
const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId });
|
||||
|
||||
expectRecordFields(result, {
|
||||
found: true,
|
||||
cancelled: false,
|
||||
reason: "Subagent completed while cancellation was in progress.",
|
||||
});
|
||||
expectRecordFields(result.task, {
|
||||
status: "succeeded",
|
||||
error: undefined,
|
||||
terminalSummary: "completed",
|
||||
});
|
||||
expectRecordFields(getTaskById(peerTask.taskId), {
|
||||
status: "succeeded",
|
||||
error: undefined,
|
||||
terminalSummary: "completed",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("does not cancel a lagging task projection after subagent completion wins", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
const task = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:worker:subagent:lagging-projection",
|
||||
runId: "run-subagent-lagging-projection",
|
||||
task: "Finish before task projection",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
hoisted.killSubagentRunAdminMock.mockResolvedValueOnce({
|
||||
found: true,
|
||||
killed: false,
|
||||
runId: task.runId!,
|
||||
sessionKey: task.childSessionKey!,
|
||||
cascadeKilled: 0,
|
||||
targetState: {
|
||||
state: "terminal",
|
||||
task: {
|
||||
status: "succeeded",
|
||||
endedAt: 200,
|
||||
progressSummary: "final answer",
|
||||
terminalSummary: "final answer",
|
||||
terminalOutcome: "blocked",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId });
|
||||
|
||||
expectRecordFields(result, {
|
||||
found: true,
|
||||
cancelled: false,
|
||||
reason: "Subagent completed while cancellation was in progress.",
|
||||
});
|
||||
expectRecordFields(result.task, {
|
||||
status: "succeeded",
|
||||
endedAt: 200,
|
||||
progressSummary: "final answer",
|
||||
terminalSummary: "final answer",
|
||||
terminalOutcome: "blocked",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("reconciles a replacement run cancellation into the original task scope", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
const task = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:worker:subagent:replacement-run",
|
||||
runId: "run-subagent-before-replacement",
|
||||
task: "Cancel after recovery",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
hoisted.killSubagentRunAdminMock.mockResolvedValueOnce({
|
||||
found: true,
|
||||
killed: true,
|
||||
runId: "run-subagent-after-replacement",
|
||||
sessionKey: task.childSessionKey!,
|
||||
cascadeKilled: 0,
|
||||
targetState: {
|
||||
state: "terminal",
|
||||
task: {
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId });
|
||||
|
||||
expectRecordFields(result, { found: true, cancelled: true });
|
||||
expectRecordFields(getTaskById(task.taskId), {
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error: "Cancelled by operator.",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores a stale killed snapshot after canonical completion persists", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
const task = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:worker:subagent:stale-kill-snapshot",
|
||||
runId: "run-subagent-stale-kill-snapshot",
|
||||
task: "Complete while admin kill unwinds",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
hoisted.killSubagentRunAdminMock.mockImplementationOnce(async () => {
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "succeeded",
|
||||
endedAt: 201,
|
||||
terminalSummary: "completed",
|
||||
});
|
||||
return {
|
||||
found: true,
|
||||
killed: false,
|
||||
runId: task.runId!,
|
||||
sessionKey: task.childSessionKey!,
|
||||
cascadeKilled: 0,
|
||||
targetState: {
|
||||
state: "terminal" as const,
|
||||
task: {
|
||||
status: "cancelled" as const,
|
||||
endedAt: 200,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
terminalSummary: null,
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId });
|
||||
|
||||
expectRecordFields(result, {
|
||||
found: true,
|
||||
cancelled: false,
|
||||
reason: "Subagent completed while cancellation was in progress.",
|
||||
});
|
||||
expectRecordFields(getTaskById(task.taskId), {
|
||||
status: "succeeded",
|
||||
endedAt: 201,
|
||||
error: undefined,
|
||||
terminalSummary: "completed",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("promotes an already-killed run projection during task cancellation", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
const task = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:worker:subagent:killed-projection",
|
||||
runId: "run-subagent-killed-projection",
|
||||
task: "Repair and cancel killed projection",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
hoisted.killSubagentRunAdminMock.mockResolvedValueOnce({
|
||||
found: true,
|
||||
killed: false,
|
||||
runId: task.runId!,
|
||||
sessionKey: task.childSessionKey!,
|
||||
cascadeKilled: 0,
|
||||
targetState: {
|
||||
state: "terminal",
|
||||
task: {
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
terminalSummary: null,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId });
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "succeeded",
|
||||
endedAt: 201,
|
||||
terminalSummary: "completed too late",
|
||||
});
|
||||
|
||||
expectRecordFields(result, { found: true, cancelled: true });
|
||||
expectRecordFields(getTaskById(task.taskId), {
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error: "Cancelled by operator.",
|
||||
terminalSummary: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("reports when terminal reconciliation cannot be persisted", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
const store = createInMemoryTaskRegistryStore();
|
||||
configureTaskRegistryRuntime({ store });
|
||||
const task = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:worker:subagent:persist-failure",
|
||||
runId: "run-subagent-persist-failure",
|
||||
task: "Finish before persistence fails",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
configureTaskRegistryRuntime({
|
||||
store: {
|
||||
...store,
|
||||
upsertTaskWithDeliveryState: () => {
|
||||
throw new Error("task store unavailable");
|
||||
},
|
||||
},
|
||||
});
|
||||
hoisted.killSubagentRunAdminMock.mockResolvedValueOnce({
|
||||
found: true,
|
||||
killed: false,
|
||||
runId: task.runId!,
|
||||
sessionKey: task.childSessionKey!,
|
||||
cascadeKilled: 0,
|
||||
targetState: {
|
||||
state: "terminal",
|
||||
task: { status: "succeeded", endedAt: 200, terminalSummary: "done" },
|
||||
},
|
||||
});
|
||||
|
||||
const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId });
|
||||
|
||||
expectRecordFields(result, {
|
||||
found: true,
|
||||
cancelled: false,
|
||||
reason: "Subagent became terminal, but task state reconciliation failed to persist.",
|
||||
});
|
||||
expectRecordFields(result.task, { status: "running" });
|
||||
});
|
||||
});
|
||||
|
||||
it("returns a subagent failure that wins during cancellation", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
const task = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:worker:subagent:failed-race",
|
||||
runId: "run-subagent-failed-race",
|
||||
task: "Fail during cancellation",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
hoisted.killSubagentRunAdminMock.mockImplementationOnce(async () => {
|
||||
return {
|
||||
found: true,
|
||||
killed: false,
|
||||
runId: task.runId!,
|
||||
sessionKey: task.childSessionKey!,
|
||||
cascadeKilled: 0,
|
||||
targetState: {
|
||||
state: "terminal",
|
||||
task: {
|
||||
status: "failed",
|
||||
endedAt: 200,
|
||||
error: "provider failed",
|
||||
progressSummary: "partial work",
|
||||
terminalSummary: null,
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId });
|
||||
|
||||
expectRecordFields(result, {
|
||||
found: true,
|
||||
cancelled: false,
|
||||
reason: "Subagent became failed while cancellation was in progress.",
|
||||
});
|
||||
expectRecordFields(result.task, {
|
||||
status: "failed",
|
||||
endedAt: 200,
|
||||
error: "provider failed",
|
||||
progressSummary: "partial work",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("defers cancellation while canonical subagent completion is still finalizing", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
const task = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:worker:subagent:finalizing-race",
|
||||
runId: "run-subagent-finalizing-race",
|
||||
task: "Capture final result",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
hoisted.killSubagentRunAdminMock.mockResolvedValueOnce({
|
||||
found: true,
|
||||
killed: false,
|
||||
runId: task.runId!,
|
||||
sessionKey: task.childSessionKey!,
|
||||
cascadeKilled: 0,
|
||||
targetState: { state: "finalizing" },
|
||||
});
|
||||
|
||||
const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId });
|
||||
|
||||
expectRecordFields(result, {
|
||||
found: true,
|
||||
cancelled: false,
|
||||
reason: "Subagent completion is still being finalized.",
|
||||
});
|
||||
expectRecordFields(result.task, { status: "running" });
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps subagent cancellation terminal when success arrives after cancellation", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
const task = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:worker:subagent:late-success",
|
||||
runId: "run-subagent-late-success",
|
||||
task: "Finish after cancellation",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
hoisted.killSubagentRunAdminMock.mockImplementationOnce(async () => {
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
});
|
||||
return { found: true, killed: true };
|
||||
});
|
||||
|
||||
const result = await cancelTaskById({
|
||||
cfg: {} as never,
|
||||
taskId: task.taskId,
|
||||
reason: SUBAGENT_KILL_TASK_ERROR,
|
||||
});
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "succeeded",
|
||||
endedAt: 201,
|
||||
terminalSummary: "completed too late",
|
||||
});
|
||||
|
||||
expectRecordFields(result, { found: true, cancelled: true });
|
||||
expectRecordFields(getTaskById(task.taskId), {
|
||||
status: "cancelled",
|
||||
error: "Cancelled by operator.",
|
||||
terminalSummary: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts subagent success that completed before cancellation", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
const task = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:worker:subagent:earlier-success",
|
||||
runId: "run-subagent-earlier-success",
|
||||
task: "Finish before cancellation",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error: "Cancelled by operator.",
|
||||
});
|
||||
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "succeeded",
|
||||
endedAt: 199,
|
||||
terminalSummary: "completed before cancellation",
|
||||
});
|
||||
|
||||
expectRecordFields(getTaskById(task.taskId), {
|
||||
status: "succeeded",
|
||||
endedAt: 199,
|
||||
error: undefined,
|
||||
terminalSummary: "completed before cancellation",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("does not let a repeated kill restore the provisional marker after cancellation", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
const task = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:worker:subagent:repeated-kill",
|
||||
runId: "run-subagent-repeated-kill",
|
||||
task: "Stay cancelled",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
for (const error of [
|
||||
SUBAGENT_KILL_TASK_ERROR,
|
||||
"Cancelled by operator.",
|
||||
SUBAGENT_KILL_TASK_ERROR,
|
||||
]) {
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error,
|
||||
});
|
||||
}
|
||||
|
||||
expectRecordFields(getTaskById(task.taskId), {
|
||||
status: "cancelled",
|
||||
error: "Cancelled by operator.",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("promotes an existing subagent kill marker to operator cancellation", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
const task = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:worker:subagent:already-killed",
|
||||
runId: "run-subagent-already-killed",
|
||||
task: "Promote killed task",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
const peerTask = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
childSessionKey: "agent:worker:subagent:already-killed",
|
||||
runId: "run-subagent-already-killed",
|
||||
task: "Peer projection",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
});
|
||||
hoisted.killSubagentRunAdminMock.mockClear();
|
||||
hoisted.killSubagentRunAdminMock.mockResolvedValueOnce({
|
||||
found: true,
|
||||
killed: false,
|
||||
runId: task.runId!,
|
||||
sessionKey: task.childSessionKey!,
|
||||
cascadeKilled: 0,
|
||||
targetState: {
|
||||
state: "terminal",
|
||||
task: {
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId });
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "succeeded",
|
||||
endedAt: 201,
|
||||
terminalSummary: "completed too late",
|
||||
});
|
||||
|
||||
expect(hoisted.killSubagentRunAdminMock).toHaveBeenCalledOnce();
|
||||
expectRecordFields(result, { found: true, cancelled: true });
|
||||
for (const taskId of [task.taskId, peerTask.taskId]) {
|
||||
expectRecordFields(getTaskById(taskId), {
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error: "Cancelled by operator.",
|
||||
terminalSummary: undefined,
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("suppresses terminal delivery when teardown finalizes a killed task", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
hoisted.sendMessageMock.mockClear();
|
||||
const task = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
requesterOrigin: { channel: "notifychat", to: "notifychat:123" },
|
||||
childSessionKey: "agent:worker:subagent:teardown",
|
||||
runId: "run-subagent-teardown",
|
||||
task: "Stop silently",
|
||||
status: "running",
|
||||
deliveryStatus: "pending",
|
||||
});
|
||||
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
suppressDelivery: true,
|
||||
});
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "cancelled",
|
||||
endedAt: 201,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
});
|
||||
await Promise.resolve();
|
||||
|
||||
expectRecordFields(getTaskById(task.taskId), {
|
||||
status: "cancelled",
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
expect(hoisted.sendMessageMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it("stops a pending terminal notifier when teardown suppresses delivery", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
hoisted.sendMessageMock.mockClear();
|
||||
const task = createTaskRecord({
|
||||
runtime: "subagent",
|
||||
ownerKey: "agent:main:main",
|
||||
scopeKind: "session",
|
||||
requesterOrigin: { channel: "notifychat", to: "notifychat:123" },
|
||||
childSessionKey: "agent:worker:subagent:pending-teardown",
|
||||
runId: "run-subagent-pending-teardown",
|
||||
task: "Stop pending delivery",
|
||||
status: "running",
|
||||
deliveryStatus: "pending",
|
||||
});
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "cancelled",
|
||||
endedAt: 200,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
});
|
||||
|
||||
const pendingDelivery = maybeDeliverTaskTerminalUpdate(task.taskId);
|
||||
finalizeTaskRunByRunId({
|
||||
runId: task.runId!,
|
||||
runtime: "subagent",
|
||||
status: "cancelled",
|
||||
endedAt: 201,
|
||||
error: SUBAGENT_KILL_TASK_ERROR,
|
||||
suppressDelivery: true,
|
||||
});
|
||||
await pendingDelivery;
|
||||
|
||||
expect(hoisted.sendMessageMock).not.toHaveBeenCalled();
|
||||
expectRecordFields(getTaskById(task.taskId), {
|
||||
status: "cancelled",
|
||||
deliveryStatus: "not_applicable",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,9 @@ import { createLazyPromiseLoader } from "../shared/lazy-runtime.js";
|
|||
import { normalizeDeliveryContext } from "../utils/delivery-context.shared.js";
|
||||
import { isDeliverableMessageChannel } from "../utils/message-channel.js";
|
||||
import { cancelActiveCronTaskRun } from "./cron-task-cancel.js";
|
||||
import { SUBAGENT_KILL_TASK_ERROR } from "./detached-task-runtime-contract.js";
|
||||
import { isChildlessNativeSubagentTask } from "./native-subagent-task.js";
|
||||
import { isTaskFlowCancellationPending } from "./task-cancellation-state.js";
|
||||
import {
|
||||
formatTaskBlockedFollowupMessage,
|
||||
formatTaskStateChangeMessage,
|
||||
|
|
@ -496,8 +498,25 @@ function normalizeTaskTerminalOutcome(
|
|||
|
||||
function shouldApplyRunScopedStatusUpdate(params: {
|
||||
currentStatus: TaskStatus;
|
||||
currentRuntime: TaskRuntime;
|
||||
currentChildSessionKey?: string;
|
||||
currentError?: string;
|
||||
currentEndedAt?: number;
|
||||
nextStatus: TaskStatus;
|
||||
nextError?: string;
|
||||
nextEndedAt?: number;
|
||||
}): boolean {
|
||||
if (
|
||||
params.currentRuntime === "subagent" &&
|
||||
params.nextStatus === "cancelled" &&
|
||||
params.nextError === SUBAGENT_KILL_TASK_ERROR &&
|
||||
isTerminalTaskStatus(params.currentStatus) &&
|
||||
!(params.currentStatus === "cancelled" && params.currentError === SUBAGENT_KILL_TASK_ERROR)
|
||||
) {
|
||||
// The kill marker is provisional. It may refresh only its own tombstone;
|
||||
// canonical completion or operator cancellation already won this race.
|
||||
return false;
|
||||
}
|
||||
if (params.currentStatus === params.nextStatus) {
|
||||
return true;
|
||||
}
|
||||
|
|
@ -507,6 +526,26 @@ function shouldApplyRunScopedStatusUpdate(params: {
|
|||
if (!isTerminalTaskStatus(params.nextStatus)) {
|
||||
return false;
|
||||
}
|
||||
// Direct subagent termination is provisional. An operator cancellation is
|
||||
// sticky only against outcomes that completed at or after cancellation.
|
||||
if (
|
||||
params.currentStatus === "cancelled" &&
|
||||
(params.nextStatus === "succeeded" ||
|
||||
params.nextStatus === "failed" ||
|
||||
params.nextStatus === "timed_out")
|
||||
) {
|
||||
const canonicalOutcomePredatesCancellation =
|
||||
params.currentRuntime === "subagent" &&
|
||||
params.currentEndedAt !== undefined &&
|
||||
params.nextEndedAt !== undefined &&
|
||||
params.nextEndedAt < params.currentEndedAt;
|
||||
return (
|
||||
canonicalOutcomePredatesCancellation ||
|
||||
(params.currentRuntime === "subagent" &&
|
||||
Boolean(params.currentChildSessionKey?.trim()) &&
|
||||
params.currentError === SUBAGENT_KILL_TASK_ERROR)
|
||||
);
|
||||
}
|
||||
return params.currentStatus === "succeeded" && params.nextStatus !== "lost";
|
||||
}
|
||||
|
||||
|
|
@ -541,6 +580,18 @@ function mapAgentRunTerminalOutcomeToTaskStatus(
|
|||
}
|
||||
}
|
||||
|
||||
function resolveTaskLifecycleTerminalError(params: {
|
||||
runtime: TaskRuntime;
|
||||
status: TaskStatus;
|
||||
error?: string;
|
||||
}): string | undefined {
|
||||
// A runner abort can race either an accepted task cancellation or a real
|
||||
// completion. Keep it provisional until the task-control owner decides.
|
||||
return params.runtime === "subagent" && params.status === "cancelled"
|
||||
? SUBAGENT_KILL_TASK_ERROR
|
||||
: params.error;
|
||||
}
|
||||
|
||||
function buildTaskLifecycleTerminalOutcome(params: {
|
||||
phase: "end" | "error";
|
||||
data?: Record<string, unknown>;
|
||||
|
|
@ -1044,7 +1095,7 @@ function syncManagedFlowCancellationFromTask(task: TaskRecord): void {
|
|||
) {
|
||||
return;
|
||||
}
|
||||
if (listTasksForFlowId(flowId).some((candidate) => isActiveTaskStatus(candidate.status))) {
|
||||
if (listTasksForFlowId(flowId).some(isTaskFlowCancellationPending)) {
|
||||
return;
|
||||
}
|
||||
const endedAt = task.endedAt ?? task.lastEventAt ?? Date.now();
|
||||
|
|
@ -1073,7 +1124,7 @@ function syncManagedFlowCancellationFromTask(task: TaskRecord): void {
|
|||
) {
|
||||
return;
|
||||
}
|
||||
if (listTasksForFlowId(flowId).some((candidate) => isActiveTaskStatus(candidate.status))) {
|
||||
if (listTasksForFlowId(flowId).some(isTaskFlowCancellationPending)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
|
@ -1360,11 +1411,27 @@ export async function maybeDeliverTaskTerminalUpdate(taskId: string): Promise<Ta
|
|||
if (!latest || !shouldAutoDeliverTaskTerminalUpdate(latest)) {
|
||||
return latest ? cloneTaskRecord(latest) : null;
|
||||
}
|
||||
const preferred = latest.runId
|
||||
? pickPreferredRunIdTask(getPeerTasksForDelivery(latest))
|
||||
: undefined;
|
||||
const peers = latest.runId ? getPeerTasksForDelivery(latest) : [];
|
||||
const isSubagentCancellation = latest.runtime === "subagent" && latest.status === "cancelled";
|
||||
const preferred = pickPreferredRunIdTask(
|
||||
isSubagentCancellation
|
||||
? peers.filter((candidate) => shouldAutoDeliverTaskTerminalUpdate(candidate))
|
||||
: peers,
|
||||
);
|
||||
const peerDeliveryCovered =
|
||||
isSubagentCancellation &&
|
||||
peers.some(
|
||||
(candidate) =>
|
||||
candidate.taskId !== latest.taskId &&
|
||||
(candidate.deliveryStatus === "delivered" ||
|
||||
candidate.deliveryStatus === "session_queued"),
|
||||
);
|
||||
if (
|
||||
shouldSuppressDuplicateTerminalDelivery({ task: latest, preferredTaskId: preferred?.taskId })
|
||||
shouldSuppressDuplicateTerminalDelivery({
|
||||
task: latest,
|
||||
preferredTaskId: preferred?.taskId,
|
||||
peerDeliveryCovered,
|
||||
})
|
||||
) {
|
||||
return updateTask(taskId, {
|
||||
deliveryStatus: "not_applicable",
|
||||
|
|
@ -1413,6 +1480,10 @@ export async function maybeDeliverTaskTerminalUpdate(taskId: string): Promise<Ta
|
|||
}
|
||||
try {
|
||||
const { sendMessage } = await loadTaskRegistryDeliveryRuntime();
|
||||
const beforeSend = tasks.get(taskId);
|
||||
if (!beforeSend || !shouldAutoDeliverTaskTerminalUpdate(beforeSend)) {
|
||||
return beforeSend ? cloneTaskRecord(beforeSend) : null;
|
||||
}
|
||||
const requesterAgentId = parseAgentSessionKey(ownerSessionKey)?.agentId;
|
||||
const idempotencyKey = resolveTaskTerminalIdempotencyKey(latest);
|
||||
await sendMessage({
|
||||
|
|
@ -1429,8 +1500,12 @@ export async function maybeDeliverTaskTerminalUpdate(taskId: string): Promise<Ta
|
|||
idempotencyKey,
|
||||
},
|
||||
});
|
||||
if (latest.terminalOutcome === "blocked") {
|
||||
queueBlockedTaskFollowup(latest);
|
||||
const afterSend = tasks.get(taskId);
|
||||
if (!afterSend || !shouldAutoDeliverTaskTerminalUpdate(afterSend)) {
|
||||
return afterSend ? cloneTaskRecord(afterSend) : null;
|
||||
}
|
||||
if (afterSend.terminalOutcome === "blocked") {
|
||||
queueBlockedTaskFollowup(afterSend);
|
||||
}
|
||||
return updateTask(taskId, {
|
||||
deliveryStatus: "delivered",
|
||||
|
|
@ -1443,10 +1518,14 @@ export async function maybeDeliverTaskTerminalUpdate(taskId: string): Promise<Ta
|
|||
requesterOrigin: owner.requesterOrigin,
|
||||
error,
|
||||
});
|
||||
const beforeFallback = tasks.get(taskId);
|
||||
if (!beforeFallback || !shouldAutoDeliverTaskTerminalUpdate(beforeFallback)) {
|
||||
return beforeFallback ? cloneTaskRecord(beforeFallback) : null;
|
||||
}
|
||||
try {
|
||||
queueTaskSystemEvent(latest, sessionEventText);
|
||||
if (latest.terminalOutcome === "blocked") {
|
||||
queueBlockedTaskFollowup(latest);
|
||||
queueTaskSystemEvent(beforeFallback, sessionEventText);
|
||||
if (beforeFallback.terminalOutcome === "blocked") {
|
||||
queueBlockedTaskFollowup(beforeFallback);
|
||||
}
|
||||
} catch (fallbackError) {
|
||||
log.warn("Failed to queue background task fallback event", {
|
||||
|
|
@ -1661,8 +1740,13 @@ function ensureListener() {
|
|||
});
|
||||
patch.status = mapAgentRunTerminalOutcomeToTaskStatus(terminal);
|
||||
patch.endedAt = terminal.endedAt ?? now;
|
||||
if (terminal.error) {
|
||||
patch.error = terminal.error;
|
||||
const error = resolveTaskLifecycleTerminalError({
|
||||
runtime: current.runtime,
|
||||
status: patch.status,
|
||||
error: terminal.error,
|
||||
});
|
||||
if (error) {
|
||||
patch.error = error;
|
||||
}
|
||||
} else if (phase === "error") {
|
||||
const terminal = buildTaskLifecycleTerminalOutcome({
|
||||
|
|
@ -1673,7 +1757,12 @@ function ensureListener() {
|
|||
});
|
||||
patch.status = mapAgentRunTerminalOutcomeToTaskStatus(terminal);
|
||||
patch.endedAt = terminal.endedAt ?? now;
|
||||
patch.error = terminal.error ?? current.error;
|
||||
patch.error =
|
||||
resolveTaskLifecycleTerminalError({
|
||||
runtime: current.runtime,
|
||||
status: patch.status,
|
||||
error: terminal.error,
|
||||
}) ?? current.error;
|
||||
}
|
||||
} else if (evt.stream === "error") {
|
||||
patch.error = typeof evt.data?.error === "string" ? evt.data.error : current.error;
|
||||
|
|
@ -1861,6 +1950,7 @@ function updateTaskStateByRunId(params: {
|
|||
terminalSummary?: string | null;
|
||||
terminalOutcome?: TaskTerminalOutcome | null;
|
||||
eventSummary?: string | null;
|
||||
suppressDelivery?: boolean;
|
||||
}) {
|
||||
ensureTaskRegistryReady();
|
||||
const matches = getTasksByRunScope(params);
|
||||
|
|
@ -1875,7 +1965,13 @@ function updateTaskStateByRunId(params: {
|
|||
params.status &&
|
||||
!shouldApplyRunScopedStatusUpdate({
|
||||
currentStatus: current.status,
|
||||
currentRuntime: current.runtime,
|
||||
currentChildSessionKey: current.childSessionKey,
|
||||
currentError: current.error,
|
||||
currentEndedAt: current.endedAt,
|
||||
nextStatus,
|
||||
nextError: params.error,
|
||||
nextEndedAt: params.endedAt,
|
||||
})
|
||||
) {
|
||||
continue;
|
||||
|
|
@ -1893,7 +1989,13 @@ function updateTaskStateByRunId(params: {
|
|||
if (params.lastEventAt != null) {
|
||||
patch.lastEventAt = params.lastEventAt;
|
||||
}
|
||||
if (params.error !== undefined) {
|
||||
if (
|
||||
current.status === "cancelled" &&
|
||||
nextStatus !== "cancelled" &&
|
||||
params.error === undefined
|
||||
) {
|
||||
patch.error = undefined;
|
||||
} else if (params.error !== undefined) {
|
||||
patch.error = params.error;
|
||||
}
|
||||
if (params.progressSummary !== undefined) {
|
||||
|
|
@ -1908,6 +2010,11 @@ function updateTaskStateByRunId(params: {
|
|||
terminalOutcome: params.terminalOutcome,
|
||||
});
|
||||
}
|
||||
if (params.suppressDelivery) {
|
||||
// Teardown suppression must survive redundant lifecycle finalizers that
|
||||
// arrive after queues are cleared, or they can repopulate the stopped session.
|
||||
patch.deliveryStatus = "not_applicable";
|
||||
}
|
||||
const eventSummary =
|
||||
normalizeTaskSummary(params.eventSummary) ??
|
||||
(nextStatus === "failed"
|
||||
|
|
@ -1931,8 +2038,10 @@ function updateTaskStateByRunId(params: {
|
|||
const task = updateTask(current.taskId, patch);
|
||||
if (task) {
|
||||
updated.push(task);
|
||||
void maybeDeliverTaskStateChangeUpdate(task.taskId, nextEvent);
|
||||
void maybeDeliverTaskTerminalUpdate(task.taskId);
|
||||
if (!params.suppressDelivery) {
|
||||
void maybeDeliverTaskStateChangeUpdate(task.taskId, nextEvent);
|
||||
void maybeDeliverTaskTerminalUpdate(task.taskId);
|
||||
}
|
||||
}
|
||||
}
|
||||
return updated;
|
||||
|
|
@ -2011,6 +2120,7 @@ export function finalizeTaskRunByRunId(params: {
|
|||
progressSummary?: string | null;
|
||||
terminalSummary?: string | null;
|
||||
terminalOutcome?: TaskTerminalOutcome | null;
|
||||
suppressDelivery?: boolean;
|
||||
}) {
|
||||
return updateTaskStateByRunId({
|
||||
runId: params.runId,
|
||||
|
|
@ -2024,6 +2134,7 @@ export function finalizeTaskRunByRunId(params: {
|
|||
progressSummary: params.progressSummary,
|
||||
terminalSummary: params.terminalSummary,
|
||||
terminalOutcome: params.terminalOutcome,
|
||||
suppressDelivery: params.suppressDelivery,
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -2081,12 +2192,22 @@ export async function cancelTaskById(params: {
|
|||
if (!task) {
|
||||
return { found: false, cancelled: false, reason: "Task not found." };
|
||||
}
|
||||
const requestedReason = params.reason?.trim();
|
||||
const cancellationError =
|
||||
requestedReason && requestedReason !== SUBAGENT_KILL_TASK_ERROR
|
||||
? requestedReason
|
||||
: "Cancelled by operator.";
|
||||
let isProvisionalSubagentKill =
|
||||
task.runtime === "subagent" &&
|
||||
task.status === "cancelled" &&
|
||||
task.error === SUBAGENT_KILL_TASK_ERROR;
|
||||
if (
|
||||
task.status === "succeeded" ||
|
||||
task.status === "failed" ||
|
||||
task.status === "timed_out" ||
|
||||
task.status === "lost" ||
|
||||
task.status === "cancelled"
|
||||
!isProvisionalSubagentKill &&
|
||||
(task.status === "succeeded" ||
|
||||
task.status === "failed" ||
|
||||
task.status === "timed_out" ||
|
||||
task.status === "lost" ||
|
||||
task.status === "cancelled")
|
||||
) {
|
||||
return {
|
||||
found: true,
|
||||
|
|
@ -2097,6 +2218,8 @@ export async function cancelTaskById(params: {
|
|||
}
|
||||
const childSessionKey = task.childSessionKey?.trim();
|
||||
try {
|
||||
// A direct kill is only a provisional terminal projection. Re-read the
|
||||
// owning subagent run before promotion so its canonical completion can win.
|
||||
if (task.runtime !== "cli") {
|
||||
if (task.runtime === "cron") {
|
||||
if (
|
||||
|
|
@ -2146,12 +2269,86 @@ export async function cancelTaskById(params: {
|
|||
cfg: params.cfg,
|
||||
sessionKey: childSessionKey,
|
||||
});
|
||||
if (!result.found || !result.killed) {
|
||||
const current = tasks.get(task.taskId);
|
||||
if (current?.status === "cancelled" && current.error === SUBAGENT_KILL_TASK_ERROR) {
|
||||
isProvisionalSubagentKill = true;
|
||||
}
|
||||
if (current?.status === "succeeded") {
|
||||
return {
|
||||
found: true,
|
||||
cancelled: false,
|
||||
reason: "Subagent completed while cancellation was in progress.",
|
||||
task: cloneTaskRecord(current),
|
||||
};
|
||||
}
|
||||
if (current && isTerminalTaskStatus(current.status) && current.status !== "cancelled") {
|
||||
return {
|
||||
found: true,
|
||||
cancelled: false,
|
||||
reason: `Subagent became ${current.status} while cancellation was in progress.`,
|
||||
task: cloneTaskRecord(current),
|
||||
};
|
||||
}
|
||||
if (current?.status === "cancelled" && !isProvisionalSubagentKill) {
|
||||
return {
|
||||
found: true,
|
||||
cancelled: false,
|
||||
reason: "Subagent was cancelled while cancellation was in progress.",
|
||||
task: cloneTaskRecord(current),
|
||||
};
|
||||
}
|
||||
if (result.found && result.targetState?.state === "terminal") {
|
||||
// A subagent run becomes terminal before its task projection settles.
|
||||
// Reconcile the original task scope: steer/orphan recovery may have
|
||||
// replaced the registry run ID without remapping durable task rows.
|
||||
const taskRunId = task.runId?.trim() || result.runId;
|
||||
const reconciledTasks = finalizeTaskRunByRunId({
|
||||
runId: taskRunId,
|
||||
runtime: "subagent",
|
||||
sessionKey: childSessionKey,
|
||||
...result.targetState.task,
|
||||
});
|
||||
const reconciled = reconciledTasks.find((candidate) => candidate.taskId === task.taskId);
|
||||
if (!reconciled) {
|
||||
return {
|
||||
found: true,
|
||||
cancelled: false,
|
||||
reason: "Subagent became terminal, but task state reconciliation failed to persist.",
|
||||
task: cloneTaskRecord(tasks.get(task.taskId) ?? task),
|
||||
};
|
||||
}
|
||||
if (
|
||||
result.targetState.task.status === "cancelled" &&
|
||||
result.targetState.task.error === SUBAGENT_KILL_TASK_ERROR
|
||||
) {
|
||||
isProvisionalSubagentKill = true;
|
||||
} else {
|
||||
const reason =
|
||||
result.targetState.task.status === "succeeded"
|
||||
? "Subagent completed while cancellation was in progress."
|
||||
: `Subagent became ${result.targetState.task.status} while cancellation was in progress.`;
|
||||
return {
|
||||
found: true,
|
||||
cancelled: false,
|
||||
reason,
|
||||
task: cloneTaskRecord(reconciled),
|
||||
};
|
||||
}
|
||||
}
|
||||
if (result.found && result.targetState?.state === "finalizing") {
|
||||
return {
|
||||
found: true,
|
||||
cancelled: false,
|
||||
reason: "Subagent completion is still being finalized.",
|
||||
task: cloneTaskRecord(current ?? task),
|
||||
};
|
||||
}
|
||||
if ((!result.found || !result.killed) && !isProvisionalSubagentKill) {
|
||||
return {
|
||||
found: true,
|
||||
cancelled: false,
|
||||
reason: result.found ? "Subagent was not running." : "Subagent task not found.",
|
||||
task: cloneTaskRecord(task),
|
||||
task: cloneTaskRecord(current ?? task),
|
||||
};
|
||||
}
|
||||
} else {
|
||||
|
|
@ -2163,24 +2360,25 @@ export async function cancelTaskById(params: {
|
|||
};
|
||||
}
|
||||
}
|
||||
const endedAt = Date.now();
|
||||
const error = params.reason?.trim() || "Cancelled by operator.";
|
||||
const eventAt = Date.now();
|
||||
const current = tasks.get(task.taskId) ?? task;
|
||||
const endedAt = isProvisionalSubagentKill ? (current.endedAt ?? eventAt) : eventAt;
|
||||
const updated =
|
||||
task.runtime === "acp" && task.runId?.trim()
|
||||
(task.runtime === "acp" || task.runtime === "subagent") && task.runId?.trim()
|
||||
? (updateTaskStateByRunId({
|
||||
runId: task.runId,
|
||||
runtime: "acp",
|
||||
runtime: task.runtime,
|
||||
sessionKey: childSessionKey,
|
||||
status: "cancelled",
|
||||
endedAt,
|
||||
lastEventAt: endedAt,
|
||||
error,
|
||||
lastEventAt: eventAt,
|
||||
error: cancellationError,
|
||||
}).find((record) => record.taskId === task.taskId) ?? null)
|
||||
: updateTask(task.taskId, {
|
||||
status: "cancelled",
|
||||
endedAt,
|
||||
lastEventAt: endedAt,
|
||||
error,
|
||||
lastEventAt: eventAt,
|
||||
error: cancellationError,
|
||||
});
|
||||
if (!updated) {
|
||||
return {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue