From 153ee2abbafff02a119e24223ca4a2b59eb2214b Mon Sep 17 00:00:00 2001 From: Masato Hoshino Date: Sun, 5 Jul 2026 18:17:10 +0900 Subject: [PATCH] fix(subagents): killed subagent runs stay running in the task list (#99806) * fix(subagents): reconcile killed task outcomes Co-authored-by: masatohoshino * fix(subagents): retain reconciliation marker narrowing * fix(subagents): honor generations in latest-run views * test(subagents): align reconciliation fixtures * fix(subagents): keep steer recovery best effort * style(tasks): use type-only runtime contract import * refactor(subagents): keep generation ordering leaf-only * fix(subagents): sanitize persisted task owner ids * fix(subagents): preserve failed replay lifecycle reason * fix(agents): clear killed lifecycle timeout grace * fix(tasks): preserve canonical subagent outcomes * fix(agents): continue requester-wide cancellation * fix(agents): fence yield revival races * test(agents): mock detached task lookup * fix: fence subagent cleanup deletion --------- Co-authored-by: Peter Steinberger --- ...s.subagents.sessions-spawn.test-harness.ts | 1 + src/agents/subagent-announce-output.test.ts | 17 + src/agents/subagent-announce-output.ts | 5 +- src/agents/subagent-announce.test.ts | 19 + src/agents/subagent-announce.ts | 3 +- src/agents/subagent-control.test.ts | 489 ++++- src/agents/subagent-control.ts | 208 +- src/agents/subagent-delivery-state.test.ts | 56 + src/agents/subagent-delivery-state.ts | 28 + src/agents/subagent-list.test.ts | 30 +- src/agents/subagent-list.ts | 3 +- src/agents/subagent-registry-completion.ts | 74 + src/agents/subagent-registry-helpers.ts | 41 +- .../subagent-registry-lifecycle.test.ts | 1561 +++++++++++++- src/agents/subagent-registry-lifecycle.ts | 953 +++++++-- src/agents/subagent-registry-maintenance.ts | 5 + src/agents/subagent-registry-queries.test.ts | 30 + src/agents/subagent-registry-queries.ts | 28 +- src/agents/subagent-registry-read.ts | 10 +- src/agents/subagent-registry-run-manager.ts | 392 ++-- .../subagent-registry.archive.e2e.test.ts | 490 ++++- .../subagent-registry.persistence.test.ts | 126 ++ .../subagent-registry.steer-restart.test.ts | 188 +- src/agents/subagent-registry.test.ts | 1861 ++++++++++++++++- src/agents/subagent-registry.ts | 371 +++- src/agents/subagent-registry.types.ts | 19 + src/agents/subagent-run-generation.ts | 45 + src/agents/subagent-run-timeout.test.ts | 10 + src/agents/subagent-run-timeout.ts | 10 + src/auto-reply/reply/abort.test.ts | 75 + src/auto-reply/reply/abort.ts | 20 +- src/tasks/detached-task-runtime-contract.ts | 30 + src/tasks/detached-task-runtime.test.ts | 159 +- src/tasks/detached-task-runtime.ts | 50 + src/tasks/task-cancellation-state.ts | 21 + src/tasks/task-executor-policy.test.ts | 47 + src/tasks/task-executor-policy.ts | 27 +- src/tasks/task-executor.test.ts | 108 + src/tasks/task-executor.ts | 41 +- src/tasks/task-flow-registry.audit.test.ts | 53 +- src/tasks/task-flow-registry.audit.ts | 5 +- .../task-flow-registry.maintenance.test.ts | 52 + src/tasks/task-flow-registry.maintenance.ts | 5 +- src/tasks/task-registry-control.types.ts | 24 +- src/tasks/task-registry.test.ts | 939 ++++++++- src/tasks/task-registry.ts | 262 ++- 46 files changed, 8471 insertions(+), 520 deletions(-) create mode 100644 src/agents/subagent-run-generation.ts create mode 100644 src/tasks/task-cancellation-state.ts diff --git a/src/agents/openclaw-tools.subagents.sessions-spawn.test-harness.ts b/src/agents/openclaw-tools.subagents.sessions-spawn.test-harness.ts index 53b24a4862b..585bae88ce0 100644 --- a/src/agents/openclaw-tools.subagents.sessions-spawn.test-harness.ts +++ b/src/agents/openclaw-tools.subagents.sessions-spawn.test-harness.ts @@ -395,6 +395,7 @@ vi.mock("../tasks/detached-task-runtime.js", () => ({ completeTaskRunByRunId: vi.fn(), createRunningTaskRun: vi.fn(), failTaskRunByRunId: vi.fn(), + findDetachedTaskRun: vi.fn(() => ({ lookup: "available" as const })), setDetachedTaskDeliveryStatusByRunId: vi.fn(), })); diff --git a/src/agents/subagent-announce-output.test.ts b/src/agents/subagent-announce-output.test.ts index 46698c209df..85c859e3959 100644 --- a/src/agents/subagent-announce-output.test.ts +++ b/src/agents/subagent-announce-output.test.ts @@ -6,6 +6,7 @@ import { applySubagentWaitOutcome, buildCompactAnnounceStatsLine, buildChildCompletionFindings, + dedupeLatestChildCompletionRows, readSubagentOutput, } from "./subagent-announce-output.js"; @@ -63,6 +64,22 @@ function sessionsYieldTurn(message = "Waiting for subagent completion.") { ]; } +describe("dedupeLatestChildCompletionRows", () => { + it("prefers the newer generation when child runs share a creation timestamp", () => { + const childSessionKey = "agent:main:subagent:reused"; + const older = { + runId: "run-older", + generation: 1, + childSessionKey, + task: "older", + createdAt: 1_000, + }; + const newer = { ...older, runId: "run-newer", generation: 2, task: "newer" }; + + expect(dedupeLatestChildCompletionRows([older, newer])).toStrictEqual([newer]); + }); +}); + describe("buildCompactAnnounceStatsLine", () => { afterEach(() => { testing.setDepsForTest(); diff --git a/src/agents/subagent-announce-output.ts b/src/agents/subagent-announce-output.ts index d46c533b6fd..4712e8b748a 100644 --- a/src/agents/subagent-announce-output.ts +++ b/src/agents/subagent-announce-output.ts @@ -19,6 +19,7 @@ import { resolveAgentIdFromSessionKey, resolveStorePath, } from "./subagent-announce.runtime.js"; +import { compareSubagentRunGeneration } from "./subagent-run-generation.js"; import { assistantCallsSessionsYield, isSessionsYieldToolResult } from "./subagent-yield-output.js"; import { extractAssistantText, sanitizeTextContent } from "./tools/chat-history-text.js"; import { isAnnounceSkip } from "./tools/sessions-send-tokens.js"; @@ -416,9 +417,11 @@ export function buildChildCompletionFindings( export function dedupeLatestChildCompletionRows( children: Array<{ + runId: string; childSessionKey: string; task: string; label?: string; + generation?: number; createdAt: number; endedAt?: number; frozenResultText?: string | null; @@ -438,7 +441,7 @@ export function dedupeLatestChildCompletionRows( const latestByChildSessionKey = new Map(); for (const child of children) { const existing = latestByChildSessionKey.get(child.childSessionKey); - if (!existing || child.createdAt > existing.createdAt) { + if (!existing || compareSubagentRunGeneration(child, existing) > 0) { latestByChildSessionKey.set(child.childSessionKey, child); } } diff --git a/src/agents/subagent-announce.test.ts b/src/agents/subagent-announce.test.ts index 36cb89d05ff..b0579f689f3 100644 --- a/src/agents/subagent-announce.test.ts +++ b/src/agents/subagent-announce.test.ts @@ -336,6 +336,25 @@ describe("subagent announce seam flow", () => { }); }); + it("skips delete cleanup when the lifecycle owner invalidates the attempt", async () => { + const didAnnounce = await runSubagentAnnounceFlow({ + childSessionKey: "agent:main:subagent:test", + childRunId: "run-invalidated-delete", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "do thing", + timeoutMs: 10, + cleanup: "delete", + waitForCompletion: false, + outcome: { status: "ok" }, + roundOneReply: "ANNOUNCE_SKIP", + onBeforeDeleteChildSession: () => false, + }); + + expect(didAnnounce).toBe(true); + expect(sessionsDeleteSpy).not.toHaveBeenCalled(); + }); + it("warns when ANNOUNCE_SKIP suppresses a cron job completion", async () => { const logSpy = vi.spyOn(defaultRuntime, "log").mockImplementation(() => {}); diff --git a/src/agents/subagent-announce.ts b/src/agents/subagent-announce.ts index a5ec24fa13b..2fe8c744b37 100644 --- a/src/agents/subagent-announce.ts +++ b/src/agents/subagent-announce.ts @@ -261,6 +261,7 @@ export async function runSubagentAnnounceFlow(params: { signal?: AbortSignal; bestEffortDeliver?: boolean; onDeliveryResult?: (delivery: SubagentAnnounceDeliveryResult) => void; + onBeforeDeleteChildSession?: () => boolean; }): Promise { let didAnnounce = false; const expectsCompletionMessage = params.expectsCompletionMessage === true; @@ -619,7 +620,7 @@ export async function runSubagentAnnounceFlow(params: { // Best-effort } } - if (shouldDeleteChildSession) { + if (shouldDeleteChildSession && (params.onBeforeDeleteChildSession?.() ?? true)) { await deleteSubagentSessionForCleanup({ callGateway: subagentAnnounceDeps.callGateway, childSessionKey: params.childSessionKey, diff --git a/src/agents/subagent-control.test.ts b/src/agents/subagent-control.test.ts index 76ba5e82611..b90cc0654c5 100644 --- a/src/agents/subagent-control.test.ts +++ b/src/agents/subagent-control.test.ts @@ -12,6 +12,7 @@ import type { import type { SessionEntry } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { CallGatewayOptions } from "../gateway/call.js"; +import { SUBAGENT_KILL_TASK_ERROR } from "../tasks/detached-task-runtime-contract.js"; import { testing, killAllControlledSubagentRuns, @@ -21,6 +22,10 @@ import { sendControlledSubagentMessage, steerControlledSubagentRun, } from "./subagent-control.js"; +import { + SUBAGENT_ENDED_REASON_COMPLETE, + SUBAGENT_ENDED_REASON_KILLED, +} from "./subagent-lifecycle-events.js"; import { testing as subagentRegistryTesting, addSubagentRunForTests, @@ -32,6 +37,23 @@ vi.mock("../gateway/call.js", () => ({ callGateway: vi.fn(), })); +const detachedTaskRuntimeMocks = vi.hoisted(() => ({ + findDetachedTaskRun: vi.fn(() => ({ lookup: "available" as const })), + finalizeTaskRunByRunId: vi.fn<(_params: unknown) => unknown[]>(() => []), +})); + +vi.mock("../tasks/detached-task-runtime.js", () => ({ + createQueuedTaskRun: vi.fn(() => null), + createRunningTaskRun: vi.fn(() => null), + startTaskRunByRunId: vi.fn(() => []), + recordTaskRunProgressByRunId: vi.fn(() => []), + finalizeTaskRunByRunId: detachedTaskRuntimeMocks.finalizeTaskRunByRunId, + completeTaskRunByRunId: vi.fn(() => []), + failTaskRunByRunId: vi.fn(() => []), + setDetachedTaskDeliveryStatusByRunId: vi.fn(() => []), + findDetachedTaskRun: detachedTaskRuntimeMocks.findDetachedTaskRun, +})); + vi.mock("./run-wait.js", () => { const readLatestAssistantReplySnapshot = async (params: { sessionKey: string; @@ -188,6 +210,7 @@ function writeSessionStoreFixture(label: string, store: Record) } beforeEach(() => { + detachedTaskRuntimeMocks.finalizeTaskRunByRunId.mockClear(); setSubagentControlDepsForTest(); subagentRegistryTesting.setDepsForTest({ cleanupBrowserSessionsForLifecycleEnd: async () => {}, @@ -502,7 +525,6 @@ describe("sendControlledSubagentMessage", () => { role: "assistant", content: [{ type: "text", text: "older reply from a previous run" }], }; - setSubagentControlDepsForTest({ callGateway: async >(request: CallGatewayOptions) => { if (request.method === "chat.history") { @@ -590,9 +612,21 @@ describe("killSubagentRunAdmin", () => { expect(result.found).toBe(true); expect(result.killed).toBe(true); + if (!result.found) { + throw new Error("expected tracked subagent run"); + } expect(result.runId).toBe("run-worker"); expect(result.sessionKey).toBe(childSessionKey); expect(getSubagentRunByChildSessionKey(childSessionKey)?.endedAt).toBeTypeOf("number"); + expect(detachedTaskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalledTimes(1); + expect(detachedTaskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalledWith( + expect.objectContaining({ + runId: "run-worker", + runtime: "subagent", + sessionKey: childSessionKey, + status: "cancelled", + }), + ); }); it("returns found=false when the session key is not tracked as a subagent run", async () => { @@ -604,6 +638,379 @@ describe("killSubagentRunAdmin", () => { expect(result).toEqual({ found: false, killed: false }); }); + it("retries task reconciliation for an already-killed run", async () => { + const childSessionKey = "agent:main:subagent:already-killed"; + const endedAt = Date.now() - 1_000; + addSubagentRunForTests({ + runId: "run-already-killed", + childSessionKey, + controllerSessionKey: "agent:main:controller", + requesterSessionKey: "agent:main:requester", + requesterDisplayKey: "requester", + task: "repair task projection", + cleanup: "keep", + createdAt: endedAt - 4_000, + startedAt: endedAt - 3_000, + endedAt, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "killed" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: endedAt }, + cleanupCompletedAt: endedAt + 500, + }); + + const first = await killSubagentRunAdmin({ cfg: {}, sessionKey: childSessionKey }); + const second = await killSubagentRunAdmin({ cfg: {}, sessionKey: childSessionKey }); + + for (const result of [first, second]) { + expect(result).toMatchObject({ + found: true, + killed: false, + targetState: { + state: "terminal", + task: { + status: "cancelled", + endedAt, + error: SUBAGENT_KILL_TASK_ERROR, + }, + }, + }); + } + expect(detachedTaskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalledTimes(2); + }); + + it("keeps a killed steer-restart run on its failed projection", async () => { + const childSessionKey = "agent:main:subagent:steer-restart"; + const endedAt = Date.now() - 1_000; + addSubagentRunForTests({ + runId: "run-steer-restart", + childSessionKey, + controllerSessionKey: "agent:main:controller", + requesterSessionKey: "agent:main:requester", + requesterDisplayKey: "requester", + task: "replace active run", + cleanup: "keep", + createdAt: endedAt - 4_000, + startedAt: endedAt - 3_000, + endedAt, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + suppressAnnounceReason: "steer-restart", + outcome: { status: "error", error: "agent run aborted" }, + completion: { required: false, resultText: null, capturedAt: endedAt }, + }); + + const result = await killSubagentRunAdmin({ cfg: {}, sessionKey: childSessionKey }); + + expect(result).toMatchObject({ + found: true, + killed: false, + targetState: { + state: "terminal", + task: { + status: "failed", + endedAt, + error: "agent run aborted", + }, + }, + }); + expect(detachedTaskRuntimeMocks.finalizeTaskRunByRunId).not.toHaveBeenCalled(); + }); + + it("restores the recoverable task marker when abort lifecycle wins the race", async () => { + const childSessionKey = "agent:main:subagent:abort-lifecycle-race"; + const storePath = writeSessionStoreFixture("admin-kill-abort-lifecycle-race", { + [childSessionKey]: { + sessionId: "sess-abort-lifecycle-race", + updatedAt: Date.now(), + }, + }); + const run = { + runId: "run-abort-lifecycle-race", + childSessionKey, + controllerSessionKey: "agent:main:controller", + requesterSessionKey: "agent:main:requester", + requesterDisplayKey: "requester", + task: "finish while aborting", + cleanup: "keep" as const, + createdAt: Date.now() - 5_000, + startedAt: Date.now() - 4_000, + }; + const abortedLastRunWrites: boolean[] = []; + addSubagentRunForTests(run); + setSubagentControlDepsForTest({ + isEmbeddedAgentRunActive: () => true, + abortEmbeddedAgentRun: () => { + const endedAt = Date.now(); + Object.assign(run, { + endedAt, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error" as const, error: "agent run aborted" }, + suppressAnnounceReason: "killed" as const, + killReconciliation: { killedAt: endedAt }, + }); + return true; + }, + patchSessionEntry: async (_scope, patcher) => { + const current = { sessionId: "sess-abort-lifecycle-race", updatedAt: Date.now() }; + const patch = await patcher(current, { existingEntry: { ...current } }); + abortedLastRunWrites.push(patch?.abortedLastRun === true); + return patch ? { ...current, ...patch } : current; + }, + }); + + const result = await killSubagentRunAdmin({ + cfg: cfgWithSessionStore(storePath), + sessionKey: childSessionKey, + }); + + expect(result).toMatchObject({ found: true, killed: true }); + expect(detachedTaskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalledWith( + expect.objectContaining({ + runId: "run-abort-lifecycle-race", + status: "cancelled", + error: SUBAGENT_KILL_TASK_ERROR, + }), + ); + expect(abortedLastRunWrites).toEqual([true]); + }); + + it("reports when completion wins while the kill path awaits persistence", async () => { + const childSessionKey = "agent:main:subagent:completion-race"; + const storePath = writeSessionStoreFixture("admin-kill-completion-race", { + [childSessionKey]: { + sessionId: "sess-completion-race", + updatedAt: Date.now(), + }, + }); + const run = { + runId: "run-completion-race", + childSessionKey, + controllerSessionKey: "agent:main:controller", + requesterSessionKey: "agent:main:requester", + requesterDisplayKey: "requester", + task: "finish while cancellation starts", + cleanup: "keep" as const, + createdAt: Date.now() - 5_000, + startedAt: Date.now() - 4_000, + }; + const abortedLastRunWrites: boolean[] = []; + addSubagentRunForTests({ + ...run, + runId: "run-stale-completion-race", + task: "stale older row", + createdAt: Date.now() - 9_000, + startedAt: Date.now() - 8_000, + }); + addSubagentRunForTests(run); + setSubagentControlDepsForTest({ + isEmbeddedAgentRunActive: () => true, + abortEmbeddedAgentRun: () => true, + patchSessionEntry: async (_scope, patcher) => { + const current = { sessionId: "sess-completion-race", updatedAt: Date.now() }; + const patch = await patcher(current, { existingEntry: { ...current } }); + abortedLastRunWrites.push(patch?.abortedLastRun === true); + if (abortedLastRunWrites.length === 1) { + Object.assign(run, { + endedAt: Date.now(), + endedReason: SUBAGENT_ENDED_REASON_COMPLETE, + outcome: { status: "ok" as const }, + completion: { + required: false, + resultText: "done", + capturedAt: Date.now(), + }, + }); + } + return patch ? { ...current, ...patch } : current; + }, + }); + + const result = await killSubagentRunAdmin({ + cfg: cfgWithSessionStore(storePath), + sessionKey: childSessionKey, + }); + + expect(result).toMatchObject({ + found: true, + killed: false, + targetState: { + state: "terminal", + task: { + status: "succeeded", + endedAt: expect.any(Number), + progressSummary: "done", + terminalSummary: null, + }, + }, + runId: "run-completion-race", + }); + expect(abortedLastRunWrites).toEqual([true, false]); + expect(run).toMatchObject({ + endedReason: SUBAGENT_ENDED_REASON_COMPLETE, + outcome: { status: "ok" }, + }); + expect(detachedTaskRuntimeMocks.finalizeTaskRunByRunId).not.toHaveBeenCalled(); + }); + + it("refreshes target completion after descendant cancellation settles", async () => { + const childSessionKey = "agent:main:subagent:cascade-completion-race"; + const descendantSessionKey = "agent:main:subagent:cascade-completion-child"; + const storePath = writeSessionStoreFixture("admin-kill-cascade-completion-race", { + [childSessionKey]: { + sessionId: "sess-cascade-completion-race", + updatedAt: Date.now(), + }, + [descendantSessionKey]: { + sessionId: "sess-cascade-completion-child", + updatedAt: Date.now(), + }, + }); + const run = { + runId: "run-cascade-completion-race", + childSessionKey, + controllerSessionKey: "agent:main:controller", + requesterSessionKey: "agent:main:requester", + requesterDisplayKey: "requester", + task: "finish while descendant cancellation settles", + cleanup: "keep" as const, + createdAt: Date.now() - 5_000, + startedAt: Date.now() - 4_000, + }; + const abortedLastRunWrites: boolean[] = []; + addSubagentRunForTests(run); + addSubagentRunForTests({ + runId: "run-cascade-completion-child", + childSessionKey: descendantSessionKey, + controllerSessionKey: childSessionKey, + requesterSessionKey: childSessionKey, + requesterDisplayKey: "parent", + task: "descendant", + cleanup: "keep", + createdAt: Date.now() - 3_000, + startedAt: Date.now() - 2_000, + }); + setSubagentControlDepsForTest({ + isEmbeddedAgentRunActive: () => true, + abortEmbeddedAgentRun: () => true, + patchSessionEntry: async (scope, patcher) => { + const current = { sessionId: `sess-${scope.sessionKey}`, updatedAt: Date.now() }; + const patch = await patcher(current, { existingEntry: { ...current } }); + if (scope.sessionKey === childSessionKey) { + abortedLastRunWrites.push(patch?.abortedLastRun === true); + } + if (scope.sessionKey === descendantSessionKey) { + const endedAt = Date.now(); + Object.assign(run, { + endedAt, + endedReason: SUBAGENT_ENDED_REASON_COMPLETE, + outcome: { status: "ok" as const }, + completion: { required: false, resultText: "done", capturedAt: endedAt }, + }); + } + return patch ? { ...current, ...patch } : current; + }, + }); + + const result = await killSubagentRunAdmin({ + cfg: cfgWithSessionStore(storePath), + sessionKey: childSessionKey, + }); + + expect(result).toMatchObject({ + found: true, + killed: true, + targetState: { + state: "terminal", + task: { + status: "succeeded", + progressSummary: "done", + }, + }, + }); + expect(abortedLastRunWrites).toEqual([true, false]); + }); + + it("kills a run that yields while the kill path awaits persistence", async () => { + const childSessionKey = "agent:main:subagent:yield-race"; + const storePath = writeSessionStoreFixture("admin-kill-yield-race", { + [childSessionKey]: { + sessionId: "sess-yield-race", + updatedAt: Date.now(), + }, + }); + const run = { + runId: "run-yield-race", + childSessionKey, + controllerSessionKey: "agent:main:controller", + requesterSessionKey: "agent:main:requester", + requesterDisplayKey: "requester", + task: "yield while cancellation starts", + cleanup: "keep" as const, + createdAt: Date.now() - 5_000, + startedAt: Date.now() - 4_000, + }; + const yieldedAt = Date.now() - 1_000; + addSubagentRunForTests(run); + setSubagentControlDepsForTest({ + isEmbeddedAgentRunActive: () => true, + abortEmbeddedAgentRun: () => true, + patchSessionEntry: async () => { + Object.assign(run, { + endedAt: yieldedAt, + pauseReason: "sessions_yield" as const, + }); + return null; + }, + }); + + const result = await killSubagentRunAdmin({ + cfg: cfgWithSessionStore(storePath), + sessionKey: childSessionKey, + }); + + expect(result).toMatchObject({ + found: true, + killed: true, + runId: "run-yield-race", + targetState: { + state: "terminal", + task: { status: "cancelled", error: SUBAGENT_KILL_TASK_ERROR }, + }, + }); + expect(getSubagentRunByChildSessionKey(childSessionKey)).toMatchObject({ + endedReason: SUBAGENT_ENDED_REASON_KILLED, + endedAt: yieldedAt, + outcome: { + status: "error", + endedAt: yieldedAt, + elapsedMs: yieldedAt - run.startedAt, + }, + }); + expect(getSubagentRunByChildSessionKey(childSessionKey)?.pauseReason).toBeUndefined(); + expect(detachedTaskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalledWith( + expect.objectContaining({ runId: "run-yield-race", status: "cancelled" }), + ); + const [finalizeArgs] = detachedTaskRuntimeMocks.finalizeTaskRunByRunId.mock.calls[0] ?? []; + const killedAt = (finalizeArgs as { endedAt?: number } | undefined)?.endedAt; + expect(killedAt).toBeGreaterThan(yieldedAt); + + const repeated = await killSubagentRunAdmin({ + cfg: cfgWithSessionStore(storePath), + sessionKey: childSessionKey, + }); + expect(repeated).toMatchObject({ + found: true, + killed: false, + targetState: { + state: "terminal", + task: { status: "cancelled", endedAt: killedAt }, + }, + }); + const [repeatedFinalizeArgs] = + detachedTaskRuntimeMocks.finalizeTaskRunByRunId.mock.calls.at(-1) ?? []; + expect((repeatedFinalizeArgs as { endedAt?: number } | undefined)?.endedAt).toBe(killedAt); + }); + it("does not mark a finalizing run killed when its abort is rejected", async () => { const childSessionKey = "agent:main:subagent:worker-finalizing"; const storePath = writeSessionStoreFixture("admin-kill-finalizing", { @@ -644,7 +1051,7 @@ describe("killSubagentRunAdmin", () => { expect(persisted[childSessionKey]?.abortedLastRun).toBeUndefined(); }); - it("does not kill a newest finished run when only a stale older row is still active", async () => { + it("does not kill a newest finalizing run when only a stale older row is still active", async () => { const childSessionKey = "agent:main:subagent:worker-stale-admin"; addSubagentRunForTests({ @@ -679,6 +1086,10 @@ describe("killSubagentRunAdmin", () => { expect(result.found).toBe(true); expect(result.killed).toBe(false); + if (!result.found) { + throw new Error("expected finalizing subagent run"); + } + expect(result.targetState).toEqual({ state: "finalizing" }); expect(result.runId).toBe("run-current-admin"); expect(result.sessionKey).toBe(childSessionKey); }); @@ -717,6 +1128,9 @@ describe("killSubagentRunAdmin", () => { expect(result.found).toBe(true); expect(result.killed).toBe(true); + if (!result.found) { + throw new Error("expected tracked subagent run"); + } expect(result.runId).toBe("run-worker-store-fail"); expect(result.sessionKey).toBe(childSessionKey); expect(getSubagentRunByChildSessionKey(childSessionKey)?.endedAt).toBeTypeOf("number"); @@ -785,7 +1199,7 @@ describe("killControlledSubagentRun", () => { expect(getSubagentRunByChildSessionKey(childSessionKey)?.runId).toBe("run-current"); }); - it("does not kill a stale child row while cascading descendants from an ended current parent", async () => { + it("kills a yielded descendant without reviving a stale child row", async () => { const parentSessionKey = "agent:main:subagent:kill-parent"; const childSessionKey = `${parentSessionKey}:subagent:child`; const leafSessionKey = `${childSessionKey}:subagent:leaf`; @@ -837,6 +1251,8 @@ describe("killControlledSubagentRun", () => { cleanup: "keep", createdAt: Date.now() - 1_000, startedAt: Date.now() - 900, + endedAt: Date.now() - 800, + pauseReason: "sessions_yield", }); const result = await killControlledSubagentRun({ @@ -980,6 +1396,67 @@ describe("killAllControlledSubagentRuns", () => { testing.setDepsForTest(); }); + it("continues bulk cancellation after one registry persistence failure", async () => { + let persistenceAttempts = 0; + subagentRegistryTesting.setDepsForTest({ + cleanupBrowserSessionsForLifecycleEnd: async () => {}, + ensureContextEnginesInitialized: () => {}, + ensureRuntimePluginsLoaded: () => {}, + getSubagentRunsSnapshotForRead: (runs) => new Map(runs), + persistSubagentRunsToDisk: () => {}, + persistSubagentRunsToDiskOrThrow: () => { + persistenceAttempts += 1; + if (persistenceAttempts === 1) { + throw new Error("sqlite busy"); + } + }, + restoreSubagentRunsFromDisk: () => 0, + resolveContextEngine: async () => ({ + info: { id: "test", name: "Test" }, + assemble: async ({ messages }) => ({ messages, estimatedTokens: 0 }), + compact: async () => ({ ok: true, compacted: false }), + ingest: async () => ({ ingested: false }), + }), + }); + const first = { + runId: "run-bulk-persistence-failure-first", + childSessionKey: "agent:main:subagent:bulk-persistence-failure-first", + controllerSessionKey: "agent:main:main", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "first bulk task", + cleanup: "keep" as const, + createdAt: Date.now() - 2_000, + startedAt: Date.now() - 1_900, + }; + const second = { + ...first, + runId: "run-bulk-persistence-failure-second", + childSessionKey: "agent:main:subagent:bulk-persistence-failure-second", + task: "second bulk task", + createdAt: Date.now() - 1_000, + startedAt: Date.now() - 900, + }; + addSubagentRunForTests(first); + addSubagentRunForTests(second); + + const result = await killAllControlledSubagentRuns({ + cfg: cfgWithSessionStore(), + controller: { + controllerSessionKey: "agent:main:main", + callerSessionKey: "agent:main:main", + callerIsSubagent: false, + controlScope: "children", + }, + runs: [first, second], + }); + + expect(result).toEqual({ status: "ok", killed: 1, labels: ["second bulk task"] }); + expect(persistenceAttempts).toBe(2); + expect(getSubagentRunByChildSessionKey(first.childSessionKey)?.endedAt).toBeUndefined(); + expect(getSubagentRunByChildSessionKey(second.childSessionKey)?.endedAt).toBeTypeOf("number"); + }); + it("ignores stale run snapshots in bulk kill requests", async () => { const childSessionKey = "agent:main:subagent:stale-kill-all-worker"; const storePath = writeSessionStoreFixture("stale-kill-all", { @@ -1036,7 +1513,7 @@ describe("killAllControlledSubagentRuns", () => { expect(getSubagentRunByChildSessionKey(childSessionKey)?.runId).toBe("run-current-bulk"); }); - it("does not let a stale bulk entry suppress the current live entry for the same child key", async () => { + it("does not let a stale bulk entry suppress the current yielded entry", async () => { const childSessionKey = "agent:main:subagent:stale-kill-all-shadow-worker"; const storePath = writeSessionStoreFixture("stale-kill-all-shadow", { [childSessionKey]: { @@ -1054,6 +1531,8 @@ describe("killAllControlledSubagentRuns", () => { cleanup: "keep", createdAt: Date.now() - 4_000, startedAt: Date.now() - 3_000, + endedAt: Date.now() - 2_000, + pauseReason: "sessions_yield", }); const result = await killAllControlledSubagentRuns({ @@ -1086,6 +1565,8 @@ describe("killAllControlledSubagentRuns", () => { cleanup: "keep", createdAt: Date.now() - 4_000, startedAt: Date.now() - 3_000, + endedAt: Date.now() - 2_000, + pauseReason: "sessions_yield", }, ], }); diff --git a/src/agents/subagent-control.ts b/src/agents/subagent-control.ts index 3885fce8bc1..4e82ee969e0 100644 --- a/src/agents/subagent-control.ts +++ b/src/agents/subagent-control.ts @@ -15,6 +15,10 @@ import { logVerbose } from "../globals.js"; import { formatErrorMessage } from "../infra/errors.js"; import { isSubagentSessionKey, parseAgentSessionKey } from "../routing/session-key.js"; import { createLazyImportLoader } from "../shared/lazy-promise.js"; +import { + SUBAGENT_KILL_TASK_ERROR, + type DetachedTaskTerminalState, +} from "../tasks/detached-task-runtime-contract.js"; import { INTERNAL_MESSAGE_CHANNEL } from "../utils/message-channel.js"; import { AGENT_LANE_SUBAGENT } from "./lanes.js"; import { @@ -22,7 +26,12 @@ import { waitForAgentRunAndReadUpdatedAssistantReply, } from "./run-wait.js"; import { resolveStoredSubagentCapabilities } from "./subagent-capabilities.js"; +import { SUBAGENT_ENDED_REASON_KILLED } from "./subagent-lifecycle-events.js"; import { buildLatestSubagentRunIndex, resolveSessionEntryForKey } from "./subagent-list.js"; +import { + resolveFinalizedSubagentTaskState, + resolveKilledSubagentTaskEndedAt, +} from "./subagent-registry-completion.js"; import { subagentRuns } from "./subagent-registry-memory.js"; import { getLatestSubagentRunByChildSessionKey, @@ -178,12 +187,109 @@ function isFinishedForSteerControl(entry: SubagentRunRecord, hasPendingDescendan return Boolean(entry.endedAt) && entry.pauseReason !== "sessions_yield" && !hasPendingDescendants; } +type SubagentKillTargetState = + | { state: "finalizing" } + | { state: "terminal"; task: DetachedTaskTerminalState }; + +function resolveSubagentKillTargetState( + entry: SubagentRunRecord, +): SubagentKillTargetState | undefined { + if ( + entry.endedReason === SUBAGENT_ENDED_REASON_KILLED && + entry.suppressAnnounceReason !== "steer-restart" + ) { + const taskEndedAt = resolveKilledSubagentTaskEndedAt(entry); + return typeof taskEndedAt === "number" + ? { + state: "terminal", + task: { + status: "cancelled", + endedAt: taskEndedAt, + lastEventAt: taskEndedAt, + error: SUBAGENT_KILL_TASK_ERROR, + progressSummary: entry.completion?.resultText ?? undefined, + terminalSummary: null, + }, + } + : undefined; + } + const terminal = resolveFinalizedSubagentTaskState(entry); + if (terminal) { + return { state: "terminal", task: terminal }; + } + return typeof entry.endedAt === "number" && + entry.pauseReason !== "sessions_yield" && + (entry.endedReason !== SUBAGENT_ENDED_REASON_KILLED || + entry.suppressAnnounceReason === "steer-restart") + ? { state: "finalizing" } + : undefined; +} + +async function persistSubagentAbortedLastRun(params: { + childSessionKey: string; + storePath: string; + hasSessionEntry: boolean; + abortedLastRun: boolean; +}): Promise { + if (!params.hasSessionEntry) { + return; + } + try { + await subagentControlDeps.patchSessionEntry( + { storePath: params.storePath, sessionKey: params.childSessionKey }, + (current) => ({ + ...current, + abortedLastRun: params.abortedLastRun, + updatedAt: Date.now(), + }), + { replaceEntry: true }, + ); + } catch (error) { + logVerbose( + `subagents control kill: failed to persist abortedLastRun=${params.abortedLastRun} for ${params.childSessionKey}: ${formatErrorMessage(error)}`, + ); + } +} + +function markSubagentRunTerminatedBestEffort( + params: Parameters[0], +): number { + try { + return markSubagentRunTerminated(params); + } catch (error) { + // The registry transition rolled back atomically. Keep multi-run control + // moving so one persistence failure cannot leave siblings running. + logVerbose( + `subagents control kill: failed to persist ${params.runId ?? params.childSessionKey ?? "unknown"}: ${formatErrorMessage(error)}`, + ); + return 0; + } +} + async function killSubagentRun(params: { cfg: OpenClawConfig; entry: SubagentRunRecord; cache: Map>; -}): Promise<{ killed: boolean; sessionId?: string }> { - if (params.entry.endedAt) { +}): Promise<{ + killed: boolean; + sessionId?: string; + targetState?: SubagentKillTargetState; +}> { + const initialTargetState = resolveSubagentKillTargetState(params.entry); + if (initialTargetState) { + if ( + params.entry.endedReason === SUBAGENT_ENDED_REASON_KILLED && + params.entry.suppressAnnounceReason !== "steer-restart" + ) { + markSubagentRunTerminatedBestEffort({ + runId: params.entry.runId, + childSessionKey: params.entry.childSessionKey, + reason: "killed", + }); + } + return { killed: false, targetState: initialTargetState }; + } + if (params.entry.endedAt && params.entry.pauseReason !== "sessions_yield") { return { killed: false }; } const childSessionKey = params.entry.childSessionKey; @@ -194,6 +300,20 @@ async function killSubagentRun(params: { }); const sessionId = resolved.entry?.sessionId; const runtime = await resolveSubagentControlRuntime(); + const targetStateAfterRuntimeLoad = resolveSubagentKillTargetState(params.entry); + if (targetStateAfterRuntimeLoad) { + if ( + params.entry.endedReason === SUBAGENT_ENDED_REASON_KILLED && + params.entry.suppressAnnounceReason !== "steer-restart" + ) { + markSubagentRunTerminatedBestEffort({ + runId: params.entry.runId, + childSessionKey, + reason: "killed", + }); + } + return { killed: false, sessionId, targetState: targetStateAfterRuntimeLoad }; + } const active = sessionId ? runtime.isEmbeddedAgentRunActive(sessionId) : false; const aborted = sessionId ? runtime.abortEmbeddedAgentRun(sessionId) : false; const cleared = runtime.clearSessionQueues([childSessionKey, sessionId]); @@ -205,30 +325,43 @@ async function killSubagentRun(params: { if (active && !aborted) { return { killed: false, sessionId }; } - if (resolved.entry) { - try { - await subagentControlDeps.patchSessionEntry( - { storePath: resolved.storePath, sessionKey: childSessionKey }, - (current) => ({ - ...current, - abortedLastRun: true, - updatedAt: Date.now(), - }), - { replaceEntry: true }, - ); - } catch (error) { - logVerbose( - `subagents control kill: failed to persist abortedLastRun for ${childSessionKey}: ${formatErrorMessage(error)}`, - ); + const persistAbortedLastRun = (abortedLastRun: boolean) => + persistSubagentAbortedLastRun({ + childSessionKey, + storePath: resolved.storePath, + hasSessionEntry: resolved.entry !== undefined, + abortedLastRun, + }); + await persistAbortedLastRun(true); + const targetState = resolveSubagentKillTargetState(params.entry); + if (targetState) { + const killedTarget = + targetState.state === "terminal" && + targetState.task.status === "cancelled" && + targetState.task.error === SUBAGENT_KILL_TASK_ERROR; + if (killedTarget) { + markSubagentRunTerminatedBestEffort({ + runId: params.entry.runId, + childSessionKey, + reason: "killed", + }); + } else { + await persistAbortedLastRun(false); } + const killed = + killedTarget && (aborted || cleared.followupCleared > 0 || cleared.laneCleared > 0); + return { killed, sessionId, targetState }; } - const marked = markSubagentRunTerminated({ + const marked = markSubagentRunTerminatedBestEffort({ runId: params.entry.runId, childSessionKey, reason: "killed", }); const killed = marked > 0 || aborted || cleared.followupCleared > 0 || cleared.laneCleared > 0; - return { killed, sessionId }; + return { + killed, + sessionId, + }; } async function cascadeKillChildren(params: { @@ -253,10 +386,7 @@ async function cascadeKillChildren(params: { ) { continue; } - const existing = childRunsBySessionKey.get(childKey); - if (!existing || run.createdAt >= existing.createdAt) { - childRunsBySessionKey.set(childKey, run); - } + childRunsBySessionKey.set(childKey, run); } const childRuns = Array.from(childRunsBySessionKey.values()); const seenChildSessionKeys = params.seenChildSessionKeys ?? new Set(); @@ -270,7 +400,7 @@ async function cascadeKillChildren(params: { } seenChildSessionKeys.add(childKey); - if (!run.endedAt) { + if (!run.endedAt || run.pauseReason === "sessions_yield") { const stopResult = await killSubagentRun({ cfg: params.cfg, entry: run, @@ -324,7 +454,7 @@ export async function killAllControlledSubagentRuns(params: { } seenChildSessionKeys.add(childKey); - if (!currentEntry.endedAt) { + if (!currentEntry.endedAt || currentEntry.pauseReason === "sessions_yield") { const stopResult = await killSubagentRun({ cfg: params.cfg, entry: currentEntry, cache }); if (stopResult.killed) { killed += 1; @@ -445,10 +575,38 @@ export async function killSubagentRunAdmin(params: { cfg: OpenClawConfig; sessio cache: killCache, seenChildSessionKeys, }); + // Descendant cleanup can yield long enough for the target run to finish. + // Return the freshest registry state so task cancellation cannot make a stale kill sticky. + const targetState = resolveSubagentKillTargetState(entry) ?? stopResult.targetState; + const killedTarget = + targetState?.state === "terminal" && + targetState.task.status === "cancelled" && + targetState.task.error === SUBAGENT_KILL_TASK_ERROR; + const stopResultAlreadyClearedAbort = + stopResult.targetState !== undefined && + !( + stopResult.targetState.state === "terminal" && + stopResult.targetState.task.status === "cancelled" && + stopResult.targetState.task.error === SUBAGENT_KILL_TASK_ERROR + ); + if (targetState && !killedTarget && !stopResultAlreadyClearedAbort) { + const resolved = resolveSessionEntryForKey({ + cfg: params.cfg, + key: targetSessionKey, + cache: killCache, + }); + await persistSubagentAbortedLastRun({ + childSessionKey: targetSessionKey, + storePath: resolved.storePath, + hasSessionEntry: resolved.entry !== undefined, + abortedLastRun: false, + }); + } return { found: true as const, killed: stopResult.killed || cascade.killed > 0, + ...(targetState ? { targetState } : {}), runId: entry.runId, sessionKey: entry.childSessionKey, cascadeKilled: cascade.killed, diff --git a/src/agents/subagent-delivery-state.test.ts b/src/agents/subagent-delivery-state.test.ts index 83fc58a2e73..8df09a226c7 100644 --- a/src/agents/subagent-delivery-state.test.ts +++ b/src/agents/subagent-delivery-state.test.ts @@ -23,6 +23,62 @@ function baseRun(overrides: Partial = {}): LegacySubage } describe("normalizeSubagentRunState", () => { + it("normalizes durable task ownership and generation metadata", () => { + const entry = normalizeSubagentRunState( + baseRun({ taskRunId: " run-task-owner ", generation: 2 }), + ); + const malformed = normalizeSubagentRunState( + baseRun({ taskRunId: " ", generation: Number.NaN }), + ); + const nonString = normalizeSubagentRunState({ + ...baseRun(), + taskRunId: 42, + } as unknown as SubagentRunRecord); + + expect(entry).toMatchObject({ taskRunId: "run-task-owner", generation: 2 }); + expect(malformed.taskRunId).toBeUndefined(); + expect(malformed.generation).toBeUndefined(); + expect(nonString.taskRunId).toBeUndefined(); + }); + + it("normalizes the durable delete-dispatch boundary", () => { + const valid = normalizeSubagentRunState(baseRun({ deleteCleanupDispatchedAt: 200 })); + const malformed = normalizeSubagentRunState(baseRun({ deleteCleanupDispatchedAt: Number.NaN })); + + expect(valid.deleteCleanupDispatchedAt).toBe(200); + expect(malformed.deleteCleanupDispatchedAt).toBeUndefined(); + }); + + it("preserves valid killed reconciliation ownership metadata", () => { + const entry = normalizeSubagentRunState( + baseRun({ + suppressCompletionDelivery: true, + killReconciliation: { + killedAt: 200, + suppressTaskDelivery: true, + supersededAt: 300, + }, + }), + ); + + expect(entry.killReconciliation).toEqual({ + killedAt: 200, + suppressTaskDelivery: true, + supersededAt: 300, + }); + expect(entry.suppressCompletionDelivery).toBe(true); + }); + + it("drops malformed killed reconciliation metadata", () => { + const entry = normalizeSubagentRunState( + baseRun({ + killReconciliation: { killedAt: Number.NaN }, + }), + ); + + expect(entry.killReconciliation).toBeUndefined(); + }); + it("migrates legacy pending delivery fields into nested completion and delivery state", () => { // Restored runs may still carry flat pendingFinalDelivery fields from older // builds; normalization must preserve retry payloads before stripping them. diff --git a/src/agents/subagent-delivery-state.ts b/src/agents/subagent-delivery-state.ts index 015be92a90a..1c366fac43c 100644 --- a/src/agents/subagent-delivery-state.ts +++ b/src/agents/subagent-delivery-state.ts @@ -48,6 +48,34 @@ type LegacyDeliveryState = SubagentCompletionDeliveryState & { /** Normalizes legacy subagent run fields into nested execution/completion/delivery state. */ export function normalizeSubagentRunState(entry: SubagentRunRecord): SubagentRunRecord { const legacy = entry as LegacySubagentRunRecord; + const taskRunId = typeof entry.taskRunId === "string" ? entry.taskRunId.trim() : ""; + entry.taskRunId = taskRunId || undefined; + entry.generation = + typeof entry.generation === "number" && + Number.isSafeInteger(entry.generation) && + entry.generation > 0 + ? entry.generation + : undefined; + entry.deleteCleanupDispatchedAt = Number.isFinite(entry.deleteCleanupDispatchedAt) + ? entry.deleteCleanupDispatchedAt + : undefined; + entry.suppressCompletionDelivery = entry.suppressCompletionDelivery === true ? true : undefined; + const killReconciliation = entry.killReconciliation; + if ( + !killReconciliation || + typeof killReconciliation !== "object" || + !Number.isFinite(killReconciliation.killedAt) + ) { + delete entry.killReconciliation; + } else { + entry.killReconciliation = { + killedAt: killReconciliation.killedAt, + suppressTaskDelivery: killReconciliation.suppressTaskDelivery === true ? true : undefined, + supersededAt: Number.isFinite(killReconciliation.supersededAt) + ? killReconciliation.supersededAt + : undefined, + }; + } entry.execution = mergeExecutionState(entry.execution, buildExecutionState(entry)); entry.completion = mergeCompletionState(entry.completion, buildCompletionState(entry, legacy)); entry.delivery = mergeDeliveryState(entry, entry.delivery, buildDeliveryState(entry, legacy)); diff --git a/src/agents/subagent-list.test.ts b/src/agents/subagent-list.test.ts index eedc91cbbf0..61cead965e0 100644 --- a/src/agents/subagent-list.test.ts +++ b/src/agents/subagent-list.test.ts @@ -6,7 +6,7 @@ import path from "node:path"; import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; import { updateSessionStore } from "../config/sessions/store.js"; -import { buildSubagentList } from "./subagent-list.js"; +import { buildLatestSubagentRunIndex, buildSubagentList } from "./subagent-list.js"; import { addSubagentRunForTests, resetSubagentRegistryForTests, @@ -34,6 +34,34 @@ beforeEach(() => { resetSubagentRegistryForTests(); }); +describe("buildLatestSubagentRunIndex", () => { + it("prefers the newer generation when runs share a creation timestamp", () => { + const childSessionKey = "agent:main:subagent:reused"; + const makeRun = (runId: string, generation: number): SubagentRunRecord => ({ + runId, + generation, + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: runId, + cleanup: "keep", + createdAt: 1_000, + startedAt: 1_000, + }); + const older = makeRun("run-older", 1); + const newer = makeRun("run-newer", 2); + + const index = buildLatestSubagentRunIndex( + new Map([ + [older.runId, older], + [newer.runId, newer], + ]), + ); + + expect(index.latestByChildSessionKey.get(childSessionKey)).toBe(newer); + }); +}); + describe("buildSubagentList", () => { it("returns empty active and recent sections when no runs exist", () => { const cfg = { diff --git a/src/agents/subagent-list.ts b/src/agents/subagent-list.ts index b05ad61c518..32bfa2764f4 100644 --- a/src/agents/subagent-list.ts +++ b/src/agents/subagent-list.ts @@ -28,6 +28,7 @@ import { } from "./subagent-registry-read.js"; import { getSubagentRunsSnapshotForRead } from "./subagent-registry-state.js"; import type { SubagentRunRecord } from "./subagent-registry.types.js"; +import { compareSubagentRunGeneration } from "./subagent-run-generation.js"; import { hasSubagentRunEnded, isLiveUnendedSubagentRun, @@ -103,7 +104,7 @@ export function buildLatestSubagentRunIndex( continue; } const existing = latestByChildSessionKey.get(childSessionKey); - if (!existing || entry.createdAt > existing.createdAt) { + if (!existing || compareSubagentRunGeneration(entry, existing) > 0) { latestByChildSessionKey.set(childSessionKey, entry); } } diff --git a/src/agents/subagent-registry-completion.ts b/src/agents/subagent-registry-completion.ts index 30d1b43fd76..6116bef63a5 100644 --- a/src/agents/subagent-registry-completion.ts +++ b/src/agents/subagent-registry-completion.ts @@ -5,8 +5,14 @@ */ import { createSubsystemLogger } from "../logging/subsystem.js"; import { getGlobalHookRunner } from "../plugins/hook-runner-global.js"; +import { + SUBAGENT_KILL_TASK_ERROR, + type DetachedTaskTerminalState, +} from "../tasks/detached-task-runtime-contract.js"; +import { resolveRequiredCompletionTerminalResult } from "../tasks/task-completion-contract.js"; import type { SubagentRunOutcome } from "./subagent-announce-output.js"; import { + SUBAGENT_ENDED_REASON_KILLED, SUBAGENT_ENDED_OUTCOME_ERROR, SUBAGENT_ENDED_OUTCOME_OK, SUBAGENT_ENDED_OUTCOME_TIMEOUT, @@ -62,6 +68,74 @@ export function shouldUpdateRunOutcome( ); } +/** Returns the complete task projection only after completion capture has settled. */ +export function resolveFinalizedSubagentTaskState( + entry: SubagentRunRecord, +): DetachedTaskTerminalState | undefined { + const endedAt = entry.endedAt; + const outcome = entry.outcome; + const completion = entry.completion; + if ( + typeof endedAt !== "number" || + !outcome || + entry.pauseReason === "sessions_yield" || + (completion?.resultText === undefined && typeof completion?.capturedAt !== "number") + ) { + return undefined; + } + const progressSummary = completion.resultText ?? undefined; + if ( + entry.endedReason === SUBAGENT_ENDED_REASON_KILLED && + entry.suppressAnnounceReason !== "steer-restart" + ) { + return { + status: "cancelled", + endedAt, + lastEventAt: endedAt, + error: SUBAGENT_KILL_TASK_ERROR, + progressSummary, + terminalSummary: null, + }; + } + if (outcome.status === "ok") { + const terminal = + entry.expectsCompletionMessage === true + ? resolveRequiredCompletionTerminalResult(completion.resultText) + : {}; + return { + status: "succeeded", + endedAt, + lastEventAt: endedAt, + progressSummary, + terminalSummary: terminal.terminalSummary ?? null, + terminalOutcome: terminal.terminalOutcome, + }; + } + return { + status: outcome.status === "timeout" ? "timed_out" : "failed", + endedAt, + lastEventAt: endedAt, + error: outcome.status === "error" ? outcome.error : undefined, + progressSummary, + terminalSummary: null, + }; +} + +/** Preserves execution end time, except when a paused run was killed after its yield. */ +export function resolveKilledSubagentTaskEndedAt(entry: SubagentRunRecord): number | undefined { + if (entry.killReconciliation) { + return entry.killReconciliation.killedAt; + } + const endedAt = entry.endedAt; + const cleanupCompletedAt = entry.cleanupCompletedAt; + return entry.suppressAnnounceReason === "killed" && + typeof endedAt === "number" && + typeof cleanupCompletedAt === "number" && + cleanupCompletedAt > endedAt + ? cleanupCompletedAt + : endedAt; +} + /** Maps registry run outcome to lifecycle event outcome. */ export function resolveLifecycleOutcomeFromRunOutcome( outcome: SubagentRunOutcome | undefined, diff --git a/src/agents/subagent-registry-helpers.ts b/src/agents/subagent-registry-helpers.ts index 2f5315cbc51..4257351dbc5 100644 --- a/src/agents/subagent-registry-helpers.ts +++ b/src/agents/subagent-registry-helpers.ts @@ -14,7 +14,10 @@ import { defaultRuntime } from "../runtime.js"; import { truncateUtf8Prefix } from "../utils/utf8-truncate.js"; import { withSubagentOutcomeTiming } from "./subagent-announce-output.js"; import { getDeliveryAttemptCount, getDeliveryLastError } from "./subagent-delivery-state.js"; -import { SUBAGENT_ENDED_REASON_ERROR } from "./subagent-lifecycle-events.js"; +import { + SUBAGENT_ENDED_REASON_ERROR, + SUBAGENT_ENDED_REASON_KILLED, +} from "./subagent-lifecycle-events.js"; import { shouldUpdateRunOutcome } from "./subagent-registry-completion.js"; import type { SubagentRunRecord } from "./subagent-registry.types.js"; import { @@ -23,6 +26,7 @@ import { resolveSubagentSessionStatus, } from "./subagent-session-metrics.js"; import { + resolveCompletionFromSessionEntry, resolveSubagentRunOrphanReason, type SubagentRunOrphanReason, } from "./subagent-session-reconciliation.js"; @@ -33,6 +37,7 @@ export { resolveSubagentSessionStatus, } from "./subagent-session-metrics.js"; +export const PROVISIONAL_KILL_RECONCILIATION_MS = 5 * 60_000; export const MIN_ANNOUNCE_RETRY_DELAY_MS = 1_000; const MAX_ANNOUNCE_RETRY_DELAY_MS = 8_000; export const MAX_ANNOUNCE_RETRY_COUNT = 3; @@ -90,7 +95,10 @@ export function logAnnounceGiveUp(entry: SubagentRunRecord, reason: "retry-limit } /** Persists child session timing/status derived from the subagent registry row. */ -export async function persistSubagentSessionTiming(entry: SubagentRunRecord) { +export async function persistSubagentSessionTiming( + entry: SubagentRunRecord, + options?: { isCurrentGeneration?: () => boolean }, +) { const childSessionKey = entry.childSessionKey?.trim(); if (!childSessionKey) { return; @@ -111,6 +119,27 @@ export async function persistSubagentSessionTiming(entry: SubagentRunRecord) { await patchSessionEntry( { storePath, sessionKey: childSessionKey }, (sessionEntry) => { + // Recheck under the session-store write lock. A completion may have + // waited behind a steer/restart that transferred this session's ownership. + if (options?.isCurrentGeneration && !options.isCurrentGeneration()) { + return null; + } + if (status === "killed") { + const existingCompletion = resolveCompletionFromSessionEntry(sessionEntry, Date.now(), { + notBeforeMs: entry.startedAt ?? entry.createdAt, + }); + if (existingCompletion && existingCompletion.reason !== SUBAGENT_ENDED_REASON_KILLED) { + // A provider result already reached durable session state. The kill + // marker is provisional and must not erase restart reconciliation evidence + // or leave the session looking aborted after that completion won. + if (sessionEntry.abortedLastRun !== true) { + return null; + } + const completedEntry = { ...sessionEntry }; + delete completedEntry.abortedLastRun; + return completedEntry; + } + } const next = { ...sessionEntry }; if (typeof startedAt === "number" && Number.isFinite(startedAt)) { @@ -136,6 +165,9 @@ export async function persistSubagentSessionTiming(entry: SubagentRunRecord) { } else { delete next.status; } + if (status && status !== "killed") { + delete next.abortedLastRun; + } return next; }, { replaceEntry: true }, @@ -286,6 +318,11 @@ export function reconcileOrphanedRestoredRuns(params: { const now = Date.now(); let changed = false; for (const [runId, entry] of params.runs.entries()) { + if (entry.killReconciliation) { + // Provider completion may still repair this provisional kill. The + // sweeper owns its bounded reconciliation even when the session vanished. + continue; + } const orphanReason = resolveSubagentRunOrphanReason({ entry, includeStaleUnended: true, diff --git a/src/agents/subagent-registry-lifecycle.test.ts b/src/agents/subagent-registry-lifecycle.test.ts index c9282101529..fdf391ff838 100644 --- a/src/agents/subagent-registry-lifecycle.test.ts +++ b/src/agents/subagent-registry-lifecycle.test.ts @@ -2,6 +2,7 @@ // detached task status, and resource retirement around child-run endings. import { beforeEach, describe, expect, it, vi } from "vitest"; import type { CallGatewayOptions } from "../gateway/call.js"; +import { SUBAGENT_KILL_TASK_ERROR } from "../tasks/detached-task-runtime-contract.js"; import { buildAnnounceIdFromChildRun, buildAnnounceIdempotencyKey, @@ -13,6 +14,7 @@ import { SUBAGENT_ENDED_REASON_KILLED, } from "./subagent-lifecycle-events.js"; import { createSubagentRegistryLifecycleController } from "./subagent-registry-lifecycle.js"; +import { markSubagentRunPausedAfterYield } from "./subagent-registry-run-manager.js"; import type { SubagentRunRecord } from "./subagent-registry.types.js"; type LifecycleControllerParams = Parameters[0]; @@ -93,6 +95,7 @@ vi.mock("./subagent-registry-helpers.js", () => ({ ANNOUNCE_EXPIRY_MS: 5 * 60_000, MAX_ANNOUNCE_RETRY_COUNT: 3, MIN_ANNOUNCE_RETRY_DELAY_MS: 1_000, + PROVISIONAL_KILL_RECONCILIATION_MS: 5 * 60_000, capFrozenResultText: (text: string) => text.trim(), logAnnounceGiveUp: helperMocks.logAnnounceGiveUp, persistSubagentSessionTiming: helperMocks.persistSubagentSessionTiming, @@ -182,12 +185,15 @@ function createLifecycleController({ resumedRuns: new Set(), subagentAnnounceTimeoutMs: 1_000, persist: vi.fn(), + persistOrThrow: vi.fn(), clearPendingLifecycleError: vi.fn(), countPendingDescendantRuns: () => 0, suppressAnnounceForSteerRestart: () => false, + resolveSubagentTask: () => ({ lookup: "available" }), shouldEmitEndedHookForRun: () => false, emitSubagentEndedHookForRun: vi.fn(async () => {}), notifyContextEngineSubagentEnded: vi.fn(async () => {}), + retireSupersededRun: vi.fn(async () => {}), resumeSubagentRun: vi.fn(), callGateway: async >(opts: CallGatewayOptions): Promise => (await gatewayMocks.callGateway(opts)) as T, @@ -272,6 +278,7 @@ describe("subagent registry lifecycle hardening", () => { it("does not reject completion when task finalization throws", async () => { const persist = vi.fn(); + const persistOrThrow = vi.fn(); const warn = vi.fn(); const entry = createRunEntry(); const runs = new Map([[entry.runId, entry]]); @@ -279,7 +286,7 @@ describe("subagent registry lifecycle hardening", () => { throw new Error("task store boom"); }); - const controller = createLifecycleController({ entry, runs, persist, warn }); + const controller = createLifecycleController({ entry, runs, persist, persistOrThrow, warn }); await expect( controller.completeSubagentRun({ @@ -292,6 +299,10 @@ describe("subagent registry lifecycle hardening", () => { ).resolves.toBeUndefined(); expect(warn).toHaveBeenCalledTimes(1); + expect(persistOrThrow).toHaveBeenCalledTimes(1); + expect(persistOrThrow.mock.invocationCallOrder[0]).toBeLessThan( + taskExecutorMocks.completeTaskRunByRunId.mock.invocationCallOrder[0]!, + ); const [warning, warningFields] = firstCall(warn); expect(warning).toBe("failed to finalize subagent background task state"); expectFields(warningFields, { @@ -309,6 +320,1393 @@ describe("subagent registry lifecycle hardening", () => { }); }); + it("restores the registry state when canonical completion persistence fails", async () => { + const entry = createRunEntry(); + const original = structuredClone(entry); + const persistOrThrow = vi.fn(() => { + throw new Error("registry store boom"); + }); + const controller = createLifecycleController({ entry, persistOrThrow }); + + await expect( + controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: false, + }), + ).rejects.toThrow("registry store boom"); + + expect(entry).toEqual(original); + expect(taskExecutorMocks.completeTaskRunByRunId).not.toHaveBeenCalled(); + }); + + it("restores a provisional kill when canonical task projection fails", async () => { + const entry = createRunEntry({ + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "agent run aborted" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: 4_000 }, + cleanupHandled: true, + cleanupCompletedAt: 4_000, + }); + const original = structuredClone(entry); + const persistOrThrow = vi.fn(); + taskExecutorMocks.completeTaskRunByRunId.mockImplementation(() => { + throw new Error("task store boom"); + }); + const controller = createLifecycleController({ + entry, + persistOrThrow, + resolveSubagentTask: () => ({ + lookup: "available", + task: { + taskId: "task-provisional", + runtime: "subagent", + status: "cancelled", + error: SUBAGENT_KILL_TASK_ERROR, + } as never, + }), + }); + + await expect( + controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_001, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: false, + }), + ).rejects.toThrow("subagent task projection did not finalize"); + + expect(entry).toEqual(original); + expect(persistOrThrow).not.toHaveBeenCalled(); + }); + + it("commits a reconciled task before its canonical registry outcome", async () => { + taskExecutorMocks.completeTaskRunByRunId.mockReturnValueOnce([{}]); + const entry = createRunEntry({ + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "agent run aborted" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: 4_000 }, + cleanupHandled: true, + cleanupCompletedAt: 4_000, + }); + const persistOrThrow = vi.fn(); + const controller = createLifecycleController({ + entry, + persistOrThrow, + resolveSubagentTask: () => ({ + lookup: "available", + task: { + taskId: "task-provisional", + runtime: "subagent", + status: "cancelled", + error: SUBAGENT_KILL_TASK_ERROR, + } as never, + }), + }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_001, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: false, + }); + + expect(taskExecutorMocks.completeTaskRunByRunId.mock.invocationCallOrder[0]).toBeLessThan( + persistOrThrow.mock.invocationCallOrder[0]!, + ); + expect(entry.killReconciliation).toBeUndefined(); + }); + + it("keeps the shared task writable when a steer restart aborts its old run", async () => { + const entry = createRunEntry({ suppressAnnounceReason: "steer-restart" }); + const controller = createLifecycleController({ entry }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "error", error: "agent run aborted" }, + reason: SUBAGENT_ENDED_REASON_KILLED, + triggerCleanup: false, + }); + + expect(entry).toMatchObject({ + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + }); + expect(taskExecutorMocks.failTaskRunByRunId).not.toHaveBeenCalled(); + expect(taskExecutorMocks.completeTaskRunByRunId).not.toHaveBeenCalled(); + }); + + it("marks standalone killed lifecycle tasks with the recoverable cancellation", async () => { + const entry = createRunEntry(); + const controller = createLifecycleController({ entry }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "error", error: "agent run aborted" }, + reason: SUBAGENT_ENDED_REASON_KILLED, + triggerCleanup: false, + }); + + expectFields(firstCallArg(taskExecutorMocks.failTaskRunByRunId), { + runId: entry.runId, + runtime: "subagent", + sessionKey: entry.childSessionKey, + status: "cancelled", + error: SUBAGENT_KILL_TASK_ERROR, + }); + }); + + it("normalizes an abort observed after its explicit deadline without a kill tombstone", async () => { + const entry = createRunEntry({ runTimeoutSeconds: 3 }); + const controller = createLifecycleController({ entry }); + + await controller.completeSubagentRun({ + runId: entry.runId, + startedAt: 2_000, + endedAt: 6_000, + outcome: { status: "error", error: "agent run aborted" }, + reason: SUBAGENT_ENDED_REASON_KILLED, + triggerCleanup: false, + }); + + expect(entry).toMatchObject({ + endedAt: 5_000, + endedReason: SUBAGENT_ENDED_REASON_COMPLETE, + outcome: { status: "timeout", startedAt: 2_000, endedAt: 5_000 }, + }); + expect(entry.killReconciliation).toBeUndefined(); + expect(entry.suppressAnnounceReason).toBeUndefined(); + expect(taskExecutorMocks.failTaskRunByRunId).toHaveBeenCalledWith( + expect.objectContaining({ status: "timed_out", endedAt: 5_000 }), + ); + }); + + it("keeps a deadline-normalized steer abort from terminalizing the shared task", async () => { + const entry = createRunEntry({ + runTimeoutSeconds: 3, + suppressAnnounceReason: "steer-restart", + }); + const controller = createLifecycleController({ entry }); + + await controller.completeSubagentRun({ + runId: entry.runId, + startedAt: 2_000, + endedAt: 6_000, + outcome: { status: "error", error: "agent run aborted" }, + reason: SUBAGENT_ENDED_REASON_KILLED, + triggerCleanup: false, + }); + + expect(entry).toMatchObject({ + endedAt: 5_000, + endedReason: SUBAGENT_ENDED_REASON_COMPLETE, + outcome: { status: "timeout" }, + suppressAnnounceReason: "steer-restart", + }); + expect(taskExecutorMocks.failTaskRunByRunId).not.toHaveBeenCalled(); + expect(taskExecutorMocks.completeTaskRunByRunId).not.toHaveBeenCalled(); + }); + + it.each([false, true])( + "defers provisional killed publication when completion delivery is %s", + async (expectsCompletionMessage) => { + const entry = createRunEntry({ expectsCompletionMessage }); + const emitSubagentEndedHookForRun = vi.fn(async () => {}); + const runSubagentAnnounceFlow = vi.fn(async () => true); + const controller = createLifecycleController({ + entry, + shouldEmitEndedHookForRun: () => true, + emitSubagentEndedHookForRun, + runSubagentAnnounceFlow, + }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "error", error: "agent run aborted" }, + reason: SUBAGENT_ENDED_REASON_KILLED, + triggerCleanup: true, + }); + + expect(entry).toMatchObject({ + endedReason: SUBAGENT_ENDED_REASON_KILLED, + suppressAnnounceReason: "killed", + }); + expect(emitSubagentEndedHookForRun).not.toHaveBeenCalled(); + expect(runSubagentAnnounceFlow).not.toHaveBeenCalled(); + expectFields(firstCallArg(taskExecutorMocks.failTaskRunByRunId), { + error: SUBAGENT_KILL_TASK_ERROR, + }); + }, + ); + + it("recaptures the final reply when success supersedes a killed lifecycle", async () => { + const entry = createRunEntry({ + expectsCompletionMessage: true, + suppressAnnounceReason: "killed", + }); + const captureSubagentCompletionReply = vi.fn( + async () => "Fixed the crash and verified the regression tests pass.", + ); + const controller = createLifecycleController({ + entry, + captureSubagentCompletionReply, + }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "error", error: "agent run aborted" }, + reason: SUBAGENT_ENDED_REASON_KILLED, + triggerCleanup: false, + }); + expect(entry.completion).toMatchObject({ resultText: null }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_001, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: false, + }); + + expect(captureSubagentCompletionReply).toHaveBeenCalledOnce(); + expect(entry.completion?.resultText).toBe( + "Fixed the crash and verified the regression tests pass.", + ); + const finalArg = taskExecutorMocks.completeTaskRunByRunId.mock.calls.at(-1)?.[0]; + expectFields(finalArg, { + runId: entry.runId, + status: undefined, + progressSummary: "Fixed the crash and verified the regression tests pass.", + terminalSummary: null, + }); + }); + + it("recaptures the partial reply when timeout supersedes a killed lifecycle", async () => { + const entry = createRunEntry({ + expectsCompletionMessage: true, + suppressAnnounceReason: "killed", + }); + const captureSubagentCompletionReply = vi.fn(async () => "Partial result before timeout."); + const controller = createLifecycleController({ + entry, + captureSubagentCompletionReply, + }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "error", error: "agent run aborted" }, + reason: SUBAGENT_ENDED_REASON_KILLED, + triggerCleanup: false, + }); + expect(entry.completion).toMatchObject({ resultText: null }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_001, + outcome: { status: "timeout" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: false, + }); + + expect(captureSubagentCompletionReply).toHaveBeenCalledOnce(); + expect(entry.completion?.resultText).toBe("Partial result before timeout."); + }); + + it("preserves a captured reply when success supersedes a delayed killed lifecycle", async () => { + const entry = createRunEntry({ + endedAt: 4_000, + archiveAtMs: 5_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "agent run aborted" }, + expectsCompletionMessage: true, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: 4_000 }, + cleanupHandled: true, + completion: { + required: true, + resultText: "Already captured final reply.", + capturedAt: 4_000, + }, + }); + const captureSubagentCompletionReply = vi.fn(async () => undefined); + const controller = createLifecycleController({ + entry, + captureSubagentCompletionReply, + }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_001, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: false, + }); + + expect(captureSubagentCompletionReply).not.toHaveBeenCalled(); + expect(entry.completion).toMatchObject({ + resultText: "Already captured final reply.", + capturedAt: 4_000, + }); + expect(entry.archiveAtMs).toBe(5_000); + expectFields(taskExecutorMocks.completeTaskRunByRunId.mock.calls.at(-1)?.[0], { + progressSummary: "Already captured final reply.", + }); + }); + + it("keeps success canonical while a killed callback waits behind reply capture", async () => { + const entry = createRunEntry({ expectsCompletionMessage: true }); + let releaseCapture: ((value: string) => void) | undefined; + const captureSubagentCompletionReply = vi.fn( + () => + new Promise((resolve) => { + releaseCapture = resolve; + }), + ); + const controller = createLifecycleController({ + entry, + captureSubagentCompletionReply, + }); + + const success = controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: false, + }); + await vi.waitFor(() => expect(captureSubagentCompletionReply).toHaveBeenCalledOnce()); + const killed = controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_001, + outcome: { status: "error", error: "agent run aborted" }, + reason: SUBAGENT_ENDED_REASON_KILLED, + triggerCleanup: false, + }); + releaseCapture?.("Canonical final reply."); + await Promise.all([success, killed]); + + expect(entry).toMatchObject({ + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_COMPLETE, + outcome: { status: "ok" }, + completion: { resultText: "Canonical final reply." }, + }); + expect(taskExecutorMocks.failTaskRunByRunId).not.toHaveBeenCalled(); + expectFields(taskExecutorMocks.completeTaskRunByRunId.mock.calls.at(-1)?.[0], { + progressSummary: "Canonical final reply.", + }); + }); + + it.each(["keep", "delete"] as const)( + "invalidates in-flight %s cleanup when an authoritative yield revives the run", + async (cleanup) => { + const entry = createRunEntry({ + cleanup, + expectsCompletionMessage: true, + }); + const runs = new Map([[entry.runId, entry]]); + let finishAnnounce: ((didAnnounce: boolean) => void) | undefined; + const runSubagentAnnounceFlow = vi.fn( + () => + new Promise((resolve) => { + finishAnnounce = resolve; + }), + ); + const controller = createLifecycleController({ + entry, + runs, + runSubagentAnnounceFlow, + captureSubagentCompletionReply: vi.fn(async () => "premature terminal reply"), + }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: true, + }); + expect(runSubagentAnnounceFlow).toHaveBeenCalledOnce(); + expect(entry.cleanupHandled).toBe(true); + + expect( + markSubagentRunPausedAfterYield({ + entry, + startedAt: 2_000, + endedAt: 4_001, + }), + ).toBe(true); + finishAnnounce?.(true); + await vi.waitFor(() => expect(entry.pauseReason).toBe("sessions_yield")); + + expect(runs.get(entry.runId)).toBe(entry); + expect(entry.cleanupHandled).toBe(false); + expect(entry.cleanupCompletedAt).toBeUndefined(); + expect(helperMocks.safeRemoveAttachmentsDir).not.toHaveBeenCalled(); + expect(gatewayMocks.callGateway).not.toHaveBeenCalledWith( + expect.objectContaining({ method: "sessions.delete" }), + ); + }, + ); + + it("rejects a yield after direct delete cleanup has been dispatched", async () => { + const entry = createRunEntry({ cleanup: "delete", expectsCompletionMessage: false }); + const runs = new Map([[entry.runId, entry]]); + let releaseDelete: (() => void) | undefined; + gatewayMocks.callGateway.mockImplementation((opts) => { + if (opts.method !== "sessions.delete") { + return Promise.resolve({}); + } + return new Promise>((resolve) => { + releaseDelete = () => resolve({}); + }); + }); + const controller = createLifecycleController({ entry, runs }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: true, + }); + await vi.waitFor(() => expect(entry.deleteCleanupDispatchedAt).toBeTypeOf("number")); + + expect(markSubagentRunPausedAfterYield({ entry, endedAt: 4_001 })).toBe(false); + expect(entry.pauseReason).toBeUndefined(); + expect(entry.endedReason).toBe(SUBAGENT_ENDED_REASON_COMPLETE); + + releaseDelete?.(); + await vi.waitFor(() => expect(runs.has(entry.runId)).toBe(false)); + }); + + it("rejects a yield after announce cleanup hands off delete dispatch", async () => { + const entry = createRunEntry({ cleanup: "delete", expectsCompletionMessage: true }); + const runs = new Map([[entry.runId, entry]]); + let releaseAnnounce: (() => void) | undefined; + const runSubagentAnnounceFlow: LifecycleControllerParams["runSubagentAnnounceFlow"] = vi.fn( + (announceParams) => + new Promise((resolve) => { + expect(announceParams.onBeforeDeleteChildSession?.()).toBe(true); + releaseAnnounce = () => resolve(true); + }), + ); + const controller = createLifecycleController({ entry, runs, runSubagentAnnounceFlow }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: true, + }); + await vi.waitFor(() => expect(entry.deleteCleanupDispatchedAt).toBeTypeOf("number")); + + expect(markSubagentRunPausedAfterYield({ entry, endedAt: 4_001 })).toBe(false); + expect(entry.pauseReason).toBeUndefined(); + expect(entry.endedReason).toBe(SUBAGENT_ENDED_REASON_COMPLETE); + + releaseAnnounce?.(); + await vi.waitFor(() => expect(runs.has(entry.runId)).toBe(false)); + }); + + it("discards completion capture when an authoritative yield arrives during the await", async () => { + const entry = createRunEntry({ expectsCompletionMessage: true }); + let finishCapture: ((result: string) => void) | undefined; + const captureSubagentCompletionReply = vi.fn( + () => + new Promise((resolve) => { + finishCapture = resolve; + }), + ); + const controller = createLifecycleController({ + entry, + captureSubagentCompletionReply, + }); + + const completion = controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: true, + }); + await vi.waitFor(() => expect(captureSubagentCompletionReply).toHaveBeenCalledOnce()); + expect(markSubagentRunPausedAfterYield({ entry, endedAt: 4_001 })).toBe(true); + finishCapture?.("stale pre-yield reply"); + await completion; + + expect(entry).toMatchObject({ + pauseReason: "sessions_yield", + completion: { required: true }, + }); + expect(entry.completion?.resultText).toBeUndefined(); + expect(entry.completion?.capturedAt).toBeUndefined(); + expect(taskExecutorMocks.completeTaskRunByRunId).not.toHaveBeenCalled(); + }); + + it("abandons a killed callback tail after success becomes canonical", async () => { + const entry = createRunEntry({ expectsCompletionMessage: true }); + let releaseKilledTiming: (() => void) | undefined; + helperMocks.persistSubagentSessionTiming + .mockImplementationOnce( + () => + new Promise((resolve) => { + releaseKilledTiming = resolve; + }), + ) + .mockResolvedValueOnce(undefined); + const runSubagentAnnounceFlow = vi.fn<(_params: unknown) => Promise>(async () => true); + const controller = createLifecycleController({ + entry, + runSubagentAnnounceFlow, + captureSubagentCompletionReply: vi.fn(async () => "Canonical success."), + }); + + const killed = controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "error", error: "agent run aborted" }, + reason: SUBAGENT_ENDED_REASON_KILLED, + triggerCleanup: true, + }); + await vi.waitFor(() => expect(helperMocks.persistSubagentSessionTiming).toHaveBeenCalledOnce()); + const success = controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_001, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: true, + }); + await vi.waitFor(() => + expect(helperMocks.persistSubagentSessionTiming).toHaveBeenCalledTimes(2), + ); + releaseKilledTiming?.(); + await Promise.all([killed, success]); + + expect(entry).toMatchObject({ + endedReason: SUBAGENT_ENDED_REASON_COMPLETE, + outcome: { status: "ok" }, + completion: { resultText: "Canonical success." }, + }); + expect(runSubagentAnnounceFlow).toHaveBeenCalledOnce(); + expect(runSubagentAnnounceFlow.mock.calls[0]?.[0]).toMatchObject({ + outcome: { status: "ok" }, + roundOneReply: "Canonical success.", + }); + }); + + it("keeps requester stop delivery suppressed when provider completion wins", async () => { + const entry = createRunEntry({ + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "agent run aborted" }, + expectsCompletionMessage: true, + suppressAnnounceReason: "killed", + killReconciliation: { + killedAt: 4_000, + suppressTaskDelivery: true, + }, + cleanupHandled: true, + cleanupCompletedAt: 4_000, + }); + const runSubagentAnnounceFlow = vi.fn<(_params: unknown) => Promise>(async () => true); + const emitSubagentEndedHookForRun = vi.fn(async () => {}); + const controller = createLifecycleController({ + entry, + runSubagentAnnounceFlow, + shouldEmitEndedHookForRun: () => true, + emitSubagentEndedHookForRun, + }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_001, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: true, + }); + + await vi.waitFor(() => expect(entry.cleanupCompletedAt).toBeTypeOf("number")); + expect(runSubagentAnnounceFlow).not.toHaveBeenCalled(); + expect(entry.delivery?.status).toBe("not_required"); + expect(entry.suppressCompletionDelivery).toBeUndefined(); + expect(emitSubagentEndedHookForRun).toHaveBeenCalledWith( + expect.objectContaining({ + entry, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + }), + ); + expectFields(firstCallArg(taskExecutorMocks.completeTaskRunByRunId), { + runId: entry.runId, + suppressDelivery: true, + }); + }); + + it.each([ + { + name: "failure", + reason: SUBAGENT_ENDED_REASON_ERROR, + outcome: { status: "error" as const, error: "provider failed" }, + }, + { + name: "timeout", + reason: SUBAGENT_ENDED_REASON_COMPLETE, + outcome: { status: "timeout" as const }, + }, + ])( + "keeps canonical $name when a delayed killed callback arrives", + async ({ reason, outcome }) => { + const entry = createRunEntry(); + const controller = createLifecycleController({ entry }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome, + reason, + triggerCleanup: false, + }); + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_001, + outcome: { status: "error", error: "agent run aborted" }, + reason: SUBAGENT_ENDED_REASON_KILLED, + triggerCleanup: false, + }); + + expect(entry.outcome?.status).toBe(outcome.status); + expect(entry.endedReason).toBe(reason); + expect(taskExecutorMocks.failTaskRunByRunId).toHaveBeenCalledTimes(1); + }, + ); + + it.each([ + { + name: "failure", + reason: SUBAGENT_ENDED_REASON_ERROR, + outcome: { status: "error" as const, error: "provider failed" }, + }, + { + name: "timeout", + reason: SUBAGENT_ENDED_REASON_COMPLETE, + outcome: { status: "timeout" as const }, + }, + ])( + "restarts cleanup when canonical $name supersedes a killed run", + async ({ reason, outcome }) => { + const entry = createRunEntry({ + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "agent run aborted" }, + expectsCompletionMessage: true, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: 4_000 }, + cleanupHandled: true, + cleanupCompletedAt: 4_000, + delivery: { + status: "delivered", + announcedAt: 4_000, + deliveredAt: 4_000, + }, + }); + const controller = createLifecycleController({ entry }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_001, + outcome, + reason, + triggerCleanup: false, + }); + + expect(entry).toMatchObject({ + endedAt: 4_001, + endedReason: reason, + outcome: { status: outcome.status }, + cleanupHandled: false, + delivery: { status: "pending" }, + }); + expect(entry.cleanupCompletedAt).toBeUndefined(); + expect(entry.suppressAnnounceReason).toBeUndefined(); + expect(entry.delivery?.announcedAt).toBeUndefined(); + expect(entry.delivery?.deliveredAt).toBeUndefined(); + }, + ); + + it("keeps accepted task cancellation canonical over a late provider result", async () => { + const entry = createRunEntry({ + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "agent run aborted" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: 4_000 }, + cleanupHandled: true, + cleanupCompletedAt: 4_000, + }); + const controller = createLifecycleController({ + entry, + resolveSubagentTask: () => ({ + lookup: "available", + task: { + taskId: "task-1", + runtime: "subagent", + status: "cancelled", + error: "Cancelled by operator.", + } as never, + }), + }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_001, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: true, + }); + + expect(entry).toMatchObject({ + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "agent run aborted" }, + suppressAnnounceReason: "killed", + }); + expect(taskExecutorMocks.completeTaskRunByRunId).not.toHaveBeenCalled(); + }); + + it("does not reinterpret a legacy killed row as a provisional cancellation", async () => { + const entry = createRunEntry({ + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "legacy cancellation" }, + suppressAnnounceReason: "killed", + cleanupHandled: true, + cleanupCompletedAt: 4_000, + }); + const original = structuredClone(entry); + const controller = createLifecycleController({ entry }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_001, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: true, + }); + + expect(entry).toEqual(original); + expect(taskExecutorMocks.completeTaskRunByRunId).not.toHaveBeenCalled(); + }); + + it("keeps cancellation canonical when a custom runtime cannot resolve its task", async () => { + const entry = createRunEntry({ + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "agent run aborted" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: 4_000 }, + cleanupHandled: true, + cleanupCompletedAt: 4_000, + }); + const controller = createLifecycleController({ + entry, + resolveSubagentTask: () => ({ lookup: "unavailable" }), + }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_001, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: true, + }); + + expect(entry).toMatchObject({ + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "agent run aborted" }, + suppressAnnounceReason: "killed", + }); + expect(taskExecutorMocks.completeTaskRunByRunId).toHaveBeenCalledTimes(1); + }); + + it("accepts provider completion when an opaque custom runtime finalizes it", async () => { + taskExecutorMocks.completeTaskRunByRunId.mockReturnValueOnce([{}]); + const entry = createRunEntry({ + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "agent run aborted" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: 4_000 }, + cleanupHandled: true, + cleanupCompletedAt: 4_000, + }); + const controller = createLifecycleController({ + entry, + resolveSubagentTask: () => ({ lookup: "unavailable" }), + }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_001, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: false, + }); + + expect(entry).toMatchObject({ + endedAt: 4_001, + endedReason: SUBAGENT_ENDED_REASON_COMPLETE, + outcome: { status: "ok" }, + }); + expect(entry.suppressAnnounceReason).toBeUndefined(); + expect(taskExecutorMocks.completeTaskRunByRunId).toHaveBeenCalled(); + }); + + it("restores an opaque provisional kill when completion persistence fails", async () => { + taskExecutorMocks.completeTaskRunByRunId.mockReturnValueOnce([{}]); + const entry = createRunEntry({ + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "agent run aborted" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: 4_000 }, + cleanupHandled: true, + cleanupCompletedAt: 4_000, + }); + const original = structuredClone(entry); + const controller = createLifecycleController({ + entry, + resolveSubagentTask: () => ({ lookup: "unavailable" }), + persistOrThrow: vi.fn(() => { + throw new Error("registry store boom"); + }), + }); + + await expect( + controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_001, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: false, + }), + ).rejects.toThrow("registry store boom"); + + expect(entry).toEqual(original); + expect(taskExecutorMocks.completeTaskRunByRunId).toHaveBeenCalledTimes(1); + }); + + it("keeps cancellation that becomes durable during completion capture", async () => { + const entry = createRunEntry({ + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "agent run aborted" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: 4_000 }, + cleanupHandled: true, + cleanupCompletedAt: 4_000, + }); + let cancellationStable = false; + let finishCapture: ((value: string) => void) | undefined; + const captureSubagentCompletionReply = vi.fn( + async () => + await new Promise((resolve) => { + finishCapture = resolve; + }), + ); + const controller = createLifecycleController({ + entry, + captureSubagentCompletionReply, + resolveSubagentTask: () => ({ + lookup: "available", + task: { + taskId: "task-1", + runtime: "subagent", + status: "cancelled", + error: cancellationStable ? "Cancelled by operator." : SUBAGENT_KILL_TASK_ERROR, + } as never, + }), + }); + + const completion = controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_001, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: true, + }); + await vi.waitFor(() => expect(captureSubagentCompletionReply).toHaveBeenCalled()); + expect(entry).toMatchObject({ + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + killReconciliation: { killedAt: 4_000 }, + }); + expect(entry.completion).toBeUndefined(); + cancellationStable = true; + finishCapture?.("late success"); + await completion; + + expect(entry).toMatchObject({ + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "agent run aborted" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: 4_000 }, + cleanupHandled: true, + cleanupCompletedAt: 4_000, + }); + expect(entry.completion).toBeUndefined(); + expect(taskExecutorMocks.completeTaskRunByRunId).not.toHaveBeenCalled(); + expect(helperMocks.persistSubagentSessionTiming).not.toHaveBeenCalled(); + }); + + it("keeps accepted kill cleanup live when a later completion is rejected", async () => { + let finishSessionTiming: (() => void) | undefined; + helperMocks.persistSubagentSessionTiming.mockImplementationOnce( + async () => + await new Promise((resolve) => { + finishSessionTiming = resolve; + }), + ); + taskExecutorMocks.failTaskRunByRunId.mockReturnValueOnce([{}]); + const entry = createRunEntry(); + let cancellationStable = false; + const controller = createLifecycleController({ + entry, + resolveSubagentTask: () => ({ + lookup: "available", + task: { + taskId: "task-1", + runtime: "subagent", + status: "cancelled", + error: cancellationStable ? "Cancelled by operator." : SUBAGENT_KILL_TASK_ERROR, + } as never, + }), + }); + + const killed = controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "error", error: "agent run aborted" }, + reason: SUBAGENT_ENDED_REASON_KILLED, + triggerCleanup: true, + }); + await vi.waitFor(() => expect(helperMocks.persistSubagentSessionTiming).toHaveBeenCalled()); + + cancellationStable = true; + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_001, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: true, + }); + finishSessionTiming?.(); + await killed; + + expect( + browserLifecycleCleanupMocks.cleanupBrowserSessionsForLifecycleEnd, + ).toHaveBeenCalledTimes(1); + expect(entry.killReconciliation).toEqual({ killedAt: 4_000 }); + }); + + it("accepts a provider result that predates task cancellation", async () => { + taskExecutorMocks.completeTaskRunByRunId.mockReturnValueOnce([{}]); + const entry = createRunEntry({ + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "agent run aborted" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: 4_000 }, + cleanupHandled: true, + cleanupCompletedAt: 4_000, + }); + const controller = createLifecycleController({ + entry, + resolveSubagentTask: () => ({ + lookup: "available", + task: { + taskId: "task-1", + runtime: "subagent", + status: "cancelled", + error: "Cancelled by operator.", + } as never, + }), + }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 3_999, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: false, + }); + + expect(entry).toMatchObject({ + endedAt: 3_999, + endedReason: SUBAGENT_ENDED_REASON_COMPLETE, + outcome: { status: "ok" }, + cleanupHandled: false, + }); + expect(entry.suppressAnnounceReason).toBeUndefined(); + expect(taskExecutorMocks.completeTaskRunByRunId).toHaveBeenCalled(); + }); + + it("lets an explicit timeout deadline predate accepted task cancellation", async () => { + taskExecutorMocks.failTaskRunByRunId.mockReturnValueOnce([{}]); + const entry = createRunEntry({ + runTimeoutSeconds: 3, + endedAt: 5_500, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "agent run aborted" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: 5_500 }, + cleanupHandled: true, + cleanupCompletedAt: 5_500, + }); + const controller = createLifecycleController({ + entry, + resolveSubagentTask: () => ({ + lookup: "available", + task: { + taskId: "task-1", + runtime: "subagent", + status: "cancelled", + error: "Cancelled by operator.", + } as never, + }), + }); + + await controller.completeSubagentRun({ + runId: entry.runId, + startedAt: 2_000, + endedAt: 6_000, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: false, + }); + + expect(entry).toMatchObject({ + endedAt: 5_000, + endedReason: SUBAGENT_ENDED_REASON_COMPLETE, + outcome: { status: "timeout", startedAt: 2_000, endedAt: 5_000 }, + }); + expect(taskExecutorMocks.failTaskRunByRunId).toHaveBeenCalledWith( + expect.objectContaining({ + runId: entry.runId, + status: "timed_out", + endedAt: 5_000, + }), + ); + }); + + it("retires an old live completion without touching a newer session generation", async () => { + taskExecutorMocks.completeTaskRunByRunId.mockReturnValueOnce([{}]); + const entry = createRunEntry({ + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "agent run aborted" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: 4_000 }, + cleanupHandled: true, + cleanupCompletedAt: 4_000, + cleanup: "delete", + }); + const newer = createRunEntry({ + runId: "run-2", + createdAt: 5_000, + startedAt: 5_000, + }); + const runs = new Map([ + [entry.runId, entry], + [newer.runId, newer], + ]); + const retireSupersededRun = vi.fn(async (runId: string) => { + runs.delete(runId); + }); + const emitSubagentEndedHookForRun = vi.fn(async () => {}); + const runSubagentAnnounceFlow = vi.fn<(_params: unknown) => Promise>(async () => true); + const controller = createLifecycleController({ + entry, + runs, + resolveSubagentTask: () => ({ + lookup: "available", + task: { + taskId: "task-before-replacement", + runId: "run-before-replacement", + runtime: "subagent", + status: "cancelled", + error: SUBAGENT_KILL_TASK_ERROR, + } as never, + }), + retireSupersededRun, + shouldEmitEndedHookForRun: () => true, + emitSubagentEndedHookForRun, + runSubagentAnnounceFlow, + }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 3_999, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: true, + }); + + expect(retireSupersededRun).toHaveBeenCalledWith(entry.runId, entry); + expect(runs.has(entry.runId)).toBe(false); + expect(runs.get(newer.runId)).toBe(newer); + expect(taskExecutorMocks.completeTaskRunByRunId).toHaveBeenCalledWith( + expect.objectContaining({ runId: "run-before-replacement" }), + ); + expect(helperMocks.persistSubagentSessionTiming).not.toHaveBeenCalled(); + expect(lifecycleEventMocks.emitSessionLifecycleEvent).not.toHaveBeenCalled(); + expect(emitSubagentEndedHookForRun).not.toHaveBeenCalled(); + expect(runSubagentAnnounceFlow).not.toHaveBeenCalled(); + expect( + browserLifecycleCleanupMocks.cleanupBrowserSessionsForLifecycleEnd, + ).not.toHaveBeenCalled(); + expect(gatewayMocks.callGateway).not.toHaveBeenCalledWith( + expect.objectContaining({ method: "sessions.delete" }), + ); + }); + + it("keeps the superseded generation boundary through task finalization", async () => { + taskExecutorMocks.completeTaskRunByRunId.mockReturnValueOnce([{}]); + const marker = { killedAt: 4_000, supersededAt: 5_000 }; + const entry = createRunEntry({ + runId: "run-after-replacement", + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "agent run aborted" }, + suppressAnnounceReason: "killed", + killReconciliation: marker, + cleanupHandled: true, + cleanupCompletedAt: 4_000, + }); + const observedSupersededAt: Array = []; + const resolveSubagentTask = vi.fn((candidate: SubagentRunRecord) => { + observedSupersededAt.push(candidate.killReconciliation?.supersededAt); + return { + lookup: "available" as const, + task: { + taskId: "task-old", + runId: candidate.killReconciliation?.supersededAt + ? "run-before-replacement" + : "run-newer-generation", + runtime: "subagent" as const, + childSessionKey: candidate.childSessionKey, + status: "cancelled" as const, + error: SUBAGENT_KILL_TASK_ERROR, + } as never, + }; + }); + const retireSupersededRun = vi.fn(async () => {}); + const controller = createLifecycleController({ + entry, + resolveSubagentTask, + retireSupersededRun, + }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 3_999, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: false, + }); + + expect(resolveSubagentTask).toHaveBeenCalledTimes(2); + expect(observedSupersededAt).toEqual([5_000, 5_000]); + expect(taskExecutorMocks.completeTaskRunByRunId).toHaveBeenCalledWith( + expect.objectContaining({ runId: "run-before-replacement" }), + ); + expect(taskExecutorMocks.completeTaskRunByRunId).not.toHaveBeenCalledWith( + expect.objectContaining({ runId: "run-newer-generation" }), + ); + expect(retireSupersededRun).toHaveBeenCalledWith(entry.runId, entry); + expect(helperMocks.persistSubagentSessionTiming).not.toHaveBeenCalled(); + expect(lifecycleEventMocks.emitSessionLifecycleEvent).not.toHaveBeenCalled(); + }); + + it("updates replacement task delivery through the durable task run id", async () => { + const entry = createRunEntry({ runId: "run-after-replacement" }); + const controller = createLifecycleController({ + entry, + resolveSubagentTask: () => ({ + lookup: "available", + task: { + taskId: "task-before-replacement", + runId: "run-before-replacement", + runtime: "subagent", + childSessionKey: entry.childSessionKey, + status: "running", + } as never, + }), + }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: true, + }); + + await vi.waitFor(() => { + expect(taskExecutorMocks.setDetachedTaskDeliveryStatusByRunId).toHaveBeenCalledWith({ + runId: "run-before-replacement", + runtime: "subagent", + sessionKey: entry.childSessionKey, + deliveryStatus: "delivered", + error: undefined, + }); + }); + }); + + it("finalizes the durable task owner when custom lookup is unavailable", async () => { + taskExecutorMocks.completeTaskRunByRunId.mockReturnValueOnce([{}]); + const entry = createRunEntry({ + runId: "run-after-opaque-replacement", + taskRunId: "run-before-opaque-replacement", + }); + const controller = createLifecycleController({ + entry, + resolveSubagentTask: () => ({ lookup: "unavailable" }), + }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: false, + }); + + expect(taskExecutorMocks.completeTaskRunByRunId).toHaveBeenCalledWith( + expect.objectContaining({ + runId: "run-before-opaque-replacement", + sessionKey: entry.childSessionKey, + }), + ); + }); + + it("discards completion capture when a newer session generation takes ownership", async () => { + const entry = createRunEntry(); + const runs = new Map([[entry.runId, entry]]); + let finishCapture: ((value: string) => void) | undefined; + const captureSubagentCompletionReply = vi.fn( + async () => + await new Promise((resolve) => { + finishCapture = resolve; + }), + ); + const retireSupersededRun = vi.fn(async (runId: string) => { + runs.delete(runId); + }); + const controller = createLifecycleController({ + entry, + runs, + captureSubagentCompletionReply, + retireSupersededRun, + }); + + const completion = controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: true, + }); + await vi.waitFor(() => expect(captureSubagentCompletionReply).toHaveBeenCalled()); + const newer = createRunEntry({ runId: "run-2", createdAt: 5_000, startedAt: 5_000 }); + runs.set(newer.runId, newer); + finishCapture?.("new generation result"); + await completion; + + expect(entry.completion).toMatchObject({ resultText: null }); + expect(retireSupersededRun).toHaveBeenCalledWith(entry.runId, entry); + expect(helperMocks.persistSubagentSessionTiming).not.toHaveBeenCalled(); + expect(lifecycleEventMocks.emitSessionLifecycleEvent).not.toHaveBeenCalled(); + }); + + it("rechecks session ownership inside a delayed timing write", async () => { + const entry = createRunEntry({ generation: 1, createdAt: 1_000, startedAt: 1_000 }); + const runs = new Map([[entry.runId, entry]]); + let releaseTiming: (() => void) | undefined; + let timingWriteStillOwned: boolean | undefined; + helperMocks.persistSubagentSessionTiming.mockImplementationOnce(async (...args: unknown[]) => { + await new Promise((resolve) => { + releaseTiming = resolve; + }); + const options = args[1] as { isCurrentGeneration?: () => boolean } | undefined; + timingWriteStillOwned = options?.isCurrentGeneration?.(); + }); + const retireSupersededRun = vi.fn(async (runId: string) => { + runs.delete(runId); + }); + const controller = createLifecycleController({ entry, runs, retireSupersededRun }); + + const completion = controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: true, + }); + await vi.waitFor(() => expect(helperMocks.persistSubagentSessionTiming).toHaveBeenCalledOnce()); + const newer = createRunEntry({ + runId: "run-same-millisecond-newer", + generation: 2, + createdAt: entry.createdAt, + startedAt: entry.startedAt, + }); + runs.set(newer.runId, newer); + releaseTiming?.(); + await completion; + + expect(timingWriteStillOwned).toBe(false); + expect(retireSupersededRun).toHaveBeenCalledWith(entry.runId, entry); + expect(runs.get(newer.runId)).toBe(newer); + expect(lifecycleEventMocks.emitSessionLifecycleEvent).not.toHaveBeenCalled(); + }); + + it("finalizes restored completion text that predates capturedAt", async () => { + const entry = createRunEntry({ + completion: { required: false, resultText: "restored final result" }, + }); + const controller = createLifecycleController({ entry }); + + await controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: false, + }); + + expectFields(firstCallArg(taskExecutorMocks.completeTaskRunByRunId), { + runId: entry.runId, + status: undefined, + progressSummary: "restored final result", + terminalSummary: null, + }); + }); + it("marks required progress-only completions blocked without failing the task", async () => { const entry = createRunEntry({ expectsCompletionMessage: true, @@ -707,6 +2105,59 @@ describe("subagent registry lifecycle hardening", () => { expect(entry.delivery?.announcedAt).toBeUndefined(); }); + it("retires a stale cleanup before deleting a newer session generation", async () => { + const entry = createRunEntry({ + cleanup: "delete", + expectsCompletionMessage: false, + spawnMode: "session", + generation: 1, + }); + const runs = new Map([[entry.runId, entry]]); + const retireSupersededRun = vi.fn(async (runId: string) => { + runs.delete(runId); + }); + const controller = createLifecycleController({ entry, runs, retireSupersededRun }); + + expect(controller.startSubagentAnnounceCleanupFlow(entry.runId, entry)).toBe(true); + const newer = createRunEntry({ + runId: "run-2", + generation: 2, + createdAt: entry.createdAt, + startedAt: entry.startedAt, + }); + runs.set(newer.runId, newer); + + await vi.waitFor(() => expect(retireSupersededRun).toHaveBeenCalledWith(entry.runId, entry)); + expect(runs.get(newer.runId)).toBe(newer); + expect(gatewayMocks.callGateway).not.toHaveBeenCalledWith( + expect.objectContaining({ method: "sessions.delete" }), + ); + }); + + it("keeps provisional killed sessions across resumed cleanup", async () => { + const entry = createRunEntry({ + cleanup: "delete", + endedAt: 4_000, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "agent run aborted" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: 4_000 }, + archiveAtMs: 304_000, + expectsCompletionMessage: false, + }); + const runs = new Map([[entry.runId, entry]]); + const controller = createLifecycleController({ entry, runs }); + + expect(controller.startSubagentAnnounceCleanupFlow(entry.runId, entry)).toBe(false); + + expect(entry.cleanupCompletedAt).toBeUndefined(); + expect(runs.get(entry.runId)).toBe(entry); + expect(controller.startSubagentAnnounceCleanupFlow(entry.runId, entry)).toBe(false); + expect(gatewayMocks.callGateway).not.toHaveBeenCalledWith( + expect.objectContaining({ method: "sessions.delete" }), + ); + }); + it("retires bundle MCP runtimes when run-mode cleanup completes", async () => { const entry = createRunEntry({ endedAt: 4_000, @@ -796,14 +2247,14 @@ describe("subagent registry lifecycle hardening", () => { }); it("persists timing when a preexisting outcome matches without timing", async () => { - const persist = vi.fn(); + const persistOrThrow = vi.fn(); const entry = createRunEntry({ startedAt: 2_000, outcome: { status: "ok" }, expectsCompletionMessage: false, }); - const controller = createLifecycleController({ entry, persist }); + const controller = createLifecycleController({ entry, persistOrThrow }); await expect( controller.completeSubagentRun({ @@ -821,7 +2272,7 @@ describe("subagent registry lifecycle hardening", () => { endedAt: 4_250, elapsedMs: 2_250, }); - expect(persist).toHaveBeenCalled(); + expect(persistOrThrow).toHaveBeenCalled(); }); it("does not wait for a completion reply when the run does not expect one", async () => { @@ -858,7 +2309,7 @@ describe("subagent registry lifecycle hardening", () => { }); it("does not freeze stale reply text for terminal error outcomes", async () => { - const persist = vi.fn(); + const persistOrThrow = vi.fn(); const captureSubagentCompletionReply = vi.fn(async () => "stale assistant text"); const entry = createRunEntry({ expectsCompletionMessage: true, @@ -866,7 +2317,7 @@ describe("subagent registry lifecycle hardening", () => { const controller = createLifecycleController({ entry, - persist, + persistOrThrow, captureSubagentCompletionReply, }); @@ -887,7 +2338,7 @@ describe("subagent registry lifecycle hardening", () => { error: "All models failed (2): timeout", progressSummary: undefined, }); - expect(persist).toHaveBeenCalled(); + expect(persistOrThrow).toHaveBeenCalled(); }); it("does not re-run announce flow after completion was already delivered", async () => { @@ -956,9 +2407,58 @@ describe("subagent registry lifecycle hardening", () => { entry, reason: SUBAGENT_ENDED_REASON_COMPLETE, sendFarewell: true, + isCurrent: expect.any(Function), }); }); + it("suppresses a deferred ended hook after a newer session generation registers", async () => { + const entry = createRunEntry({ + delivery: { status: "delivered", announcedAt: 3_500, deliveredAt: 3_500 }, + endedAt: 4_000, + expectsCompletionMessage: true, + generation: 1, + }); + const runs = new Map([[entry.runId, entry]]); + let finishPluginLoad: (() => void) | undefined; + const emitted = vi.fn(); + const emitSubagentEndedHookForRun = vi.fn(async (params: { isCurrent?: () => boolean }) => { + await new Promise((resolve) => { + finishPluginLoad = resolve; + }); + if (params.isCurrent?.() !== false) { + emitted(); + } + }); + const controller = createLifecycleController({ + entry, + runs, + shouldEmitEndedHookForRun: () => true, + emitSubagentEndedHookForRun, + }); + + const completion = controller.completeSubagentRun({ + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "ok" }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: true, + }); + await vi.waitFor(() => expect(emitSubagentEndedHookForRun).toHaveBeenCalled()); + runs.set( + "run-2", + createRunEntry({ + runId: "run-2", + createdAt: 5_000, + startedAt: 5_000, + generation: 2, + }), + ); + finishPluginLoad?.(); + await completion; + + expect(emitted).not.toHaveBeenCalled(); + }); + it("produces valid cleanupCompletedAt on give-up path when completionAnnouncedAt is undefined", async () => { const persist = vi.fn(); const entry = createRunEntry({ @@ -1068,9 +2568,11 @@ describe("subagent registry lifecycle hardening", () => { outcome, retainAttachmentsOnKeep: true, }); + const runs = new Map([[entry.runId, entry]]); const controller = createLifecycleController({ entry, + runs, persist, captureSubagentCompletionReply: vi.fn(async () => undefined), }); @@ -1084,7 +2586,11 @@ describe("subagent registry lifecycle hardening", () => { expect(entry.delivery?.payload).toBeUndefined(); expect(entry.delivery?.suspendedAt).toBeUndefined(); expect(entry.delivery?.suspendedReason).toBeUndefined(); - expect(entry.cleanupCompletedAt).toBeTypeOf("number"); + if (endedReason === SUBAGENT_ENDED_REASON_KILLED) { + expect(runs.has(entry.runId)).toBe(false); + } else { + expect(entry.cleanupCompletedAt).toBeTypeOf("number"); + } expect(persist).toHaveBeenCalled(); }, ); @@ -1437,11 +2943,12 @@ describe("subagent registry lifecycle hardening", () => { // Second caller observes the flag set, skips the cleanup wrapper, and must // still drain the retire + announce tail without waiting on the first // caller's still-pending cleanup. - await controller.completeSubagentRun(completeParams); + await controller.completeSubagentRun({ ...completeParams, endedAt: 3_999 }); expect( browserLifecycleCleanupMocks.cleanupBrowserSessionsForLifecycleEnd, ).toHaveBeenCalledTimes(1); + expect(entry.endedAt).toBe(4_000); expect(bundleMcpRuntimeMocks.retireSessionMcpRuntimeForSessionKey).toHaveBeenCalled(); expect(runSubagentAnnounceFlow).toHaveBeenCalled(); @@ -1449,4 +2956,40 @@ describe("subagent registry lifecycle hardening", () => { releaseFirstCleanup?.(); await expect(firstCompletion).resolves.toBeUndefined(); }); + + it("does not invalidate an active timeout tail when a published timeout is observed again", async () => { + const entry = createRunEntry({ + expectsCompletionMessage: true, + runTimeoutSeconds: 2, + }); + let releaseTiming: (() => void) | undefined; + helperMocks.persistSubagentSessionTiming.mockImplementationOnce( + () => + new Promise((resolve) => { + releaseTiming = resolve; + }), + ); + const runSubagentAnnounceFlow = vi.fn<(_params: unknown) => Promise>(async () => true); + const controller = createLifecycleController({ entry, runSubagentAnnounceFlow }); + const completeParams = { + runId: entry.runId, + endedAt: 4_000, + outcome: { status: "timeout" as const }, + reason: SUBAGENT_ENDED_REASON_COMPLETE, + triggerCleanup: true, + }; + + const firstCompletion = controller.completeSubagentRun(completeParams); + await vi.waitFor(() => expect(helperMocks.persistSubagentSessionTiming).toHaveBeenCalledOnce()); + entry.endedHookEmittedAt = 4_000; + + await controller.completeSubagentRun(completeParams); + releaseTiming?.(); + await firstCompletion; + + expect(runSubagentAnnounceFlow).toHaveBeenCalledOnce(); + expect(runSubagentAnnounceFlow.mock.calls[0]?.[0]).toMatchObject({ + outcome: { status: "timeout" }, + }); + }); }); diff --git a/src/agents/subagent-registry-lifecycle.ts b/src/agents/subagent-registry-lifecycle.ts index 4b370c07340..299a434165c 100644 --- a/src/agents/subagent-registry-lifecycle.ts +++ b/src/agents/subagent-registry-lifecycle.ts @@ -12,15 +12,14 @@ import { defaultRuntime } from "../runtime.js"; import { emitSessionLifecycleEvent } from "../sessions/session-lifecycle-events.js"; import { extractTextFromChatContent } from "../shared/chat-content.js"; import { createLazyImportLoader } from "../shared/lazy-promise.js"; +import type { DetachedTaskFindResult } from "../tasks/detached-task-runtime-contract.js"; import { completeTaskRunByRunId, failTaskRunByRunId, setDetachedTaskDeliveryStatusByRunId, } from "../tasks/detached-task-runtime.js"; -import { - resolveRequiredCompletionDeliveryFailureTerminalResult, - resolveRequiredCompletionTerminalResult, -} from "../tasks/task-completion-contract.js"; +import { isProvisionalSubagentKillTask } from "../tasks/task-cancellation-state.js"; +import { resolveRequiredCompletionDeliveryFailureTerminalResult } from "../tasks/task-completion-contract.js"; import { normalizeDeliveryContext } from "../utils/delivery-context.shared.js"; import { retireSessionMcpRuntimeForSessionKey } from "./agent-bundle-mcp-tools.js"; import { @@ -39,13 +38,18 @@ import { } from "./subagent-delivery-state.js"; import { SUBAGENT_ENDED_REASON_COMPLETE, + SUBAGENT_ENDED_REASON_KILLED, type SubagentLifecycleEndedReason, } from "./subagent-lifecycle-events.js"; import { resolveCleanupCompletionReason, resolveDeferredCleanupDecision, } from "./subagent-registry-cleanup.js"; -import { shouldUpdateRunOutcome } from "./subagent-registry-completion.js"; +import { + resolveKilledSubagentTaskEndedAt, + resolveFinalizedSubagentTaskState, + shouldUpdateRunOutcome, +} from "./subagent-registry-completion.js"; import { ANNOUNCE_COMPLETION_HARD_EXPIRY_MS, ANNOUNCE_EXPIRY_MS, @@ -58,7 +62,11 @@ import { safeRemoveAttachmentsDir, } from "./subagent-registry-helpers.js"; import type { PendingFinalDeliveryPayload, SubagentRunRecord } from "./subagent-registry.types.js"; -import { resolveSubagentRunDeadlineMs } from "./subagent-run-timeout.js"; +import { compareSubagentRunGeneration } from "./subagent-run-generation.js"; +import { + resolveSubagentRunDeadlineMs, + resolveSubagentRunEffectiveEndedAt, +} from "./subagent-run-timeout.js"; import { deleteSubagentSessionForCleanup } from "./subagent-session-cleanup.js"; type CaptureSubagentCompletionReply = @@ -109,12 +117,37 @@ function shouldPreservePublishedExplicitRunTimeout(params: { entry: SubagentRunR function resolveExpiredExplicitRunDeadlineMs(params: { entry: SubagentRunRecord; - nextOutcome: SubagentRunOutcome; nextEndedAt: number; observedStartedAt?: number; }): number | undefined { - const deadlineMs = resolveSubagentRunDeadlineMs(params.entry, params.observedStartedAt); - return deadlineMs !== undefined && params.nextEndedAt > deadlineMs ? deadlineMs : undefined; + const effectiveEndedAt = resolveSubagentRunEffectiveEndedAt( + params.entry, + params.nextEndedAt, + params.observedStartedAt, + ); + return effectiveEndedAt < params.nextEndedAt ? effectiveEndedAt : undefined; +} + +function isOlderEquivalentTerminalCallback(params: { + entry: SubagentRunRecord; + endedAt: number; + outcome: SubagentRunOutcome; + reason: SubagentLifecycleEndedReason; +}): boolean { + const current = params.entry.outcome; + if ( + typeof params.entry.endedAt !== "number" || + params.endedAt >= params.entry.endedAt || + params.entry.endedReason !== params.reason || + current?.status !== params.outcome.status + ) { + return false; + } + return ( + current.status !== "error" || + params.outcome.status !== "error" || + current.error === params.outcome.error + ); } export function createSubagentRegistryLifecycleController(params: { @@ -122,9 +155,11 @@ export function createSubagentRegistryLifecycleController(params: { resumedRuns: Set; subagentAnnounceTimeoutMs: number; persist(): void; + persistOrThrow(): void; clearPendingLifecycleError(runId: string): void; countPendingDescendantRuns(rootSessionKey: string): number; suppressAnnounceForSteerRestart(entry?: SubagentRunRecord): boolean; + resolveSubagentTask(entry: SubagentRunRecord): DetachedTaskFindResult; shouldEmitEndedHookForRun(args: { entry: SubagentRunRecord; reason: SubagentLifecycleEndedReason; @@ -134,6 +169,7 @@ export function createSubagentRegistryLifecycleController(params: { reason?: SubagentLifecycleEndedReason; sendFarewell?: boolean; accountId?: string; + isCurrent?: () => boolean; }): Promise; notifyContextEngineSubagentEnded(args: { childSessionKey: string; @@ -141,6 +177,7 @@ export function createSubagentRegistryLifecycleController(params: { agentDir?: string; workspaceDir?: string; }): Promise; + retireSupersededRun(runId: string, entry: SubagentRunRecord): Promise; resumeSubagentRun(runId: string): void; callGateway: typeof defaultCallGateway; captureSubagentCompletionReply: CaptureSubagentCompletionReply; @@ -149,6 +186,34 @@ export function createSubagentRegistryLifecycleController(params: { warn(message: string, meta?: Record): void; }) { const scheduledResumeTimers = new Set>(); + const terminalCompletionLocks = new Map>(); + const terminalGenerations = new WeakMap(); + const cleanupGenerations = new WeakMap(); + + const newerGenerationOwnsSession = (entry: SubagentRunRecord): boolean => + entry.killReconciliation?.supersededAt !== undefined || + Array.from(params.runs.values()).some( + (candidate) => + candidate.runId !== entry.runId && + candidate.childSessionKey === entry.childSessionKey && + compareSubagentRunGeneration(candidate, entry) > 0, + ); + + const acquireTerminalCompletionLock = async (runId: string): Promise<() => void> => { + const previous = terminalCompletionLocks.get(runId) ?? Promise.resolve(); + let releaseLock = () => {}; + const current = new Promise((resolve) => { + releaseLock = resolve; + }); + terminalCompletionLocks.set(runId, current); + await previous; + return () => { + releaseLock(); + if (terminalCompletionLocks.get(runId) === current) { + terminalCompletionLocks.delete(runId); + } + }; + }; const scheduleResumeSubagentRun = (runId: string, entry: SubagentRunRecord, delayMs: number) => { const timer = setTimeout(() => { @@ -290,25 +355,42 @@ export function createSubagentRegistryLifecycleController(params: { } }; + const resolveSubagentTaskTarget = ( + entry: SubagentRunRecord, + resolution = params.resolveSubagentTask(entry), + ) => { + const durableTaskRunId = entry.taskRunId ?? entry.runId; + return { + runId: + resolution.lookup === "available" + ? (resolution.task?.runId ?? durableTaskRunId) + : durableTaskRunId, + sessionKey: + resolution.lookup === "available" + ? (resolution.task?.childSessionKey ?? entry.childSessionKey) + : entry.childSessionKey, + }; + }; + const safeSetSubagentTaskDeliveryStatus = (args: { - runId: string; - childSessionKey: string; + entry: SubagentRunRecord; deliveryStatus: "delivered" | "failed"; deliveryError?: string; }) => { + const target = resolveSubagentTaskTarget(args.entry); try { setDetachedTaskDeliveryStatusByRunId({ - runId: args.runId, + runId: target.runId, runtime: "subagent", - sessionKey: args.childSessionKey, + sessionKey: target.sessionKey, deliveryStatus: args.deliveryStatus, error: args.deliveryStatus === "failed" ? args.deliveryError : undefined, }); } catch (err) { params.warn("failed to update subagent background task delivery state", { error: buildSafeLifecycleErrorMeta(err), - runId: maskRunId(args.runId), - childSessionKey: maskSessionKey(args.childSessionKey), + runId: maskRunId(target.runId), + childSessionKey: maskSessionKey(target.sessionKey), deliveryStatus: args.deliveryStatus, }); } @@ -317,38 +399,34 @@ export function createSubagentRegistryLifecycleController(params: { const safeFinalizeSubagentTaskRun = (args: { entry: SubagentRunRecord; outcome: SubagentRunOutcome; - }) => { - const endedAt = args.entry.endedAt ?? Date.now(); - const lastEventAt = endedAt; + taskResolution?: DetachedTaskFindResult; + }): ReturnType => { + const terminal = resolveFinalizedSubagentTaskState(args.entry); + if (!terminal) { + return []; + } + const target = resolveSubagentTaskTarget(args.entry, args.taskResolution); + const { status, error, terminalOutcome, ...details } = terminal; + const suppressDelivery = args.entry.suppressCompletionDelivery === true; try { - if (args.outcome.status === "ok") { - const completion = ensureCompletionState(args.entry); - const terminalResult = - args.entry.expectsCompletionMessage === true - ? resolveRequiredCompletionTerminalResult(completion.resultText) - : {}; - completeTaskRunByRunId({ - runId: args.entry.runId, + if (status === "succeeded") { + return completeTaskRunByRunId({ + runId: target.runId, runtime: "subagent", - sessionKey: args.entry.childSessionKey, - endedAt, - lastEventAt, - progressSummary: completion.resultText ?? undefined, - terminalSummary: terminalResult.terminalSummary ?? null, - terminalOutcome: terminalResult.terminalOutcome, + sessionKey: target.sessionKey, + ...details, + terminalOutcome, + suppressDelivery, }); - return; } - failTaskRunByRunId({ - runId: args.entry.runId, + return failTaskRunByRunId({ + runId: target.runId, runtime: "subagent", - sessionKey: args.entry.childSessionKey, - status: args.outcome.status === "timeout" ? "timed_out" : "failed", - endedAt, - lastEventAt, - error: args.outcome.status === "error" ? args.outcome.error : undefined, - progressSummary: ensureCompletionState(args.entry).resultText ?? undefined, - terminalSummary: null, + sessionKey: target.sessionKey, + ...details, + status, + error, + suppressDelivery, }); } catch (err) { params.warn("failed to finalize subagent background task state", { @@ -357,6 +435,7 @@ export function createSubagentRegistryLifecycleController(params: { childSessionKey: maskSessionKey(args.entry.childSessionKey), outcomeStatus: args.outcome.status, }); + return []; } }; @@ -369,11 +448,12 @@ export function createSubagentRegistryLifecycleController(params: { } const endedAt = args.entry.endedAt ?? Date.now(); const terminalResult = resolveRequiredCompletionDeliveryFailureTerminalResult(args.reason); + const target = resolveSubagentTaskTarget(args.entry); try { completeTaskRunByRunId({ - runId: args.entry.runId, + runId: target.runId, runtime: "subagent", - sessionKey: args.entry.childSessionKey, + sessionKey: target.sessionKey, endedAt, lastEventAt: Date.now(), progressSummary: ensureCompletionState(args.entry).resultText ?? undefined, @@ -393,25 +473,39 @@ export function createSubagentRegistryLifecycleController(params: { entry: SubagentRunRecord, outcome: SubagentRunOutcome, ): Promise => { - const completion = ensureCompletionState(entry); - if (completion.resultText !== undefined) { + if (ensureCompletionState(entry).resultText !== undefined) { return false; } if (outcome.status === "error") { + const completion = ensureCompletionState(entry); completion.resultText = null; completion.capturedAt = Date.now(); return true; } + let resultText: string | null; try { const captured = await params.captureSubagentCompletionReply(entry.childSessionKey, { waitForReply: entry.expectsCompletionMessage === true, outcome, sessionFile: entry.execution?.transcriptFile, }); - completion.resultText = captured?.trim() ? capFrozenResultText(captured) : null; + resultText = captured?.trim() ? capFrozenResultText(captured) : null; } catch { - completion.resultText = null; + resultText = null; } + const liveEntry = params.runs.get(entry.runId); + if ( + entry.pauseReason === "sessions_yield" || + liveEntry?.pauseReason === "sessions_yield" || + newerGenerationOwnsSession(entry) + ) { + return false; + } + const completion = ensureCompletionState(entry); + if (completion.resultText !== undefined) { + return false; + } + completion.resultText = resultText; completion.capturedAt = Date.now(); return true; }; @@ -487,18 +581,14 @@ export function createSubagentRegistryLifecycleController(params: { const emitCompletionEndedHookIfNeeded = async ( entry: SubagentRunRecord, reason: SubagentLifecycleEndedReason, + isCurrent?: () => boolean, ) => { - if ( - entry.expectsCompletionMessage === true && - params.shouldEmitEndedHookForRun({ - entry, - reason, - }) - ) { + if (params.shouldEmitEndedHookForRun({ entry, reason })) { await params.emitSubagentEndedHookForRun({ entry, reason, sendFarewell: true, + isCurrent, }); } }; @@ -599,8 +689,7 @@ export function createSubagentRegistryLifecycleController(params: { completion.fallbackCapturedAt = undefined; params.resumedRuns.delete(args.runId); safeSetSubagentTaskDeliveryStatus({ - runId: args.runId, - childSessionKey: args.entry.childSessionKey, + entry: args.entry, deliveryStatus: "failed", deliveryError: getDeliveryLastError(args.entry) ?? args.reason, }); @@ -638,8 +727,7 @@ export function createSubagentRegistryLifecycleController(params: { failedDelivery.status = "failed"; failedDelivery.lastError = deliveryError; safeSetSubagentTaskDeliveryStatus({ - runId: giveUpParams.runId, - childSessionKey: giveUpParams.entry.childSessionKey, + entry: giveUpParams.entry, deliveryStatus: "failed", deliveryError, }); @@ -666,7 +754,9 @@ export function createSubagentRegistryLifecycleController(params: { cleanup: giveUpParams.entry.cleanup, completedAt: Date.now(), }); - await emitCompletionEndedHookIfNeeded(giveUpParams.entry, completionReason); + await emitCompletionEndedHookIfNeeded(giveUpParams.entry, completionReason, () => + isEndedHookOwnerCurrent(giveUpParams.runId, giveUpParams.entry), + ); }; const beginSubagentCleanup = (runId: string) => { @@ -678,10 +768,55 @@ export function createSubagentRegistryLifecycleController(params: { return false; } entry.cleanupHandled = true; + cleanupGenerations.set(entry, (cleanupGenerations.get(entry) ?? 0) + 1); params.persist(); return true; }; + const isCleanupAttemptCurrent = ( + runId: string, + entry: SubagentRunRecord, + generation: number, + ): boolean => + params.runs.get(runId) === entry && + entry.cleanupHandled === true && + entry.pauseReason !== "sessions_yield" && + cleanupGenerations.get(entry) === generation && + !newerGenerationOwnsSession(entry); + + const retireSupersededCleanupIfNeeded = async ( + runId: string, + entry: SubagentRunRecord, + generation: number, + ): Promise => { + if ( + params.runs.get(runId) !== entry || + cleanupGenerations.get(entry) !== generation || + !newerGenerationOwnsSession(entry) + ) { + return false; + } + // Cleanup can yield to attachment, mirror, or announce work. A successor + // registered while it was suspended owns every session-scoped side effect. + await params.retireSupersededRun(runId, entry); + params.persist(); + return true; + }; + + const isTerminalCallbackCurrent = ( + runId: string, + entry: SubagentRunRecord, + generation: number, + ): boolean => + params.runs.get(runId) === entry && + entry.pauseReason !== "sessions_yield" && + terminalGenerations.get(entry) === generation; + + const isEndedHookOwnerCurrent = (runId: string, entry: SubagentRunRecord): boolean => { + const current = params.runs.get(runId); + return (current === undefined || current === entry) && !newerGenerationOwnsSession(entry); + }; + const retryDeferredCompletedAnnounces = (excludeRunId?: string) => { const now = Date.now(); for (const [runId, entry] of params.runs.entries()) { @@ -732,8 +867,12 @@ export function createSubagentRegistryLifecycleController(params: { entry: SubagentRunRecord; cleanup: "delete" | "keep"; completedAt: number; + preserveTranscript?: boolean; + provisionalKill?: boolean; }) => { - void removeInternalSessionEffectsTranscript(cleanupParams.entry.execution?.transcriptFile); + if (!cleanupParams.preserveTranscript) { + void removeInternalSessionEffectsTranscript(cleanupParams.entry.execution?.transcriptFile); + } if (cleanupParams.entry.spawnMode !== "session") { void retireSessionMcpRuntimeForSessionKey({ sessionKey: cleanupParams.entry.childSessionKey, @@ -750,24 +889,42 @@ export function createSubagentRegistryLifecycleController(params: { } if (cleanupParams.cleanup === "delete") { params.clearPendingLifecycleError(cleanupParams.runId); - void params.notifyContextEngineSubagentEnded({ - childSessionKey: cleanupParams.entry.childSessionKey, - reason: "deleted", - agentDir: cleanupParams.entry.agentDir, - workspaceDir: cleanupParams.entry.workspaceDir, - }); + if (!cleanupParams.provisionalKill) { + void params.notifyContextEngineSubagentEnded({ + childSessionKey: cleanupParams.entry.childSessionKey, + reason: "deleted", + agentDir: cleanupParams.entry.agentDir, + workspaceDir: cleanupParams.entry.workspaceDir, + }); + } params.runs.delete(cleanupParams.runId); params.persist(); retryDeferredCompletedAnnounces(cleanupParams.runId); return; } - void params.notifyContextEngineSubagentEnded({ - childSessionKey: cleanupParams.entry.childSessionKey, - reason: "completed", - agentDir: cleanupParams.entry.agentDir, - workspaceDir: cleanupParams.entry.workspaceDir, - }); - cleanupParams.entry.cleanupCompletedAt = cleanupParams.completedAt; + if (!cleanupParams.provisionalKill) { + void params.notifyContextEngineSubagentEnded({ + childSessionKey: cleanupParams.entry.childSessionKey, + reason: "completed", + agentDir: cleanupParams.entry.agentDir, + workspaceDir: cleanupParams.entry.workspaceDir, + }); + } + if ( + cleanupParams.entry.endedReason === SUBAGENT_ENDED_REASON_KILLED && + cleanupParams.entry.suppressAnnounceReason !== "killed" + ) { + // A reconciled killed row has served its tombstone purpose. Retire only + // the registry record; keep-mode still preserves the child session. + params.clearPendingLifecycleError(cleanupParams.runId); + params.runs.delete(cleanupParams.runId); + params.persist(); + retryDeferredCompletedAnnounces(cleanupParams.runId); + return; + } + if (!cleanupParams.provisionalKill) { + cleanupParams.entry.cleanupCompletedAt = cleanupParams.completedAt; + } params.persist(); retryDeferredCompletedAnnounces(cleanupParams.runId); }; @@ -798,28 +955,45 @@ export function createSubagentRegistryLifecycleController(params: { runId: string, cleanup: "delete" | "keep", didAnnounce: boolean, + cleanupGeneration: number, options?: { skipAnnounce?: boolean; skipDeliveryStatus?: boolean; + skipRequesterDelivery?: boolean; }, ) => { const entry = params.runs.get(runId); if (!entry) { return; } - if (entry.expectsCompletionMessage === false) { + if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { + await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration); + return; + } + if (entry.expectsCompletionMessage === false || options?.skipRequesterDelivery) { clearPendingFinalDelivery(entry); + if (options?.skipRequesterDelivery) { + ensureDeliveryState(entry).status = "not_required"; + entry.suppressCompletionDelivery = undefined; + } entry.wakeOnDescendantSettle = undefined; const shouldDeleteAttachments = cleanup === "delete" || !entry.retainAttachmentsOnKeep; if (shouldDeleteAttachments) { await safeRemoveAttachmentsDir(entry); } + if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { + await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration); + return; + } completeCleanupBookkeeping({ runId, entry, cleanup, completedAt: Date.now(), }); + await emitCompletionEndedHookIfNeeded(entry, resolveCleanupCompletionReason(entry), () => + isEndedHookOwnerCurrent(runId, entry), + ); return; } if (didAnnounce) { @@ -847,8 +1021,7 @@ export function createSubagentRegistryLifecycleController(params: { } if (shouldCreditDelivery && !options?.skipDeliveryStatus) { safeSetSubagentTaskDeliveryStatus({ - runId, - childSessionKey: entry.childSessionKey, + entry, deliveryStatus: "delivered", }); } @@ -859,11 +1032,14 @@ export function createSubagentRegistryLifecycleController(params: { completion.fallbackResultText = undefined; completion.fallbackCapturedAt = undefined; const completionReason = resolveCleanupCompletionReason(entry); - await emitCompletionEndedHookIfNeeded(entry, completionReason); const shouldDeleteAttachments = cleanup === "delete" || !entry.retainAttachmentsOnKeep; if (shouldDeleteAttachments) { await safeRemoveAttachmentsDir(entry); } + if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { + await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration); + return; + } if (cleanup === "delete") { completion.resultText = undefined; completion.capturedAt = undefined; @@ -874,6 +1050,11 @@ export function createSubagentRegistryLifecycleController(params: { cleanup, completedAt: Date.now(), }); + // Hook loading is best-effort; durable delivery and cleanup must already + // be terminal before plugin code can fail or stall. + await emitCompletionEndedHookIfNeeded(entry, completionReason, () => + isEndedHookOwnerCurrent(runId, entry), + ); return; } @@ -919,8 +1100,7 @@ export function createSubagentRegistryLifecycleController(params: { failedDelivery.lastAttemptAt = now; } safeSetSubagentTaskDeliveryStatus({ - runId, - childSessionKey: entry.childSessionKey, + entry, deliveryStatus: "failed", deliveryError, }); @@ -936,6 +1116,10 @@ export function createSubagentRegistryLifecycleController(params: { if (shouldDeleteAttachments) { await safeRemoveAttachmentsDir(entry); } + if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { + await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration); + return; + } const completionReason = resolveCleanupCompletionReason(entry); logAnnounceGiveUp(entry, deferredDecision.reason); // Giving up on announce delivery is terminal for cleanup even if the @@ -946,7 +1130,9 @@ export function createSubagentRegistryLifecycleController(params: { cleanup, completedAt: now, }); - await emitCompletionEndedHookIfNeeded(entry, completionReason); + await emitCompletionEndedHookIfNeeded(entry, completionReason, () => + isEndedHookOwnerCurrent(runId, entry), + ); return; } @@ -964,16 +1150,27 @@ export function createSubagentRegistryLifecycleController(params: { }; const startSubagentAnnounceCleanupFlow = (runId: string, entry: SubagentRunRecord): boolean => { + if (entry.killReconciliation) { + // Restores and unrelated cleanup retries must not publish a provisional + // kill. The sweeper re-enters here after durable reconciliation. + return false; + } + const cleanup = entry.cleanup; if (typeof entry.delivery?.announcedAt === "number" || entry.delivery?.status === "delivered") { if (!beginSubagentCleanup(runId)) { return false; } - void finalizeSubagentCleanup(runId, entry.cleanup, true, { + const cleanupGeneration = cleanupGenerations.get(entry)!; + void finalizeSubagentCleanup(runId, cleanup, true, cleanupGeneration, { skipAnnounce: true, }).catch((err: unknown) => { defaultRuntime.log(`[warn] subagent cleanup finalize failed (${runId}): ${String(err)}`); const current = params.runs.get(runId); - if (!current || current.cleanupCompletedAt) { + if ( + !current || + current.cleanupCompletedAt || + !isCleanupAttemptCurrent(runId, entry, cleanupGeneration) + ) { return; } current.cleanupHandled = false; @@ -984,9 +1181,22 @@ export function createSubagentRegistryLifecycleController(params: { if (!beginSubagentCleanup(runId)) { return false; } - if (entry.expectsCompletionMessage === false) { + const cleanupGeneration = cleanupGenerations.get(entry)!; + const skipRequesterDelivery = entry.suppressCompletionDelivery === true; + if (entry.expectsCompletionMessage === false || skipRequesterDelivery) { void (async () => { - if (entry.cleanup === "delete") { + // This driver is detached. Yield once so synchronous successor + // registration can invalidate it before sessions.delete is submitted. + await Promise.resolve(); + if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { + await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration); + return; + } + if (cleanup === "delete") { + // This durable boundary prevents a late yield from reviving a run + // after deletion may already have reached the gateway. + entry.deleteCleanupDispatchedAt ??= Date.now(); + params.persist(); await deleteSubagentSessionForCleanup({ callGateway: params.callGateway, childSessionKey: entry.childSessionKey, @@ -999,14 +1209,23 @@ export function createSubagentRegistryLifecycleController(params: { }), }); } - await finalizeSubagentCleanup(runId, entry.cleanup, true, { + if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { + await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration); + return; + } + await finalizeSubagentCleanup(runId, cleanup, true, cleanupGeneration, { skipAnnounce: true, skipDeliveryStatus: true, + skipRequesterDelivery, }); })().catch((err: unknown) => { defaultRuntime.log(`[warn] subagent cleanup finalize failed (${runId}): ${String(err)}`); const current = params.runs.get(runId); - if (!current || current.cleanupCompletedAt) { + if ( + !current || + current.cleanupCompletedAt || + !isCleanupAttemptCurrent(runId, entry, cleanupGeneration) + ) { return; } current.cleanupHandled = false; @@ -1018,8 +1237,16 @@ export function createSubagentRegistryLifecycleController(params: { const requesterOrigin = normalizeDeliveryContext(pendingPayload.requesterOrigin); let latestDeliveryError = getDeliveryLastError(entry); const finalizeAnnounceCleanup = async (didAnnounce: boolean) => { + if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { + await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration); + return; + } const shouldCreditPriorDelivery = !didAnnounce && (await hasPriorRequesterDeliveryMirror(entry)); + if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { + await retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration); + return; + } if (shouldCreditPriorDelivery) { latestDeliveryError = undefined; } @@ -1028,12 +1255,17 @@ export function createSubagentRegistryLifecycleController(params: { } void finalizeSubagentCleanup( runId, - entry.cleanup, + cleanup, didAnnounce || shouldCreditPriorDelivery, + cleanupGeneration, ).catch((err: unknown) => { defaultRuntime.log(`[warn] subagent cleanup finalize failed (${runId}): ${String(err)}`); const current = params.runs.get(runId); - if (!current || current.cleanupCompletedAt) { + if ( + !current || + current.cleanupCompletedAt || + !isCleanupAttemptCurrent(runId, entry, cleanupGeneration) + ) { return; } current.cleanupHandled = false; @@ -1050,7 +1282,7 @@ export function createSubagentRegistryLifecycleController(params: { requesterDisplayKey: pendingPayload.requesterDisplayKey, task: pendingPayload.task, timeoutMs: params.subagentAnnounceTimeoutMs, - cleanup: entry.cleanup, + cleanup, roundOneReply: pendingPayload.frozenResultText ?? undefined, fallbackReply: pendingPayload.fallbackFrozenResultText ?? undefined, waitForCompletion: false, @@ -1061,7 +1293,24 @@ export function createSubagentRegistryLifecycleController(params: { spawnMode: pendingPayload.spawnMode, expectsCompletionMessage: pendingPayload.expectsCompletionMessage, wakeOnDescendantSettle: pendingPayload.wakeOnDescendantSettle === true, + onBeforeDeleteChildSession: + cleanup === "delete" + ? () => { + if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { + return false; + } + // Announce owns delete submission; fence late yields at the + // exact handoff instead of when cleanup merely starts. + entry.deleteCleanupDispatchedAt ??= Date.now(); + params.persist(); + return true; + } + : undefined, onDeliveryResult: (delivery) => { + if (!isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) { + void retireSupersededCleanupIfNeeded(runId, entry, cleanupGeneration); + return; + } recordAnnounceDeliveryResult(entry, delivery); if (delivery.delivered) { const deliveryState = ensureDeliveryState(entry); @@ -1094,7 +1343,7 @@ export function createSubagentRegistryLifecycleController(params: { return true; }; - const completeSubagentRun = async (completeParams: { + type CompleteSubagentRunParams = { runId: string; endedAt?: number; outcome: SubagentRunOutcome; @@ -1103,128 +1352,392 @@ export function createSubagentRegistryLifecycleController(params: { accountId?: string; triggerCleanup: boolean; startedAt?: number; - }) => { - params.clearPendingLifecycleError(completeParams.runId); - const entry = params.runs.get(completeParams.runId); + suppressSessionEffects?: boolean; + completionSnapshot?: { resultText: string | null; capturedAt: number }; + }; + + const completeSubagentRun = async (completeParams: CompleteSubagentRunParams) => { + const releaseCompletionLock = await acquireTerminalCompletionLock(completeParams.runId); + let entry: SubagentRunRecord | undefined; + let terminalGeneration = 0; + let mutated = false; + let completionReason = completeParams.reason; + let sessionSuperseded = false; + let suppressTaskFinalization: boolean; + let provisionalKillSnapshot: SubagentRunRecord | undefined; + let postCaptureTaskResolution: DetachedTaskFindResult | undefined; + let entrySnapshot: SubagentRunRecord | undefined; + try { + params.clearPendingLifecycleError(completeParams.runId); + entry = params.runs.get(completeParams.runId); + if (!entry) { + return; + } + const currentEntry = entry; + entrySnapshot = structuredClone(entry); + const restoreEntrySnapshot = (snapshot?: SubagentRunRecord) => { + if (!snapshot) { + return; + } + const target = currentEntry as unknown as Record; + for (const key of Object.keys(target)) { + delete target[key]; + } + Object.assign(target, snapshot); + }; + sessionSuperseded = newerGenerationOwnsSession(currentEntry); + if ( + completeParams.reason === SUBAGENT_ENDED_REASON_KILLED && + entry.endedReason !== undefined && + entry.endedReason !== SUBAGENT_ENDED_REASON_KILLED && + entry.outcome !== undefined + ) { + // Any finalized provider outcome is canonical. A delayed abort listener + // must not replace success, failure, or timeout with a killed marker. + return; + } + let requestedEndedAt = + typeof completeParams.endedAt === "number" ? completeParams.endedAt : Date.now(); + if ( + shouldPreservePublishedExplicitRunTimeout({ + entry, + }) + ) { + return; + } + const shouldDrainExistingTerminal = isOlderEquivalentTerminalCallback({ + entry, + endedAt: requestedEndedAt, + outcome: completeParams.outcome, + reason: completeParams.reason, + }); + if (shouldDrainExistingTerminal) { + // Preserve the newer canonical timing while allowing this duplicate + // caller to rescue a stalled cleanup and delivery tail. + requestedEndedAt = entry.endedAt!; + completionReason = entry.endedReason ?? completeParams.reason; + } + let endedAt = requestedEndedAt; + let completionOutcome = + shouldDrainExistingTerminal && entry.outcome ? entry.outcome : completeParams.outcome; + const observedStartedAt = + !shouldDrainExistingTerminal && + typeof completeParams.startedAt === "number" && + Number.isFinite(completeParams.startedAt) + ? completeParams.startedAt + : undefined; + const expiredDeadlineMs = resolveExpiredExplicitRunDeadlineMs({ + entry, + nextEndedAt: endedAt, + observedStartedAt, + }); + if (expiredDeadlineMs !== undefined) { + endedAt = expiredDeadlineMs; + completionOutcome = { status: "timeout" }; + completionReason = SUBAGENT_ENDED_REASON_COMPLETE; + } + if ( + completionReason !== SUBAGENT_ENDED_REASON_KILLED && + entry.endedReason === SUBAGENT_ENDED_REASON_KILLED && + entry.killReconciliation === undefined + ) { + // Only current-version provisional kills carry reconciliation state. + // Legacy or already-stabilized killed rows are terminal cancellation. + return; + } + const isSteerRestartKill = + completeParams.reason === SUBAGENT_ENDED_REASON_KILLED && + entry.suppressAnnounceReason === "steer-restart"; + suppressTaskFinalization = isSteerRestartKill; + if (completionReason === SUBAGENT_ENDED_REASON_KILLED && !isSteerRestartKill) { + entry.suppressAnnounceReason = "killed"; + entry.killReconciliation ??= { + killedAt: requestedEndedAt, + }; + mutated = true; + } + + if ( + completionReason !== SUBAGENT_ENDED_REASON_KILLED && + entry.endedReason === SUBAGENT_ENDED_REASON_KILLED && + entry.killReconciliation !== undefined + ) { + const killReconciliation = entry.killReconciliation; + const taskResolution = params.resolveSubagentTask(entry); + const stableTaskCancellation = + taskResolution.lookup === "available" && + taskResolution.task?.status === "cancelled" && + !isProvisionalSubagentKillTask(taskResolution.task); + const cancellationEndedAt = resolveKilledSubagentTaskEndedAt(entry); + const completionPredatesCancellation = + typeof cancellationEndedAt === "number" && endedAt < cancellationEndedAt; + if (stableTaskCancellation && !completionPredatesCancellation) { + // tasks.cancel promotes the provisional marker to durable operator + // intent. Only an already-durable earlier completion may reopen it. + return; + } + provisionalKillSnapshot = structuredClone(currentEntry); + // The sweeper uses marker identity to reject a concurrently replaced + // kill generation. A completion rollback must retain the same marker. + provisionalKillSnapshot.killReconciliation = killReconciliation; + // Completion capture yields. Stage the provider result off-registry so + // an unrelated persistence write cannot publish a tentative winner. + entry = structuredClone(currentEntry); + entry.suppressCompletionDelivery = + killReconciliation.suppressTaskDelivery === true ? true : undefined; + entry.suppressAnnounceReason = undefined; + entry.killReconciliation = undefined; + entry.cleanupHandled = false; + entry.cleanupCompletedAt = undefined; + clearDeliveryState(entry); + mutated = true; + } + + if (observedStartedAt !== undefined && entry.startedAt !== observedStartedAt) { + entry.startedAt = observedStartedAt; + if (typeof entry.sessionStartedAt !== "number") { + entry.sessionStartedAt = observedStartedAt; + } + mutated = true; + } + + if ( + completionReason === SUBAGENT_ENDED_REASON_COMPLETE && + completionOutcome.status !== "error" && + provisionalKillSnapshot !== undefined + ) { + // A killed lifecycle may freeze an empty result before the canonical end + // wins. Preserve any reply already captured by an earlier successful callback. + const completion = ensureCompletionState(entry); + const hasCapturedReply = + typeof completion.resultText === "string" && completion.resultText.trim().length > 0; + if ( + !hasCapturedReply && + (completion.resultText !== undefined || completion.capturedAt !== undefined) + ) { + completion.resultText = undefined; + completion.capturedAt = undefined; + mutated = true; + } + } + if (entry.endedAt !== endedAt) { + entry.endedAt = endedAt; + entry.execution = { + ...entry.execution, + status: "terminal", + startedAt: entry.startedAt, + endedAt, + }; + mutated = true; + } + const outcome = withSubagentOutcomeTiming(completionOutcome, { + startedAt: entry.startedAt, + endedAt, + }); + if (shouldUpdateRunOutcome(entry.outcome, outcome)) { + entry.outcome = outcome; + mutated = true; + } + if ( + entry.execution?.status !== "terminal" || + entry.execution.endedAt !== endedAt || + entry.execution.outcome !== outcome + ) { + entry.execution = { + ...entry.execution, + status: "terminal", + startedAt: entry.startedAt, + endedAt, + outcome, + }; + mutated = true; + } + if (entry.endedReason !== completionReason) { + entry.endedReason = completionReason; + mutated = true; + } + if (entry.pauseReason !== undefined) { + entry.pauseReason = undefined; + mutated = true; + } + + if (completeParams.completionSnapshot) { + const completion = ensureCompletionState(entry); + if ( + completion.resultText !== completeParams.completionSnapshot.resultText || + completion.capturedAt !== completeParams.completionSnapshot.capturedAt + ) { + completion.resultText = completeParams.completionSnapshot.resultText; + completion.capturedAt = completeParams.completionSnapshot.capturedAt; + mutated = true; + } + } + + // A newer generation may share the session key. Its transcript/reply is + // not evidence for this older run, so reconcile only the terminal task state. + if (sessionSuperseded) { + const completion = ensureCompletionState(entry); + if (completion.resultText === undefined) { + completion.resultText = null; + completion.capturedAt = Date.now(); + mutated = true; + } + } else { + const didFreezeResult = await freezeRunResultAtCompletion(entry, outcome); + sessionSuperseded = newerGenerationOwnsSession(entry); + if (sessionSuperseded) { + const completion = ensureCompletionState(entry); + completion.resultText = null; + completion.capturedAt = Date.now(); + mutated = true; + } else if (didFreezeResult) { + mutated = true; + } + } + if (provisionalKillSnapshot) { + // Keep the tombstone's superseded generation boundary through task + // commit. Clearing it on the canonical registry row must not let a + // late old-run result select a newer task sharing the session key. + const taskResolution = params.resolveSubagentTask(provisionalKillSnapshot); + postCaptureTaskResolution = taskResolution; + const stableTaskCancellation = + taskResolution.lookup === "available" && + taskResolution.task?.status === "cancelled" && + !isProvisionalSubagentKillTask(taskResolution.task); + const cancellationEndedAt = resolveKilledSubagentTaskEndedAt(provisionalKillSnapshot); + const completionPredatesCancellation = + typeof cancellationEndedAt === "number" && endedAt < cancellationEndedAt; + if (stableTaskCancellation && !completionPredatesCancellation) { + // Cancellation can become durable while completion capture yields. + // The provider transition is staged, so the live tombstone is intact. + return; + } + } + if (refreshPendingFinalDeliveryPayload(entry)) { + mutated = true; + } + + const opaqueTaskArbitration = + provisionalKillSnapshot !== undefined && + postCaptureTaskResolution?.lookup === "unavailable"; + // A steer abort ends one agent run but continues the same detached task. + // The successor must remain able to publish its eventual terminal state. + if (provisionalKillSnapshot) { + const finalizedTasks = safeFinalizeSubagentTaskRun({ + entry, + outcome, + taskResolution: postCaptureTaskResolution, + }); + const taskWasAbsent = + postCaptureTaskResolution?.lookup === "available" && + postCaptureTaskResolution.task === undefined; + if ((!finalizedTasks || finalizedTasks.length === 0) && !taskWasAbsent) { + if (opaqueTaskArbitration) { + // The optional lookup cannot prove cancellation. Let the legacy + // runtime's own finalizer decide whether provider completion won. + return; + } + const latestTaskResolution = params.resolveSubagentTask(provisionalKillSnapshot); + const latestTask = latestTaskResolution.task; + const stableTaskCancellation = + latestTask?.status === "cancelled" && !isProvisionalSubagentKillTask(latestTask); + const cancellationEndedAt = resolveKilledSubagentTaskEndedAt(provisionalKillSnapshot); + const completionPredatesCancellation = + typeof cancellationEndedAt === "number" && endedAt < cancellationEndedAt; + if (stableTaskCancellation && !completionPredatesCancellation) { + return; + } + throw new Error("subagent task projection did not finalize"); + } + + // Task results do not auto-publish for subagents. Commit that durable, + // idempotent projection first: after a crash the persisted kill marker + // can replay it, while the inverse ordering could strand a provisional task. + entry.browserCleanupDispatchedAt ??= currentEntry.browserCleanupDispatchedAt; + if (currentEntry.killReconciliation?.suppressTaskDelivery === true) { + entry.suppressCompletionDelivery = true; + } + const liveBeforeCommit = structuredClone(currentEntry); + restoreEntrySnapshot(entry); + entry = currentEntry; + try { + params.persistOrThrow(); + } catch (error) { + restoreEntrySnapshot(liveBeforeCommit); + throw error; + } + // A provider result supersedes provisional cleanup only after both + // durable owners accept it. Rejected callbacks leave the kill tail live. + cleanupGenerations.set(entry, (cleanupGenerations.get(entry) ?? 0) + 1); + } else { + try { + if (mutated) { + params.persistOrThrow(); + } + } catch (error) { + restoreEntrySnapshot(entrySnapshot); + throw error; + } + if (!suppressTaskFinalization) { + safeFinalizeSubagentTaskRun({ entry, outcome }); + } + } + terminalGeneration = (terminalGenerations.get(entry) ?? 0) + 1; + terminalGenerations.set(entry, terminalGeneration); + } finally { + // Only the canonical state/capture transition is serialized. Cleanup + // remains re-entrant so a stalled browser close cannot strand a duplicate callback. + releaseCompletionLock(); + } + if (!entry) { return; } - - let mutated = false; - if ( - completeParams.reason === SUBAGENT_ENDED_REASON_COMPLETE && - entry.suppressAnnounceReason === "killed" && - (entry.cleanupHandled || typeof entry.cleanupCompletedAt === "number") - ) { - entry.suppressAnnounceReason = undefined; - entry.cleanupHandled = false; - entry.cleanupCompletedAt = undefined; - ensureDeliveryState(entry).announcedAt = undefined; - mutated = true; + if (!isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { + return; } + const retireSupersededSession = async (currentEntry: SubagentRunRecord) => { + if (completionReason !== SUBAGENT_ENDED_REASON_KILLED) { + await params.retireSupersededRun(completeParams.runId, currentEntry); + params.persist(); + } + }; + sessionSuperseded = sessionSuperseded || newerGenerationOwnsSession(entry); + if (sessionSuperseded) { + // This callback belongs to an older run that shared the session key. + // Update only its task projection; the newer generation owns all session effects. + await retireSupersededSession(entry); + return; + } + const isProvisionalKill = entry.killReconciliation !== undefined; - let endedAt = typeof completeParams.endedAt === "number" ? completeParams.endedAt : Date.now(); - let completionOutcome = completeParams.outcome; - let completionReason = completeParams.reason; - if ( - shouldPreservePublishedExplicitRunTimeout({ - entry, - }) - ) { + if (!completeParams.suppressSessionEffects) { + try { + await persistSubagentSessionTiming(entry, { + // Recheck while patchSessionEntry owns its write lock so this old + // completion cannot commit after a synchronous ownership transfer. + isCurrentGeneration: () => + isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration) && + !newerGenerationOwnsSession(entry), + }); + } catch (err) { + params.warn("failed to persist subagent session timing", { + err, + runId: entry.runId, + childSessionKey: entry.childSessionKey, + }); + } + } + if (!isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { + return; + } + if (newerGenerationOwnsSession(entry)) { + await retireSupersededSession(entry); return; } - const observedStartedAt = - typeof completeParams.startedAt === "number" && Number.isFinite(completeParams.startedAt) - ? completeParams.startedAt - : undefined; - if (observedStartedAt !== undefined && entry.startedAt !== observedStartedAt) { - entry.startedAt = observedStartedAt; - if (typeof entry.sessionStartedAt !== "number") { - entry.sessionStartedAt = observedStartedAt; - } - mutated = true; - } - - const expiredDeadlineMs = resolveExpiredExplicitRunDeadlineMs({ - entry, - nextOutcome: completionOutcome, - nextEndedAt: endedAt, - observedStartedAt, - }); - if (expiredDeadlineMs !== undefined) { - endedAt = expiredDeadlineMs; - completionOutcome = { status: "timeout" }; - completionReason = SUBAGENT_ENDED_REASON_COMPLETE; - } - if (entry.endedAt !== endedAt) { - entry.endedAt = endedAt; - entry.execution = { - ...entry.execution, - status: "terminal", - startedAt: entry.startedAt, - endedAt, - }; - mutated = true; - } - const outcome = withSubagentOutcomeTiming(completionOutcome, { - startedAt: entry.startedAt, - endedAt, - }); - if (shouldUpdateRunOutcome(entry.outcome, outcome)) { - entry.outcome = outcome; - mutated = true; - } - if ( - entry.execution?.status !== "terminal" || - entry.execution.endedAt !== endedAt || - entry.execution.outcome !== outcome - ) { - entry.execution = { - ...entry.execution, - status: "terminal", - startedAt: entry.startedAt, - endedAt, - outcome, - }; - mutated = true; - } - if (entry.endedReason !== completionReason) { - entry.endedReason = completionReason; - mutated = true; - } - if (entry.pauseReason !== undefined) { - entry.pauseReason = undefined; - mutated = true; - } - - if (await freezeRunResultAtCompletion(entry, outcome)) { - mutated = true; - } - if (refreshPendingFinalDeliveryPayload(entry)) { - mutated = true; - } - - if (mutated) { - params.persist(); - } - safeFinalizeSubagentTaskRun({ - entry, - outcome, - }); - - try { - await persistSubagentSessionTiming(entry); - } catch (err) { - params.warn("failed to persist subagent session timing", { - err, - runId: entry.runId, - childSessionKey: entry.childSessionKey, - }); - } - const suppressedForSteerRestart = params.suppressAnnounceForSteerRestart(entry); - if (mutated && !suppressedForSteerRestart) { + if (mutated && !suppressedForSteerRestart && !completeParams.suppressSessionEffects) { emitSessionLifecycleEvent({ sessionKey: entry.childSessionKey, reason: "subagent-status", @@ -1234,6 +1747,8 @@ export function createSubagentRegistryLifecycleController(params: { } const shouldEmitEndedHook = !suppressedForSteerRestart && + !isProvisionalKill && + !completeParams.suppressSessionEffects && params.shouldEmitEndedHookForRun({ entry, reason: completionReason, @@ -1249,7 +1764,17 @@ export function createSubagentRegistryLifecycleController(params: { reason: completionReason, sendFarewell: completeParams.sendFarewell, accountId: completeParams.accountId, + isCurrent: () => + isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration) && + !newerGenerationOwnsSession(entry), }); + if (!isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { + return; + } + if (newerGenerationOwnsSession(entry)) { + await retireSupersededSession(entry); + return; + } } if (!completeParams.triggerCleanup || suppressedForSteerRestart) { @@ -1279,6 +1804,13 @@ export function createSubagentRegistryLifecycleController(params: { childSessionKey: maskSessionKey(entry.childSessionKey), }); } + if (!isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { + return; + } + if (newerGenerationOwnsSession(entry)) { + await retireSupersededSession(entry); + return; + } } try { @@ -1294,6 +1826,19 @@ export function createSubagentRegistryLifecycleController(params: { childSessionKey: maskSessionKey(entry.childSessionKey), }); } + if (!isTerminalCallbackCurrent(completeParams.runId, entry, terminalGeneration)) { + return; + } + if (newerGenerationOwnsSession(entry)) { + await retireSupersededSession(entry); + return; + } + + if (isProvisionalKill) { + // Browser and MCP resources can close immediately, but completion delivery + // waits for the provider result or the killed tombstone reconciliation. + return; + } startSubagentAnnounceCleanupFlow(completeParams.runId, entry); }; diff --git a/src/agents/subagent-registry-maintenance.ts b/src/agents/subagent-registry-maintenance.ts index 02d73fed04e..f0b814ecf23 100644 --- a/src/agents/subagent-registry-maintenance.ts +++ b/src/agents/subagent-registry-maintenance.ts @@ -26,6 +26,11 @@ function isAwaitingCompletionAnnounceForMaintenance(entry: SubagentRunRecord): b } function shouldPreserveForMaintenance(entry: SubagentRunRecord): boolean { + if (entry.killReconciliation) { + // The killed row is a reconciliation tombstone. Its session owns the + // provider result until the sweeper accepts completion or finalizes cancellation. + return true; + } if (isCleanupCompleteForMaintenance(entry)) { return false; } diff --git a/src/agents/subagent-registry-queries.test.ts b/src/agents/subagent-registry-queries.test.ts index 8e7ca835f40..37469f37499 100644 --- a/src/agents/subagent-registry-queries.test.ts +++ b/src/agents/subagent-registry-queries.test.ts @@ -36,6 +36,36 @@ function toRunMap(runs: SubagentRunRecord[]): Map { } describe("subagent registry query regressions", () => { + it("selects the newer generation when child runs share a timestamp", () => { + const childSessionKey = "agent:main:subagent:same-millisecond"; + const runs = toRunMap([ + makeRun({ + runId: "run-old-tombstone", + childSessionKey, + requesterSessionKey: "agent:main:old-parent", + generation: 1, + createdAt: 100, + endedAt: 100, + }), + makeRun({ + runId: "run-live-successor", + childSessionKey, + requesterSessionKey: "agent:main:new-parent", + generation: 2, + createdAt: 100, + startedAt: 100, + }), + ]); + + expect(isSubagentSessionRunActiveFromRuns(runs, childSessionKey)).toBe(true); + expect(resolveRequesterForChildSessionFromRuns(runs, childSessionKey)).toMatchObject({ + requesterSessionKey: "agent:main:new-parent", + }); + expect(getSubagentRunByChildSessionKeyFromRuns(runs, childSessionKey)?.runId).toBe( + "run-live-successor", + ); + }); + it("does not treat stale unended rows as active child-session liveness", () => { const now = Date.now(); const childSessionKey = "agent:main:subagent:stale-live-check"; diff --git a/src/agents/subagent-registry-queries.ts b/src/agents/subagent-registry-queries.ts index 0f141fa1fe4..90a23857938 100644 --- a/src/agents/subagent-registry-queries.ts +++ b/src/agents/subagent-registry-queries.ts @@ -5,6 +5,7 @@ */ import type { DeliveryContext } from "../utils/delivery-context.types.js"; import type { SubagentRunRecord } from "./subagent-registry.types.js"; +import { compareSubagentRunGeneration } from "./subagent-run-generation.js"; import { hasSubagentRunEnded, isLiveUnendedSubagentRun } from "./subagent-run-liveness.js"; function resolveControllerSessionKey(entry: SubagentRunRecord): string { @@ -76,7 +77,7 @@ function rememberLatestRunEntry( entry: SubagentRunRecord, ): void { const existing = map.get(key); - if (!existing || entry.createdAt > existing.createdAt) { + if (!existing || compareSubagentRunGeneration(entry, existing) > 0) { map.set(key, entry); } } @@ -88,7 +89,7 @@ function rememberLatestRunPair( entry: SubagentRunRecord, ): void { const existing = map.get(key); - if (!existing || entry.createdAt > existing.entry.createdAt) { + if (!existing || compareSubagentRunGeneration(entry, existing.entry) > 0) { map.set(key, { runId, entry }); } } @@ -126,12 +127,18 @@ export function buildSubagentRunReadIndexFromRuns(params: { inMemoryDisplayByChildSessionKey.set(childSessionKey, display); } if (hasSubagentRunEnded(entry)) { - if (!display.latestInMemoryEnded || entry.createdAt > display.latestInMemoryEnded.createdAt) { + if ( + !display.latestInMemoryEnded || + compareSubagentRunGeneration(entry, display.latestInMemoryEnded) > 0 + ) { display.latestInMemoryEnded = entry; } continue; } - if (!display.latestInMemoryActive || entry.createdAt > display.latestInMemoryActive.createdAt) { + if ( + !display.latestInMemoryActive || + compareSubagentRunGeneration(entry, display.latestInMemoryActive) > 0 + ) { display.latestInMemoryActive = entry; } } @@ -181,7 +188,8 @@ export function buildSubagentRunReadIndexFromRuns(params: { if (latestInMemoryEnded || latestInMemoryActive) { if ( latestInMemoryEnded && - (!latestInMemoryActive || latestInMemoryEnded.createdAt > latestInMemoryActive.createdAt) + (!latestInMemoryActive || + compareSubagentRunGeneration(latestInMemoryEnded, latestInMemoryActive) > 0) ) { return latestInMemoryEnded; } @@ -258,7 +266,7 @@ function findLatestRunForChildSession( if (entry.childSessionKey !== key) { continue; } - if (!latest || entry.createdAt > latest.createdAt) { + if (!latest || compareSubagentRunGeneration(entry, latest) > 0) { latest = entry; } } @@ -291,12 +299,12 @@ export function getSubagentRunByChildSessionKeyFromRuns( continue; } if (isLiveUnendedSubagentRun(entry)) { - if (!latestActive || entry.createdAt > latestActive.createdAt) { + if (!latestActive || compareSubagentRunGeneration(entry, latestActive) > 0) { latestActive = entry; } continue; } - if (!latestEnded || entry.createdAt > latestEnded.createdAt) { + if (!latestEnded || compareSubagentRunGeneration(entry, latestEnded) > 0) { latestEnded = entry; } } @@ -363,7 +371,7 @@ export function countActiveRunsForSessionFromRuns( continue; } const existing = latestByChildSessionKey.get(entry.childSessionKey); - if (!existing || entry.createdAt > existing.createdAt) { + if (!existing || compareSubagentRunGeneration(entry, existing) > 0) { latestByChildSessionKey.set(entry.childSessionKey, entry); } } @@ -403,7 +411,7 @@ function forEachDescendantRun( } const childKey = entry.childSessionKey.trim(); const existing = latestByChildSessionKey.get(childKey); - if (!existing || entry.createdAt > existing[1].createdAt) { + if (!existing || compareSubagentRunGeneration(entry, existing[1]) > 0) { latestByChildSessionKey.set(childKey, [runId, entry]); } } diff --git a/src/agents/subagent-registry-read.ts b/src/agents/subagent-registry-read.ts index b8c266886a0..0b04f900644 100644 --- a/src/agents/subagent-registry-read.ts +++ b/src/agents/subagent-registry-read.ts @@ -15,6 +15,7 @@ import { } from "./subagent-registry-queries.js"; import { getSubagentRunsSnapshotForRead } from "./subagent-registry-state.js"; import type { SubagentRunRecord } from "./subagent-registry.types.js"; +import { compareSubagentRunGeneration } from "./subagent-run-generation.js"; export { getSubagentSessionRuntimeMs, @@ -89,12 +90,12 @@ export function getSessionDisplaySubagentRunByChildSessionKey( continue; } if (typeof entry.endedAt === "number") { - if (!latestInMemoryEnded || entry.createdAt > latestInMemoryEnded.createdAt) { + if (!latestInMemoryEnded || compareSubagentRunGeneration(entry, latestInMemoryEnded) > 0) { latestInMemoryEnded = entry; } continue; } - if (!latestInMemoryActive || entry.createdAt > latestInMemoryActive.createdAt) { + if (!latestInMemoryActive || compareSubagentRunGeneration(entry, latestInMemoryActive) > 0) { latestInMemoryActive = entry; } } @@ -103,7 +104,8 @@ export function getSessionDisplaySubagentRunByChildSessionKey( // Fresh in-memory terminal state is more accurate than an older active snapshot row. if ( latestInMemoryEnded && - (!latestInMemoryActive || latestInMemoryEnded.createdAt > latestInMemoryActive.createdAt) + (!latestInMemoryActive || + compareSubagentRunGeneration(latestInMemoryEnded, latestInMemoryActive) > 0) ) { return latestInMemoryEnded; } @@ -127,7 +129,7 @@ export function getLatestSubagentRunByChildSessionKey( if (entry.childSessionKey !== key) { continue; } - if (!latest || entry.createdAt > latest.createdAt) { + if (!latest || compareSubagentRunGeneration(entry, latest) > 0) { latest = entry; } } diff --git a/src/agents/subagent-registry-run-manager.ts b/src/agents/subagent-registry-run-manager.ts index a6c0c0293d7..b3fab6444fb 100644 --- a/src/agents/subagent-registry-run-manager.ts +++ b/src/agents/subagent-registry-run-manager.ts @@ -7,14 +7,16 @@ import { getRuntimeConfig } from "../config/config.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { callGateway } from "../gateway/call.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; -import { getGlobalHookRunner } from "../plugins/hook-runner-global.js"; -import { createRunningTaskRun } from "../tasks/detached-task-runtime.js"; +import { + SUBAGENT_KILL_TASK_ERROR, + type DetachedTaskFindResult, +} from "../tasks/detached-task-runtime-contract.js"; +import { createRunningTaskRun, finalizeTaskRunByRunId } from "../tasks/detached-task-runtime.js"; import { normalizeDeliveryContext } from "../utils/delivery-context.shared.js"; import type { DeliveryContext } from "../utils/delivery-context.types.js"; import { buildAgentRunTerminalOutcomeFromWaitResult } from "./agent-run-terminal-outcome.js"; import { removeInternalSessionEffectsTranscript } from "./internal-session-effects.js"; import { isRecoverableAgentWaitError, waitForAgentRun } from "./run-wait.js"; -import type { ensureRuntimePluginsLoaded as ensureRuntimePluginsLoadedFn } from "./runtime-plugins.js"; import { type SubagentRunOutcome, withSubagentOutcomeTiming } from "./subagent-announce-output.js"; import { clearDeliveryState, @@ -22,15 +24,14 @@ import { normalizeSubagentRunState, } from "./subagent-delivery-state.js"; import { - SUBAGENT_ENDED_OUTCOME_KILLED, SUBAGENT_ENDED_REASON_COMPLETE, SUBAGENT_ENDED_REASON_ERROR, SUBAGENT_ENDED_REASON_KILLED, type SubagentLifecycleEndedReason, } from "./subagent-lifecycle-events.js"; import { - emitSubagentEndedHookOnce, - shouldUpdateRunOutcome, + resolveFinalizedSubagentTaskState, + resolveKilledSubagentTaskEndedAt, } from "./subagent-registry-completion.js"; import { getSubagentSessionRuntimeMs, @@ -40,6 +41,10 @@ import { safeRemoveAttachmentsDir, } from "./subagent-registry-helpers.js"; import type { SubagentRunRecord } from "./subagent-registry.types.js"; +import { + compareSubagentRunGeneration, + nextSubagentRunGeneration, +} from "./subagent-run-generation.js"; import { resolveSubagentRunDeadlineMs } from "./subagent-run-timeout.js"; import type { SubagentSessionCompletion } from "./subagent-session-reconciliation.js"; @@ -100,6 +105,16 @@ export function markSubagentRunPausedAfterYield(params: { now?: number; }): boolean { const { entry } = params; + if ( + entry.endedReason === SUBAGENT_ENDED_REASON_KILLED || + entry.suppressAnnounceReason === "killed" || + (entry.cleanup === "delete" && Number.isFinite(entry.deleteCleanupDispatchedAt)) + ) { + // agent.wait and lifecycle events can report the old yield after control + // killed the run. Once delete dispatch starts, reviving the row would expose + // a live run whose backing session may already be gone. + return false; + } let mutated = false; if (typeof params.startedAt === "number" && entry.startedAt !== params.startedAt) { entry.startedAt = params.startedAt; @@ -129,6 +144,14 @@ export function markSubagentRunPausedAfterYield(params: { entry.cleanupHandled = false; mutated = true; } + if (entry.cleanupCompletedAt !== undefined) { + entry.cleanupCompletedAt = undefined; + mutated = true; + } + if (entry.delivery !== undefined) { + clearDeliveryState(entry); + mutated = true; + } const completion = ensureCompletionState(entry); if (completion.resultText !== undefined) { completion.resultText = undefined; @@ -165,18 +188,10 @@ export type RegisterSubagentRunParams = { export function createSubagentRunManager(params: { runs: Map; resumedRuns: Set; - endedHookInFlightRunIds: Set; persist(): void; persistOrThrow(): void; callGateway: typeof callGateway; getRuntimeConfig: typeof getRuntimeConfig; - ensureRuntimePluginsLoaded: - | typeof ensureRuntimePluginsLoadedFn - | ((args: { - config: OpenClawConfig; - workspaceDir?: string; - allowGatewaySubagentBinding?: boolean; - }) => void | Promise); ensureListener(): void; startSweeper(): void; stopSweeper(): void; @@ -205,6 +220,8 @@ export function createSubagentRunManager(params: { entry: SubagentRunRecord; cleanup: "delete" | "keep"; completedAt: number; + preserveTranscript?: boolean; + provisionalKill?: boolean; }): void; completeSubagentRun(args: { runId: string; @@ -216,7 +233,45 @@ export function createSubagentRunManager(params: { triggerCleanup: boolean; startedAt?: number; }): Promise; + resolveSubagentTask(entry: SubagentRunRecord): DetachedTaskFindResult; }) { + const markOlderKillReconciliationsSuperseded = (next: SubagentRunRecord) => { + const snapshots = new Map(); + for (const candidate of params.runs.values()) { + if ( + candidate.runId === next.runId || + candidate.childSessionKey !== next.childSessionKey || + compareSubagentRunGeneration(candidate, next) >= 0 || + !candidate.killReconciliation + ) { + continue; + } + snapshots.set(candidate, structuredClone(candidate.killReconciliation)); + candidate.killReconciliation.supersededAt = Math.min( + candidate.killReconciliation.supersededAt ?? next.createdAt, + next.createdAt, + ); + } + return snapshots; + }; + + const currentRunOwnsSession = (entry: SubagentRunRecord): boolean => + params.runs.get(entry.runId) === entry && + entry.killReconciliation?.supersededAt === undefined && + !Array.from(params.runs.values()).some( + (candidate) => + candidate.childSessionKey === entry.childSessionKey && + compareSubagentRunGeneration(candidate, entry) > 0, + ); + + const restoreKillReconciliationSnapshots = ( + snapshots: Map, + ) => { + for (const [entry, snapshot] of snapshots) { + entry.killReconciliation = snapshot; + } + }; + const waitForSubagentCompletion = async ( runId: string, waitTimeoutMs: number, @@ -292,12 +347,6 @@ export function createSubagentRunManager(params: { notBeforeMs: entry.startedAt ?? entry.createdAt, }); const completeAsRunTimeout = async (endedAt?: number, startedAt?: number) => { - if (typeof startedAt === "number" && Number.isFinite(startedAt)) { - entry.startedAt = startedAt; - if (typeof entry.sessionStartedAt !== "number") { - entry.sessionStartedAt = startedAt; - } - } const timeoutCompletion: Parameters[0] = { runId, outcome: { status: "timeout" }, @@ -321,13 +370,6 @@ export function createSubagentRunManager(params: { typeof wait.stopReason === "string" || typeof wait.livenessState === "string"; const now = Date.now(); - if (observedStartedAt !== undefined && entry.startedAt !== observedStartedAt) { - entry.startedAt = observedStartedAt; - if (typeof entry.sessionStartedAt !== "number") { - entry.sessionStartedAt = observedStartedAt; - } - params.persist(); - } // A plain agent.wait timeout has no terminal snapshot. For explicit // subagent run timeouts, the stored run deadline is the completion // contract so parent sessions are woken instead of retrying forever. @@ -378,6 +420,13 @@ export function createSubagentRunManager(params: { await completeAsRunTimeout(timeoutEndedAt, observedStartedAt); return; } + if (observedStartedAt !== undefined && entry.startedAt !== observedStartedAt) { + entry.startedAt = observedStartedAt; + if (typeof entry.sessionStartedAt !== "number") { + entry.sessionStartedAt = observedStartedAt; + } + params.persist(); + } scheduleWaitRetry( entry, "subagent wait timed out; deferring terminal state until session reconciliation", @@ -394,22 +443,7 @@ export function createSubagentRunManager(params: { await completeAsRunTimeout(completionAfterDeadline, observedStartedAt); return; } - let mutated = false; - if (typeof observedStartedAt === "number") { - entry.startedAt = observedStartedAt; - if (typeof entry.sessionStartedAt !== "number") { - entry.sessionStartedAt = observedStartedAt; - } - mutated = true; - } - if (typeof wait.endedAt === "number") { - entry.endedAt = wait.endedAt; - mutated = true; - } - if (!entry.endedAt) { - entry.endedAt = Date.now(); - mutated = true; - } + const endedAt = typeof wait.endedAt === "number" ? wait.endedAt : Date.now(); const rawWaitError = typeof wait.error === "string" ? wait.error : undefined; const waitError = waitAborted ? "subagent run terminated" @@ -417,19 +451,12 @@ export function createSubagentRunManager(params: { const baseOutcome: SubagentRunOutcome = waitStatus === "error" ? { status: "error", error: waitError } : { status: "ok" }; const outcome = withSubagentOutcomeTiming(baseOutcome, { - startedAt: entry.startedAt, - endedAt: entry.endedAt, + startedAt: observedStartedAt ?? entry.startedAt, + endedAt, }); - if (shouldUpdateRunOutcome(entry.outcome, outcome)) { - entry.outcome = outcome; - mutated = true; - } - if (mutated) { - params.persist(); - } completionForRetry = { runId, - endedAt: entry.endedAt, + endedAt, outcome, reason: waitAborted ? SUBAGENT_ENDED_REASON_KILLED @@ -449,27 +476,31 @@ export function createSubagentRunManager(params: { childSessionKey: current?.childSessionKey ?? expectedEntry?.childSessionKey, error, }); + if (!current) { + return; + } + if (completionForRetry) { + try { + await params.completeSubagentRun(completionForRetry); + return; + } catch (retryError) { + log.warn("failed to complete subagent run after retry; retrying ended cleanup", { + runId, + childSessionKey: current.childSessionKey, + error: retryError, + }); + } + } if ( - current && typeof current.endedAt === "number" && !current.cleanupCompletedAt && current.pauseReason !== "sessions_yield" ) { - if (completionForRetry) { - try { - await params.completeSubagentRun(completionForRetry); - return; - } catch (retryError) { - log.warn("failed to complete subagent run after retry; retrying ended cleanup", { - runId, - childSessionKey: current.childSessionKey, - error: retryError, - }); - } - } current.cleanupHandled = false; params.resumedRuns.delete(runId); params.resumeSubagentRun(runId); + } else if (completionForRetry && typeof current.endedAt !== "number") { + params.scheduleOrphanRecovery({ delayMs: 1_000 }); } } }; @@ -503,6 +534,40 @@ export function createSubagentRunManager(params: { if (entry.suppressAnnounceReason !== "steer-restart") { return true; } + if (typeof entry.endedAt === "number") { + const taskResolution = params.resolveSubagentTask(entry); + const task = taskResolution.lookup === "available" ? taskResolution.task : undefined; + const terminal = + entry.endedReason === SUBAGENT_ENDED_REASON_KILLED + ? { + status: "cancelled" as const, + endedAt: entry.endedAt, + lastEventAt: entry.endedAt, + error: "Subagent restart failed after the prior run was interrupted.", + } + : resolveFinalizedSubagentTaskState(entry); + if (terminal) { + const targetRunId = task?.runId ?? entry.taskRunId ?? entry.runId; + const targetSessionKey = task?.childSessionKey ?? entry.childSessionKey; + try { + finalizeTaskRunByRunId({ + runId: targetRunId, + runtime: "subagent", + sessionKey: targetSessionKey, + ...terminal, + suppressDelivery: true, + }); + } catch (err) { + // A task-runtime failure must not leave the interrupted run's + // announcement and cleanup path permanently suppressed. + log.warn("failed to finalize abandoned steer-restart task run", { + err, + runId: targetRunId, + childSessionKey: targetSessionKey, + }); + } + } + } entry.suppressAnnounceReason = undefined; params.persist(); // If the interrupted run already finished while suppression was active, retry @@ -535,22 +600,11 @@ export function createSubagentRunManager(params: { return false; } - if (previousRunId !== nextRunId) { - params.clearPendingLifecycleError(previousRunId); - if (shouldDeleteAttachments(source)) { - void safeRemoveAttachmentsDir(source); - } - if ( - source.execution?.transcriptFile && - source.execution.transcriptFile !== replaceParams.transcriptFile - ) { - void removeInternalSessionEffectsTranscript(source.execution.transcriptFile); - } - params.runs.delete(previousRunId); - params.resumedRuns.delete(previousRunId); - } - const now = Date.now(); + const generation = nextSubagentRunGeneration( + [...params.runs.values(), source], + source.childSessionKey, + ); const cfg = params.getRuntimeConfig(); const archiveAfterMs = resolveArchiveAfterMs(cfg); const spawnMode = source.spawnMode === "session" ? "session" : "run"; @@ -588,7 +642,12 @@ export function createSubagentRunManager(params: { const next: SubagentRunRecord = normalizeSubagentRunState({ ...source, runId: nextRunId, + // New rows carry an exact owner. Legacy replacement rows must retain an + // unknown owner so their bounded session fallback can still find the + // original detached task across another restart. + taskRunId: source.taskRunId, task: nextTask, + generation, createdAt: now, startedAt: now, sessionStartedAt, @@ -598,6 +657,7 @@ export function createSubagentRunManager(params: { pauseReason: undefined, endedHookEmittedAt: undefined, browserCleanupDispatchedAt: undefined, + deleteCleanupDispatchedAt: undefined, wakeOnDescendantSettle: undefined, outcome: undefined, execution: { @@ -613,6 +673,8 @@ export function createSubagentRunManager(params: { cleanupCompletedAt: undefined, cleanupHandled: false, suppressAnnounceReason: undefined, + killReconciliation: undefined, + suppressCompletionDelivery: undefined, delivery: { status: source.expectsCompletionMessage === false ? "not_required" : "pending", }, @@ -622,9 +684,38 @@ export function createSubagentRunManager(params: { }); clearDeliveryState(next); + if (previousRunId !== nextRunId) { + params.runs.delete(previousRunId); + } params.runs.set(nextRunId, next); + markOlderKillReconciliationsSuperseded(next); + try { + params.persistOrThrow(); + } catch (error) { + // The gateway has already started nextRunId. Keep its in-memory owner + // authoritative and retry best-effort persistence; rolling back here + // would orphan a live run that can still mutate the shared session. + log.warn("failed to persist replacement subagent run; retaining live successor", { + error, + previousRunId, + nextRunId, + }); + params.persist(); + } + if (previousRunId !== nextRunId) { + params.clearPendingLifecycleError(previousRunId); + params.resumedRuns.delete(previousRunId); + if (shouldDeleteAttachments(source)) { + void safeRemoveAttachmentsDir(source); + } + if ( + source.execution?.transcriptFile && + source.execution.transcriptFile !== replaceParams.transcriptFile + ) { + void removeInternalSessionEffectsTranscript(source.execution.transcriptFile); + } + } params.ensureListener(); - params.persist(); // Always start sweeper — session-mode runs (no archiveAtMs) also need TTL cleanup. params.startSweeper(); void waitForSubagentCompletion(nextRunId, waitTimeoutMs, next); @@ -640,6 +731,7 @@ export function createSubagentRunManager(params: { return; } const now = Date.now(); + const generation = nextSubagentRunGeneration(params.runs.values(), childSessionKey); const cfg = params.getRuntimeConfig(); const archiveAfterMs = resolveArchiveAfterMs(cfg); const spawnMode = registerParams.spawnMode === "session" ? "session" : "run"; @@ -654,6 +746,7 @@ export function createSubagentRunManager(params: { const requesterOrigin = normalizeDeliveryContext(registerParams.requesterOrigin); const entry: SubagentRunRecord = normalizeSubagentRunState({ runId, + taskRunId: runId, childSessionKey, controllerSessionKey, requesterSessionKey, @@ -669,6 +762,7 @@ export function createSubagentRunManager(params: { agentDir: registerParams.agentDir, workspaceDir: registerParams.workspaceDir, runTimeoutSeconds, + generation, createdAt: now, startedAt: now, execution: { @@ -691,10 +785,12 @@ export function createSubagentRunManager(params: { retainAttachmentsOnKeep: registerParams.retainAttachmentsOnKeep, }); params.runs.set(runId, entry); + const killReconciliationSnapshots = markOlderKillReconciliationsSuperseded(entry); try { params.persistOrThrow(); } catch (error) { params.runs.delete(runId); + restoreKillReconciliationSnapshots(killReconciliationSnapshots); throw error; } try { @@ -762,6 +858,7 @@ export function createSubagentRunManager(params: { runId?: string; childSessionKey?: string; reason?: string; + suppressTaskDelivery?: boolean; }): number => { const runIds = new Set(); if (typeof markParams.runId === "string" && markParams.runId.trim()) { @@ -782,47 +879,120 @@ export function createSubagentRunManager(params: { const reason = markParams.reason?.trim() || "killed"; let updated = 0; const entriesByChildSessionKey = new Map(); + const entrySnapshots = new Map(); + const pendingTaskFinalizations: Array<{ entry: SubagentRunRecord; endedAt: number }> = []; + const finalizeKilledTask = (entry: SubagentRunRecord, endedAt: number) => { + const taskResolution = params.resolveSubagentTask(entry); + const task = taskResolution.lookup === "available" ? taskResolution.task : undefined; + const targetRunId = task?.runId ?? entry.taskRunId ?? entry.runId; + const targetSessionKey = task?.childSessionKey ?? entry.childSessionKey; + try { + finalizeTaskRunByRunId({ + runId: targetRunId, + runtime: "subagent", + sessionKey: targetSessionKey, + status: "cancelled", + endedAt, + lastEventAt: endedAt, + error: SUBAGENT_KILL_TASK_ERROR, + suppressDelivery: entry.killReconciliation?.suppressTaskDelivery === true, + }); + } catch (err) { + log.warn("failed to finalize killed subagent task run", { + err, + runId: targetRunId, + childSessionKey: targetSessionKey, + }); + } + }; for (const runId of runIds) { params.clearPendingLifecycleError(runId); + params.clearPendingLifecycleTimeout(runId); const entry = params.runs.get(runId); if (!entry) { continue; } - if (typeof entry.endedAt === "number") { + const wasKilledLifecycle = + entry.endedReason === SUBAGENT_ENDED_REASON_KILLED && + entry.killReconciliation !== undefined; + const existingKillReconciliation = entry.killReconciliation; + if ( + typeof entry.endedAt === "number" && + entry.pauseReason !== "sessions_yield" && + !wasKilledLifecycle + ) { + // An abort lifecycle event can mark the run killed before this shared + // termination path runs. Re-enter only for that provisional state so + // it receives the same reconciliation tombstone as a direct kill. continue; } - entry.endedAt = now; + entrySnapshots.set(entry, structuredClone(entry)); + const wasYielded = entry.pauseReason === "sessions_yield"; + const endedAt = + (wasYielded || wasKilledLifecycle) && typeof entry.endedAt === "number" + ? entry.endedAt + : now; + entry.endedAt = endedAt; entry.outcome = withSubagentOutcomeTiming( { status: "error", error: reason }, { startedAt: entry.startedAt, - endedAt: now, + endedAt, }, ); entry.endedReason = SUBAGENT_ENDED_REASON_KILLED; entry.cleanupHandled = true; - entry.cleanupCompletedAt = now; + entry.cleanupCompletedAt = existingKillReconciliation + ? (entry.cleanupCompletedAt ?? endedAt) + : wasKilledLifecycle + ? endedAt + : now; entry.suppressAnnounceReason = "killed"; + entry.pauseReason = undefined; + // Setting endedAt above short-circuits the completion watcher, so the + // lifecycle finalizer never reaches the detached task row for killed runs. + const taskEndedAt = existingKillReconciliation + ? (resolveKilledSubagentTaskEndedAt(entry) ?? endedAt) + : wasYielded + ? now + : endedAt; + entry.killReconciliation = { + killedAt: existingKillReconciliation?.killedAt ?? taskEndedAt, + suppressTaskDelivery: + existingKillReconciliation?.suppressTaskDelivery === true || + markParams.suppressTaskDelivery === true + ? true + : undefined, + supersededAt: existingKillReconciliation?.supersededAt, + }; + pendingTaskFinalizations.push({ entry, endedAt: taskEndedAt }); if (!entriesByChildSessionKey.has(entry.childSessionKey)) { entriesByChildSessionKey.set(entry.childSessionKey, entry); } updated += 1; } if (updated > 0) { - params.persist(); + try { + // The registry tombstone is the recovery source for the provisional + // task marker. It must commit first so the sweeper can always finish it. + params.persistOrThrow(); + } catch (error) { + for (const [entry, snapshot] of entrySnapshots) { + const target = entry as unknown as Record; + for (const key of Object.keys(target)) { + delete target[key]; + } + Object.assign(target, snapshot); + } + throw error; + } + for (const pending of pendingTaskFinalizations) { + finalizeKilledTask(pending.entry, pending.endedAt); + } for (const entry of entriesByChildSessionKey.values()) { - const emitEndedHook = () => - emitSubagentEndedHookOnce({ - entry, - reason: SUBAGENT_ENDED_REASON_KILLED, - sendFarewell: true, - accountId: entry.requesterOrigin?.accountId, - outcome: SUBAGENT_ENDED_OUTCOME_KILLED, - error: reason, - inFlightRunIds: params.endedHookInFlightRunIds, - persist: () => params.persist(), - }); - void persistSubagentSessionTiming(entry).catch((err: unknown) => { + void persistSubagentSessionTiming(entry, { + isCurrentGeneration: () => currentRunOwnsSession(entry), + }).catch((err: unknown) => { log.warn("failed to persist killed subagent session timing", { err, runId: entry.runId, @@ -835,27 +1005,13 @@ export function createSubagentRunManager(params: { params.completeCleanupBookkeeping({ runId: entry.runId, entry, - cleanup: entry.cleanup, + // A direct kill is provisional until the runner reports its final + // outcome. Keep delete-mode rows as reconciliation tombstones. + cleanup: "keep", completedAt: now, + preserveTranscript: true, + provisionalKill: true, }); - if (getGlobalHookRunner()) { - void emitEndedHook().catch(() => { - // Hook failures should not break termination flow. - }); - continue; - } - const cfg = params.getRuntimeConfig(); - void Promise.resolve( - params.ensureRuntimePluginsLoaded({ - config: cfg, - workspaceDir: entry.workspaceDir, - allowGatewaySubagentBinding: true, - }), - ) - .then(emitEndedHook) - .catch(() => { - // Hook failures should not break termination flow. - }); } } return updated; diff --git a/src/agents/subagent-registry.archive.e2e.test.ts b/src/agents/subagent-registry.archive.e2e.test.ts index 3dde791774f..d2c6d71034f 100644 --- a/src/agents/subagent-registry.archive.e2e.test.ts +++ b/src/agents/subagent-registry.archive.e2e.test.ts @@ -5,6 +5,20 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { callGateway } from "../gateway/call.js"; +import { SUBAGENT_KILL_TASK_ERROR } from "../tasks/detached-task-runtime-contract.js"; +import { + getDetachedTaskLifecycleRuntime, + resetDetachedTaskLifecycleRuntimeForTests, + setDetachedTaskLifecycleRuntime, +} from "../tasks/detached-task-runtime.js"; + +const taskRuntimeMocks = vi.hoisted(() => ({ + finalizeTaskRunByRunId: vi.fn<(_params: unknown) => unknown[]>(() => [{}]), +})); +const taskStatusMocks = vi.hoisted(() => ({ + findTaskByRunIdForStatus: vi.fn(), + listTasksForSessionKeyForStatus: vi.fn(() => [] as never[]), +})); const noop = () => {}; let currentConfig = { @@ -29,6 +43,19 @@ vi.mock("../gateway/call.js", () => ({ }), })); +vi.mock("../tasks/detached-task-runtime.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + finalizeTaskRunByRunId: taskRuntimeMocks.finalizeTaskRunByRunId, + }; +}); + +vi.mock("../tasks/task-status-access.js", () => ({ + findTaskByRunIdForStatus: taskStatusMocks.findTaskByRunIdForStatus, + listTasksForSessionKeyForStatus: taskStatusMocks.listTasksForSessionKeyForStatus, +})); + vi.mock("../infra/agent-events.js", () => ({ getAgentRunContext: vi.fn(() => undefined), onAgentEvent: vi.fn((_handler: unknown) => noop), @@ -68,11 +95,19 @@ describe("subagent registry archive behavior", () => { mod.testing.setDepsForTest({ callGateway, getRuntimeConfig: loadConfigMock as typeof import("../config/config.js").getRuntimeConfig, + ensureRuntimePluginsLoaded: vi.fn(), ...overrides, }); }; + const waitForNoRequesterRuns = async () => { + await vi.waitFor(() => { + expect(mod.listSubagentRunsForRequester("agent:main:main")).toHaveLength(0); + }); + }; + beforeEach(() => { + resetDetachedTaskLifecycleRuntimeForTests(); vi.useFakeTimers(); vi.setSystemTime(new Date("2026-01-01T00:00:00Z")); currentConfig = { @@ -88,11 +123,32 @@ describe("subagent registry archive behavior", () => { return {}; }); loadConfigMock.mockClear(); + taskRuntimeMocks.finalizeTaskRunByRunId.mockClear(); + taskStatusMocks.findTaskByRunIdForStatus.mockReset(); + taskStatusMocks.listTasksForSessionKeyForStatus.mockReset(); + taskStatusMocks.listTasksForSessionKeyForStatus.mockReturnValue([]); + taskStatusMocks.findTaskByRunIdForStatus.mockImplementation((runId: string) => { + const entry = mod + .listSubagentRunsForRequester("agent:main:main") + .find((candidate) => candidate.runId === runId); + return entry + ? ({ + taskId: `task-${runId}`, + runId, + runtime: "subagent", + childSessionKey: entry.childSessionKey, + createdAt: entry.createdAt, + status: "cancelled", + error: SUBAGENT_KILL_TASK_ERROR, + } as never) + : undefined; + }); setRegistryTestDeps(); mod.resetSubagentRegistryForTests({ persist: false }); }); afterEach(() => { + resetDetachedTaskLifecycleRuntimeForTests(); mod.testing.setDepsForTest(); mod.resetSubagentRegistryForTests({ persist: false }); vi.useRealTimers(); @@ -133,7 +189,7 @@ describe("subagent registry archive behavior", () => { await vi.advanceTimersByTimeAsync(60_000); - expect(mod.listSubagentRunsForRequester("agent:main:main")).toHaveLength(0); + await waitForNoRequesterRuns(); }); it("keeps archived delete-mode runs for retry when sessions.delete fails", async () => { @@ -194,6 +250,438 @@ describe("subagent registry archive behavior", () => { expect(mod.listSubagentRunsForRequester("agent:main:main")).toHaveLength(0); }); + it("stabilizes provisional killed tasks before deleting expired tombstones", async () => { + const now = Date.now(); + mod.addSubagentRunForTests({ + runId: "run-killed-tombstone-expired", + childSessionKey: "agent:main:subagent:killed-tombstone-expired", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "expire killed tombstone", + cleanup: "delete", + createdAt: now - 10 * 60_000, + startedAt: now - 10 * 60_000, + endedAt: now - 5 * 60_000, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: now - 5 * 60_000 }, + cleanupHandled: true, + cleanupCompletedAt: now - 5 * 60_000, + archiveAtMs: now, + }); + + await mod.testing.sweepOnceForTests(); + await flushSweepMicrotasks(); + + expect(taskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalledWith({ + runId: "run-killed-tombstone-expired", + runtime: "subagent", + sessionKey: "agent:main:subagent:killed-tombstone-expired", + status: "cancelled", + endedAt: now - 5 * 60_000, + lastEventAt: now - 5 * 60_000, + error: "manual kill", + suppressDelivery: true, + }); + await waitForNoRequesterRuns(); + }); + + it("retains expired tombstones when provisional task finalization is rejected", async () => { + const now = Date.now(); + taskRuntimeMocks.finalizeTaskRunByRunId.mockReturnValueOnce([]); + mod.addSubagentRunForTests({ + runId: "run-killed-tombstone-retry", + childSessionKey: "agent:main:subagent:killed-tombstone-retry", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "retry killed tombstone", + cleanup: "delete", + createdAt: now - 10 * 60_000, + endedAt: now - 5 * 60_000, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: now - 5 * 60_000 }, + cleanupHandled: true, + cleanupCompletedAt: now - 5 * 60_000, + archiveAtMs: now, + }); + + await mod.testing.sweepOnceForTests(); + + expect(mod.listSubagentRunsForRequester("agent:main:main")).toHaveLength(1); + expect( + vi + .mocked(callGateway) + .mock.calls.some( + ([request]) => (request as { method?: string } | undefined)?.method === "sessions.delete", + ), + ).toBe(false); + }); + + it("retires expired tombstones when their task row is already gone", async () => { + const now = Date.now(); + taskStatusMocks.findTaskByRunIdForStatus.mockReturnValue(undefined); + mod.addSubagentRunForTests({ + runId: "run-killed-task-missing", + childSessionKey: "agent:main:subagent:killed-task-missing", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "retire missing task tombstone", + cleanup: "keep", + createdAt: now - 10 * 60_000, + endedAt: now - 5 * 60_000, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: now - 5 * 60_000 }, + cleanupHandled: true, + cleanupCompletedAt: now - 5 * 60_000, + archiveAtMs: now, + }); + + await mod.testing.sweepOnceForTests(); + await flushSweepMicrotasks(); + + expect(taskRuntimeMocks.finalizeTaskRunByRunId).not.toHaveBeenCalled(); + await waitForNoRequesterRuns(); + }); + + it("preserves stable operator cancellation when retiring expired tombstones", async () => { + const now = Date.now(); + taskStatusMocks.findTaskByRunIdForStatus.mockReturnValue({ + taskId: "task-killed-operator-cancelled", + runId: "run-killed-operator-cancelled", + runtime: "subagent", + childSessionKey: "agent:main:subagent:killed-operator-cancelled", + createdAt: now - 10 * 60_000, + status: "cancelled", + error: "Cancelled by operator.", + } as never); + mod.addSubagentRunForTests({ + runId: "run-killed-operator-cancelled", + childSessionKey: "agent:main:subagent:killed-operator-cancelled", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "preserve operator cancellation", + cleanup: "keep", + createdAt: now - 10 * 60_000, + endedAt: now - 5 * 60_000, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: now - 5 * 60_000 }, + cleanupHandled: true, + cleanupCompletedAt: now - 5 * 60_000, + archiveAtMs: now, + }); + + await mod.testing.sweepOnceForTests(); + await flushSweepMicrotasks(); + + expect(taskRuntimeMocks.finalizeTaskRunByRunId).not.toHaveBeenCalled(); + await waitForNoRequesterRuns(); + }); + + it("keeps stable cancellation tombstones through the completion grace window", async () => { + const now = Date.now(); + taskStatusMocks.findTaskByRunIdForStatus.mockReturnValue({ + taskId: "task-killed-grace", + runId: "run-killed-grace", + runtime: "subagent", + childSessionKey: "agent:main:subagent:killed-grace", + createdAt: now - 2 * 60_000, + status: "cancelled", + error: "Cancelled by operator.", + } as never); + mod.addSubagentRunForTests({ + runId: "run-killed-grace", + childSessionKey: "agent:main:subagent:killed-grace", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "retain cancellation evidence", + cleanup: "keep", + createdAt: now - 2 * 60_000, + endedAt: now - 60_000, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: now - 60_000 }, + cleanupHandled: true, + cleanupCompletedAt: now - 60_000, + archiveAtMs: now, + }); + + await mod.testing.sweepOnceForTests(); + + expect(mod.listSubagentRunsForRequester("agent:main:main")).toHaveLength(1); + expect(taskRuntimeMocks.finalizeTaskRunByRunId).not.toHaveBeenCalled(); + }); + + it("retires expired tombstones after an opaque legacy runtime finalizer misses", async () => { + const legacyRuntime = { ...getDetachedTaskLifecycleRuntime() }; + delete legacyRuntime.findTaskRun; + setDetachedTaskLifecycleRuntime(legacyRuntime); + taskStatusMocks.findTaskByRunIdForStatus.mockReturnValue(undefined); + taskStatusMocks.listTasksForSessionKeyForStatus.mockReturnValue([]); + taskRuntimeMocks.finalizeTaskRunByRunId.mockReturnValueOnce([]); + const now = Date.now(); + mod.addSubagentRunForTests({ + runId: "run-killed-opaque-runtime", + childSessionKey: "agent:main:subagent:killed-opaque-runtime", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "bound opaque runtime reconciliation", + cleanup: "keep", + createdAt: now - 10 * 60_000, + endedAt: now - 5 * 60_000, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: now - 5 * 60_000 }, + cleanupHandled: true, + cleanupCompletedAt: now - 5 * 60_000, + archiveAtMs: now, + }); + + await mod.testing.sweepOnceForTests(); + await flushSweepMicrotasks(); + + expect(taskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalled(); + await waitForNoRequesterRuns(); + }); + + it("stabilizes replacement runs through their durable task session scope", async () => { + const now = Date.now(); + taskStatusMocks.findTaskByRunIdForStatus.mockReturnValue(undefined); + taskStatusMocks.listTasksForSessionKeyForStatus.mockReturnValue([ + { + taskId: "task-before-replacement", + runId: "run-before-replacement", + runtime: "subagent", + childSessionKey: "agent:main:subagent:replacement", + status: "running", + createdAt: now - 11 * 60_000, + }, + ] as never); + mod.addSubagentRunForTests({ + runId: "run-after-replacement", + childSessionKey: "agent:main:subagent:replacement", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "stabilize replacement task", + cleanup: "keep", + createdAt: now - 10 * 60_000, + sessionStartedAt: now - 11 * 60_000, + endedAt: now - 5 * 60_000, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: now - 5 * 60_000 }, + cleanupHandled: true, + cleanupCompletedAt: now - 5 * 60_000, + archiveAtMs: now, + }); + + await mod.testing.sweepOnceForTests(); + await flushSweepMicrotasks(); + + expect(taskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalledWith( + expect.objectContaining({ + runId: "run-before-replacement", + sessionKey: "agent:main:subagent:replacement", + }), + ); + await waitForNoRequesterRuns(); + }); + + it("directly kills a replacement run through its durable task ID", () => { + const now = Date.now(); + const childSessionKey = "agent:main:subagent:replacement-direct-kill"; + taskStatusMocks.findTaskByRunIdForStatus.mockReturnValue(undefined); + taskStatusMocks.listTasksForSessionKeyForStatus.mockReturnValue([ + { + taskId: "task-before-replacement-direct-kill", + runId: "run-before-replacement-direct-kill", + runtime: "subagent", + childSessionKey, + status: "running", + createdAt: now - 11 * 60_000, + }, + ] as never); + mod.addSubagentRunForTests({ + runId: "run-after-replacement-direct-kill", + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "kill replacement task", + cleanup: "keep", + createdAt: now - 10 * 60_000, + sessionStartedAt: now - 11 * 60_000, + }); + + expect( + mod.markSubagentRunTerminated({ + runId: "run-after-replacement-direct-kill", + reason: "manual kill", + }), + ).toBe(1); + + expect(taskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalledWith( + expect.objectContaining({ + runId: "run-before-replacement-direct-kill", + sessionKey: childSessionKey, + status: "cancelled", + }), + ); + }); + + it("does not reconcile an older tombstone through a newer session task", async () => { + const now = Date.now(); + taskStatusMocks.findTaskByRunIdForStatus.mockReturnValue(undefined); + taskStatusMocks.listTasksForSessionKeyForStatus.mockReturnValue([ + { + taskId: "task-new-generation", + runId: "run-new-generation", + runtime: "subagent", + childSessionKey: "agent:main:subagent:reused-session", + status: "running", + createdAt: now - 60_000, + }, + ] as never); + mod.addSubagentRunForTests({ + runId: "run-old-generation", + childSessionKey: "agent:main:subagent:reused-session", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "expire old generation", + cleanup: "keep", + createdAt: now - 10 * 60_000, + sessionStartedAt: now - 10 * 60_000, + endedAt: now - 5 * 60_000, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { + killedAt: now - 5 * 60_000, + supersededAt: now - 60_000, + }, + cleanupHandled: true, + cleanupCompletedAt: now - 5 * 60_000, + }); + + await mod.testing.sweepOnceForTests(); + await flushSweepMicrotasks(); + + expect(taskRuntimeMocks.finalizeTaskRunByRunId).not.toHaveBeenCalled(); + await waitForNoRequesterRuns(); + }); + + it("retires expired keep-mode reconciliation rows without deleting their sessions", async () => { + const now = Date.now(); + mod.addSubagentRunForTests({ + runId: "run-killed-keep-expired", + childSessionKey: "agent:main:subagent:killed-keep-expired", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "stabilize retained session kill", + cleanup: "keep", + createdAt: now - 10 * 60_000, + endedAt: now - 5 * 60_000, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: now - 5 * 60_000 }, + cleanupHandled: true, + cleanupCompletedAt: now - 5 * 60_000, + archiveAtMs: now, + }); + + await mod.testing.sweepOnceForTests(); + await flushSweepMicrotasks(); + + expect(taskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalled(); + await waitForNoRequesterRuns(); + expect( + vi + .mocked(callGateway) + .mock.calls.some( + ([request]) => (request as { method?: string } | undefined)?.method === "sessions.delete", + ), + ).toBe(false); + }); + + it("stabilizes killed tasks before their configured session archive deadline", async () => { + const now = Date.now(); + const archiveAtMs = now + 55 * 60_000; + mod.addSubagentRunForTests({ + runId: "run-killed-retained-session", + childSessionKey: "agent:main:subagent:killed-retained-session", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "stabilize before archive", + cleanup: "delete", + createdAt: now - 10 * 60_000, + endedAt: now - 5 * 60_000, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: now - 5 * 60_000 }, + cleanupHandled: true, + cleanupCompletedAt: now - 5 * 60_000, + archiveAtMs, + }); + + await mod.testing.sweepOnceForTests(); + + expect(taskRuntimeMocks.finalizeTaskRunByRunId).toHaveBeenCalled(); + expect(mod.listSubagentRunsForRequester("agent:main:main")).toEqual([ + expect.objectContaining({ + runId: "run-killed-retained-session", + archiveAtMs, + suppressAnnounceReason: undefined, + }), + ]); + expect( + vi + .mocked(callGateway) + .mock.calls.some( + ([request]) => (request as { method?: string } | undefined)?.method === "sessions.delete", + ), + ).toBe(false); + }); + + it("continues killed cleanup when ended hook loading fails", async () => { + const now = Date.now(); + setRegistryTestDeps({ + ensureRuntimePluginsLoaded: vi.fn(() => { + throw new Error("plugin load failed"); + }), + }); + mod.addSubagentRunForTests({ + runId: "run-killed-hook-load-failure", + childSessionKey: "agent:main:subagent:killed-hook-load-failure", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "cleanup despite hook failure", + cleanup: "keep", + createdAt: now - 10 * 60_000, + endedAt: now - 5 * 60_000, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: now - 5 * 60_000 }, + cleanupHandled: true, + cleanupCompletedAt: now - 5 * 60_000, + }); + + await expect(mod.testing.sweepOnceForTests()).resolves.toBeUndefined(); + await flushSweepMicrotasks(); + + await waitForNoRequesterRuns(); + }); + it("does not overlap archive sweep retries while sessions.delete is still in flight", async () => { currentConfig = { agents: { defaults: { subagents: { archiveAfterMinutes: 1 } } }, diff --git a/src/agents/subagent-registry.persistence.test.ts b/src/agents/subagent-registry.persistence.test.ts index 5eb7b15eb45..739f5b0e669 100644 --- a/src/agents/subagent-registry.persistence.test.ts +++ b/src/agents/subagent-registry.persistence.test.ts @@ -271,6 +271,90 @@ describe("subagent registry persistence", () => { expect(persisted?.startedAt).toBeLessThanOrEqual(endedAt); }); + it("rejects a stale timing write after session ownership changes", async () => { + tempStateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-subagent-")); + setTestEnvValue("OPENCLAW_STATE_DIR", tempStateDir); + + const startedAt = Date.now(); + const storePath = await writeChildSessionEntry({ + sessionKey: "agent:main:subagent:stale-timing", + sessionId: "sess-stale-timing", + updatedAt: startedAt - 1, + }); + await persistSubagentSessionTiming( + { + runId: "run-stale-timing", + childSessionKey: "agent:main:subagent:stale-timing", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "do not persist stale timing", + cleanup: "keep", + createdAt: startedAt, + startedAt, + endedAt: startedAt + 500, + outcome: { status: "ok" }, + } as never, + { isCurrentGeneration: () => false }, + ); + + const persisted = (await readSubagentSessionStore(storePath))[ + "agent:main:subagent:stale-timing" + ]; + expect(persisted).toMatchObject({ + sessionId: "sess-stale-timing", + updatedAt: startedAt - 1, + }); + expect(persisted?.startedAt).toBeUndefined(); + expect(persisted?.endedAt).toBeUndefined(); + expect(persisted?.status).toBeUndefined(); + }); + + it("does not overwrite durable completion with a provisional killed status", async () => { + tempStateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-subagent-")); + setTestEnvValue("OPENCLAW_STATE_DIR", tempStateDir); + + const startedAt = Date.now(); + const completedAt = startedAt + 500; + const storePath = await writeChildSessionEntry({ + sessionKey: "agent:main:subagent:kill-race", + sessionId: "sess-kill-race", + updatedAt: completedAt, + }); + const store = await readSubagentSessionStore(storePath); + store["agent:main:subagent:kill-race"] = { + ...store["agent:main:subagent:kill-race"], + status: "done", + startedAt, + endedAt: completedAt, + runtimeMs: 500, + abortedLastRun: true, + }; + await fs.writeFile(storePath, `${JSON.stringify(store)}\n`, "utf8"); + + await persistSubagentSessionTiming({ + runId: "run-kill-race", + childSessionKey: "agent:main:subagent:kill-race", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "preserve completion", + cleanup: "keep", + createdAt: startedAt, + startedAt, + endedAt: completedAt + 1, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + } as never); + + const persisted = (await readSubagentSessionStore(storePath))["agent:main:subagent:kill-race"]; + expect(persisted).toMatchObject({ + status: "done", + startedAt, + endedAt: completedAt, + runtimeMs: 500, + }); + expect(persisted?.abortedLastRun).toBeUndefined(); + }); + it("skips cleanup when cleanupHandled was persisted", async () => { tempStateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-subagent-")); setTestEnvValue("OPENCLAW_STATE_DIR", tempStateDir); @@ -691,6 +775,48 @@ describe("subagent registry persistence", () => { expect(listSubagentRunsForRequester("agent:main:main")).toHaveLength(0); }); + it("preserves restored killed tombstones until bounded reconciliation", async () => { + const now = Date.now(); + const runId = "run-killed-restore-tombstone"; + await writePersistedRegistry( + { + version: 2, + runs: { + [runId]: { + runId, + childSessionKey: "agent:main:subagent:killed-restore-tombstone", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "restore killed tombstone", + cleanup: "keep", + createdAt: now - 100, + startedAt: now - 50, + endedAt: now, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: now }, + cleanupHandled: true, + cleanupCompletedAt: now, + }, + }, + }, + { seedChildSessions: false }, + ); + + restartRegistry(); + await flushQueuedRegistryWork(); + + expect(announceSpy).not.toHaveBeenCalled(); + expect(listSubagentRunsForRequester("agent:main:main")).toEqual([ + expect.objectContaining({ + runId, + endedReason: "subagent-killed", + suppressAnnounceReason: "killed", + }), + ]); + }); + it("reconciles stale unended restored runs that are not restart-recoverable", async () => { const now = Date.now(); const runId = "run-stale-unended-restore"; diff --git a/src/agents/subagent-registry.steer-restart.test.ts b/src/agents/subagent-registry.steer-restart.test.ts index 546fdd090b3..045b0cc139f 100644 --- a/src/agents/subagent-registry.steer-restart.test.ts +++ b/src/agents/subagent-registry.steer-restart.test.ts @@ -2,6 +2,11 @@ // commands while preserving lifecycle hooks and completion delivery. import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import type { ContextEngine } from "../context-engine/types.js"; +import { + getDetachedTaskLifecycleRuntime, + setDetachedTaskLifecycleRuntime, +} from "../tasks/detached-task-runtime.js"; +import { findTaskByRunIdForStatus } from "../tasks/task-status-access.js"; const noop = () => {}; let lifecycleHandler: @@ -14,6 +19,7 @@ let lifecycleHandler: endedAt?: number; aborted?: boolean; error?: string; + stopReason?: string; }; }) => void) | undefined; @@ -275,6 +281,7 @@ describe("subagent registry steer restarts", () => { endedAt?: number; aborted?: boolean; error?: string; + stopReason?: string; } = {}, ) => { lifecycleHandler?.({ @@ -565,6 +572,8 @@ describe("subagent registry steer restarts", () => { const previous = listMainRuns()[0]; expect(previous?.runId).toBe("run-steer-task-old"); + expect(previous?.taskRunId).toBe("run-steer-task-old"); + expect(previous?.generation).toBe(1); const run = replaceRunAfterSteer({ previousRunId: "run-steer-task-old", @@ -574,6 +583,31 @@ describe("subagent registry steer restarts", () => { }); expect(run.task).toBe("new steer instruction from user"); + expect(run.taskRunId).toBe("run-steer-task-old"); + expect(run.generation).toBe(2); + }); + + it("advances the generation from a fallback outside the live registry", () => { + registerRun({ + runId: "run-fallback-generation-old", + childSessionKey: "agent:main:subagent:fallback-generation", + task: "restored replacement source", + }); + const fallback = listMainRuns()[0]; + expect(fallback?.runId).toBe("run-fallback-generation-old"); + if (!fallback) { + throw new Error("expected fallback run"); + } + fallback.generation = 2; + mod.releaseSubagentRun(fallback.runId); + + const run = replaceRunAfterSteer({ + previousRunId: fallback.runId, + nextRunId: "run-fallback-generation-new", + fallback, + }); + + expect(run.generation).toBe(3); }); it("preserves the previous task when no replacement is provided", () => { @@ -598,6 +632,29 @@ describe("subagent registry steer restarts", () => { expect(run.task).toBe("preserve me verbatim"); }); + it("retains a legacy task owner fallback across another restart", () => { + registerRun({ + runId: "run-legacy-owner-original", + childSessionKey: "agent:main:subagent:legacy-owner", + task: "legacy owner task", + }); + const first = replaceRunAfterSteer({ + previousRunId: "run-legacy-owner-original", + nextRunId: "run-legacy-owner-restored", + }); + // Pre-change persisted replacement rows did not record taskRunId. + first.taskRunId = undefined; + first.sessionStartedAt = first.createdAt - 1; + + const second = replaceRunAfterSteer({ + previousRunId: "run-legacy-owner-restored", + nextRunId: "run-legacy-owner-next", + fallback: first, + }); + expect(second.taskRunId).toBeUndefined(); + expect(second.generation).toBe(3); + }); + it("preserves cumulative session timing across steer replacement runs", () => { registerRun({ runId: "run-runtime-old", @@ -731,7 +788,107 @@ describe("subagent registry steer restarts", () => { expect(announce.childRunId).toBe("run-failed-restart"); }); - it("marks killed runs terminated and inactive", async () => { + it("restores announce when abandoned steer task finalization fails", async () => { + registerRun({ + runId: "run-failed-task-finalize", + childSessionKey: "agent:main:subagent:failed-task-finalize", + task: "recover despite task runtime failure", + }); + expect(mod.markSubagentRunForSteerRestart("run-failed-task-finalize")).toBe(true); + emitLifecycleEnd("run-failed-task-finalize"); + await flushAnnounce(); + expect(announceSpy).not.toHaveBeenCalled(); + + const runtime = getDetachedTaskLifecycleRuntime(); + setDetachedTaskLifecycleRuntime({ + ...runtime, + finalizeTaskRunByRunId: () => { + throw new Error("task store unavailable"); + }, + }); + try { + expect(mod.clearSubagentRunSteerRestart("run-failed-task-finalize")).toBe(true); + } finally { + setDetachedTaskLifecycleRuntime(runtime); + } + + await flushAnnounce(); + expect(announceSpy).toHaveBeenCalledTimes(1); + expect(listMainRuns()[0]?.suppressAnnounceReason).toBeUndefined(); + }); + + it("terminalizes the shared task when an interrupted steer restart is abandoned", async () => { + registerRun({ + runId: "run-abandoned-restart", + childSessionKey: "agent:main:subagent:abandoned-restart", + task: "restart me or settle me", + }); + expect(mod.markSubagentRunForSteerRestart("run-abandoned-restart")).toBe(true); + + emitLifecycleEnd("run-abandoned-restart", { + endedAt: Date.now(), + aborted: true, + stopReason: "aborted", + }); + await waitForRegistrySideEffect(() => { + expect(listMainRuns()[0]).toMatchObject({ + endedReason: "subagent-killed", + suppressAnnounceReason: "steer-restart", + }); + }); + expect(findTaskByRunIdForStatus("run-abandoned-restart")).toMatchObject({ + status: "running", + }); + + expect(mod.clearSubagentRunSteerRestart("run-abandoned-restart")).toBe(true); + expect(findTaskByRunIdForStatus("run-abandoned-restart")).toMatchObject({ + status: "cancelled", + error: "Subagent restart failed after the prior run was interrupted.", + deliveryStatus: "not_applicable", + }); + }); + + it("terminalizes a deadline-normalized timeout when its steer restart is abandoned", async () => { + const endedAt = Date.now(); + const startedAt = endedAt - 2_000; + mod.registerSubagentRun({ + runId: "run-abandoned-timeout-restart", + childSessionKey: "agent:main:subagent:abandoned-timeout-restart", + requesterSessionKey: MAIN_REQUESTER_SESSION_KEY, + requesterDisplayKey: MAIN_REQUESTER_DISPLAY_KEY, + task: "restart before the deadline or time out", + cleanup: "keep", + runTimeoutSeconds: 1, + }); + expect(mod.markSubagentRunForSteerRestart("run-abandoned-timeout-restart")).toBe(true); + + emitLifecycleEnd("run-abandoned-timeout-restart", { + startedAt, + endedAt, + aborted: true, + stopReason: "aborted", + }); + await waitForRegistrySideEffect(() => { + expect(listMainRuns()[0]).toMatchObject({ + endedAt: startedAt + 1_000, + endedReason: "subagent-complete", + outcome: { status: "timeout" }, + suppressAnnounceReason: "steer-restart", + }); + }); + await flushAnnounce(); + expect(findTaskByRunIdForStatus("run-abandoned-timeout-restart")).toMatchObject({ + status: "running", + }); + + expect(mod.clearSubagentRunSteerRestart("run-abandoned-timeout-restart")).toBe(true); + expect(findTaskByRunIdForStatus("run-abandoned-timeout-restart")).toMatchObject({ + status: "timed_out", + deliveryStatus: "not_applicable", + }); + }); + + it("marks killed runs terminated and inactive while reconciliation is pending", async () => { const childSessionKey = "agent:main:subagent:killed"; registerRun({ @@ -739,6 +896,15 @@ describe("subagent registry steer restarts", () => { childSessionKey, task: "kill me", }); + const activeRun = listMainRuns()[0]; + if (!activeRun) { + throw new Error("expected active run"); + } + activeRun.execution = { + ...activeRun.execution, + status: activeRun.execution?.status ?? "running", + transcriptFile: "/tmp/recovered-subagent.jsonl", + }; expect(mod.isSubagentSessionRunActive(childSessionKey)).toBe(true); const updated = mod.markSubagentRunTerminated({ @@ -759,19 +925,8 @@ describe("subagent registry steer restarts", () => { expect(run?.cleanupHandled).toBe(true); expect(typeof run?.cleanupCompletedAt).toBe("number"); await flushAnnounce(); - const hookCall = requireSubagentEndedHookCall("run-killed"); - expect(hookCall.event.targetSessionKey).toBe(childSessionKey); - expect(hookCall.event.targetKind).toBe("subagent"); - expect(hookCall.event.reason).toBe("subagent-killed"); - expect(hookCall.event.sendFarewell).toBe(true); - expect(hookCall.event.accountId).toBeUndefined(); - expect(hookCall.event.runId).toBe("run-killed"); - expect(typeof hookCall.event.endedAt).toBe("number"); - expect(hookCall.event.outcome).toBe("killed"); - expect(hookCall.event.error).toBe("manual kill"); - expect(hookCall.ctx.runId).toBe("run-killed"); - expect(hookCall.ctx.childSessionKey).toBe(childSessionKey); - expect(hookCall.ctx.requesterSessionKey).toBe(MAIN_REQUESTER_SESSION_KEY); + expect(runSubagentEndedHookMock).not.toHaveBeenCalled(); + expect(removeInternalSessionEffectsTranscriptMock).not.toHaveBeenCalled(); }); it("treats a child session as inactive when only a stale older row is still unended", () => { @@ -832,7 +987,10 @@ describe("subagent registry steer restarts", () => { expect(run?.suppressAnnounceReason).toBeUndefined(); expect(run?.cleanupHandled).toBe(true); expect(typeof run?.cleanupCompletedAt).toBe("number"); - expect(runSubagentEndedHookMock).toHaveBeenCalledTimes(1); + const hookCall = requireSubagentEndedHookCall("run-kill-race"); + expect(hookCall.event.reason).toBe("subagent-complete"); + expect(hookCall.event.outcome).toBe("ok"); + expect(hookCall.event.error).toBeUndefined(); }); it("retries deferred parent cleanup after a descendant announces", async () => { diff --git a/src/agents/subagent-registry.test.ts b/src/agents/subagent-registry.test.ts index a8efe947937..79847214ddf 100644 --- a/src/agents/subagent-registry.test.ts +++ b/src/agents/subagent-registry.test.ts @@ -10,6 +10,24 @@ import type { SessionEntryPatchContext, SessionEntryPatchOptions, } from "../config/sessions/session-accessor.js"; +import type { AgentEventPayload } from "../infra/agent-events.js"; +import { SUBAGENT_KILL_TASK_ERROR } from "../tasks/detached-task-runtime-contract.js"; +import { + createRunningTaskRun, + findDetachedTaskRun, + finalizeTaskRunByRunId, + getDetachedTaskLifecycleRuntime, + resetDetachedTaskLifecycleRuntimeForTests, + setDetachedTaskLifecycleRuntime, +} from "../tasks/detached-task-runtime.js"; +import { resetTaskFlowRegistryForTests } from "../tasks/task-flow-registry.js"; +import { resetTaskRegistryForTests } from "../tasks/task-registry.js"; +import { findTaskByRunIdForStatus } from "../tasks/task-status-access.js"; +import { + SUBAGENT_ENDED_REASON_COMPLETE, + SUBAGENT_ENDED_REASON_ERROR, + SUBAGENT_ENDED_REASON_KILLED, +} from "./subagent-lifecycle-events.js"; const noop = () => {}; const waitForFast = (callback: () => T | Promise) => @@ -79,9 +97,9 @@ async function expectPathMissing(targetPath: string): Promise { } const mocks = vi.hoisted(() => ({ - callGateway: vi.fn(), - onAgentEvent: vi.fn(() => noop), - getAgentRunContext: vi.fn(() => undefined), + callGateway: vi.fn<(request: { method?: string }) => Promise>>(), + onAgentEvent: vi.fn<(_handler: (event: AgentEventPayload) => void) => typeof noop>(() => noop), + getAgentRunContext: vi.fn<(_runId: string) => unknown>(() => undefined), getRuntimeConfig: vi.fn(() => ({ agents: { defaults: { subagents: { archiveAfterMinutes: 0 } } }, session: { mainKey: "main", scope: "per-sender" as const }, @@ -148,8 +166,11 @@ const mocks = vi.hoisted(() => ({ ensureRuntimePluginsLoaded: vi.fn(), ensureContextEnginesInitialized: vi.fn(), resolveContextEngine: vi.fn(), - onSubagentEnded: vi.fn(async () => {}), + onSubagentEnded: vi.fn< + (params: { childSessionKey?: string }, context?: unknown) => Promise + >(async () => {}), runSubagentEnded: vi.fn(async () => {}), + removeInternalSessionEffectsTranscript: vi.fn(async () => {}), resolveAgentTimeoutMs: vi.fn(() => 1_000), scheduleOrphanRecovery: vi.fn(), })); @@ -222,6 +243,10 @@ vi.mock("./subagent-orphan-recovery.js", () => ({ scheduleOrphanRecovery: mocks.scheduleOrphanRecovery, })); +vi.mock("./internal-session-effects.js", () => ({ + removeInternalSessionEffectsTranscript: mocks.removeInternalSessionEffectsTranscript, +})); + describe("subagent registry seam flow", () => { let mod: typeof import("./subagent-registry.js"); @@ -230,6 +255,7 @@ describe("subagent registry seam flow", () => { }); beforeEach(() => { + resetDetachedTaskLifecycleRuntimeForTests(); vi.clearAllMocks(); vi.useFakeTimers(); vi.setSystemTime(new Date("2026-03-24T12:00:00Z")); @@ -268,7 +294,7 @@ describe("subagent registry seam flow", () => { return {}; }); mod.testing.setDepsForTest({ - callGateway: mocks.callGateway, + callGateway: mocks.callGateway as typeof import("../gateway/call.js").callGateway, captureSubagentCompletionReply: mocks.captureSubagentCompletionReply, cleanupBrowserSessionsForLifecycleEnd: mocks.cleanupBrowserSessionsForLifecycleEnd, onAgentEvent: mocks.onAgentEvent, @@ -285,6 +311,7 @@ describe("subagent registry seam flow", () => { }); afterEach(() => { + resetDetachedTaskLifecycleRuntimeForTests(); mod.testing.setDepsForTest(); mod.resetSubagentRegistryForTests({ persist: false }); vi.useRealTimers(); @@ -328,9 +355,25 @@ describe("subagent registry seam flow", () => { delivery: { status: "delivered", announcedAt: now - 2, deliveredAt: now - 2 }, cleanupCompletedAt: now - 1, }); + mod.addSubagentRunForTests({ + runId: "run-killed-reconciling", + childSessionKey: "agent:main:subagent:killed-reconciling", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "reconcile killed task", + cleanup: "delete", + expectsCompletionMessage: false, + createdAt: now - 6, + endedAt: now - 5, + endedReason: "subagent-killed", + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: now - 5 }, + cleanupCompletedAt: now - 4, + }); expect(mod.listSessionMaintenanceProtectedSubagentSessionKeys().toSorted()).toEqual([ "agent:main:subagent:active", + "agent:main:subagent:killed-reconciling", "agent:main:subagent:pending", ]); }); @@ -2052,6 +2095,112 @@ describe("subagent registry seam flow", () => { expect(replacement?.endedAt).toBeUndefined(); }); + it("ignores a late yield lifecycle event after the paused run is killed", async () => { + mocks.callGateway.mockImplementation(async (request: { method?: string }) => + request.method === "agent.wait" ? { status: "pending" } : {}, + ); + const runId = "run-yield-killed-before-late-event"; + const childSessionKey = "agent:main:subagent:yield-killed-before-late-event"; + mod.registerSubagentRun({ + runId, + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "stop while paused", + cleanup: "keep", + }); + const lastOnAgentEventCall = mocks.onAgentEvent.mock.calls.at(-1) as unknown as + | [(evt: { runId: string; stream: string; data: Record }) => void] + | undefined; + const lifecycleHandler = lastOnAgentEventCall?.[0]; + expect(lifecycleHandler).toBeTypeOf("function"); + + lifecycleHandler?.({ + runId, + stream: "lifecycle", + data: { phase: "end", startedAt: 111, endedAt: 222, yielded: true }, + }); + expect(mod.markSubagentRunTerminated({ runId, childSessionKey, reason: "killed" })).toBe(1); + const killed = mod + .listSubagentRunsForRequester("agent:main:main") + .find((run) => run.runId === runId); + expect(killed).toMatchObject({ + endedAt: 222, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + cleanupHandled: true, + suppressAnnounceReason: "killed", + }); + expect(killed?.pauseReason).toBeUndefined(); + const killedCleanupAt = killed?.cleanupCompletedAt; + + lifecycleHandler?.({ + runId, + stream: "lifecycle", + data: { phase: "end", startedAt: 111, endedAt: 333, yielded: true }, + }); + + const afterLateYield = mod + .listSubagentRunsForRequester("agent:main:main") + .find((run) => run.runId === runId); + expect(afterLateYield).toMatchObject({ + endedAt: 222, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + cleanupHandled: true, + cleanupCompletedAt: killedCleanupAt, + suppressAnnounceReason: "killed", + }); + expect(afterLateYield?.pauseReason).toBeUndefined(); + }); + + it("accepts an authoritative late yield after non-kill cleanup started", async () => { + mocks.callGateway.mockImplementation(async (request: { method?: string }) => + request.method === "agent.wait" ? { status: "pending" } : {}, + ); + const runId = "run-yield-after-success-cleanup"; + mod.registerSubagentRun({ + runId, + childSessionKey: "agent:main:subagent:yield-after-success-cleanup", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "pause after terminal projection", + cleanup: "keep", + }); + const lifecycleHandler = ( + mocks.onAgentEvent.mock.calls.at(-1) as unknown as + | [(evt: { runId: string; stream: string; data: Record }) => void] + | undefined + )?.[0]; + const run = mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === runId); + expect(lifecycleHandler).toBeTypeOf("function"); + expect(run).toBeDefined(); + Object.assign(run!, { + endedAt: 222, + endedReason: SUBAGENT_ENDED_REASON_COMPLETE, + outcome: { status: "ok" as const }, + cleanupHandled: true, + cleanupCompletedAt: 223, + delivery: { status: "delivered" as const, deliveredAt: 223 }, + }); + + lifecycleHandler?.({ + runId, + stream: "lifecycle", + data: { phase: "end", startedAt: 111, endedAt: 333, yielded: true }, + }); + + expect(run).toMatchObject({ + endedAt: 333, + pauseReason: "sessions_yield", + cleanupHandled: false, + delivery: { status: "pending" }, + }); + expect(run?.endedReason).toBeUndefined(); + expect(run?.outcome).toBeUndefined(); + expect(run?.cleanupCompletedAt).toBeUndefined(); + }); + it("keeps yield terminals paused when the lifecycle event also signals abort (#92448)", async () => { // sessions_yield ends the turn by aborting the run signal, so a depth-1 // subagent's yield terminal can arrive carrying yielded plus aborted (or @@ -2169,6 +2318,46 @@ describe("subagent registry seam flow", () => { expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled(); }); + it("cancels a pending timeout grace timer when the run is explicitly killed", async () => { + mocks.callGateway.mockImplementation(async (request: { method?: string }) => + request.method === "agent.wait" ? { status: "pending" } : {}, + ); + const runId = "run-killed-after-pending-timeout"; + mod.registerSubagentRun({ + runId, + childSessionKey: "agent:main:subagent:killed-after-pending-timeout", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "stop during timeout grace", + cleanup: "keep", + }); + + const lifecycleHandler = ( + mocks.onAgentEvent.mock.calls.at(-1) as unknown as + | [(evt: { runId: string; stream: string; data: Record }) => void] + | undefined + )?.[0]; + expect(lifecycleHandler).toBeTypeOf("function"); + + lifecycleHandler?.({ + runId, + stream: "lifecycle", + data: { phase: "end", startedAt: 111, endedAt: 222, aborted: true }, + }); + expect(mod.markSubagentRunTerminated({ runId, reason: "manual kill" })).toBe(1); + + await vi.advanceTimersByTimeAsync(60_000); + const run = mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === runId); + expect(run).toMatchObject({ + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "manual kill" }, + }); + expect(run?.outcome?.status).not.toBe("timeout"); + expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled(); + }); + it("cancels a pending grace timer when agent.wait observes the yield after an aborted terminal (#92448)", async () => { let resolveWait: (value: { status: "ok"; @@ -2334,7 +2523,7 @@ describe("subagent registry seam flow", () => { expect(run?.outcome?.status).toBe("timeout"); }); - it("announces aborted agent.wait snapshots as killed subagent failures", async () => { + it("publishes aborted agent.wait snapshots only after killed reconciliation", async () => { mocks.callGateway.mockImplementation(async (request: { method?: string }) => { if (request.method === "agent.wait") { return { @@ -2358,8 +2547,16 @@ describe("subagent registry seam flow", () => { }); await waitForFast(() => { - expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledTimes(1); + const run = mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === "run-aborted-wait"); + expect(run?.endedReason).toBe("subagent-killed"); + expect(run?.suppressAnnounceReason).toBe("killed"); }); + expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled(); + + await mod.testing.sweepOnceForTests(); + await waitForFast(() => expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledTimes(1)); const announceParams = expectRecordFields( getMockCallArg(mocks.runSubagentAnnounceFlow, 0, 0, "aborted wait announce"), { childRunId: "run-aborted-wait" }, @@ -2377,11 +2574,13 @@ describe("subagent registry seam flow", () => { "aborted wait announce outcome", ); - const run = mod - .listSubagentRunsForRequester("agent:main:main") - .find((entry) => entry.runId === "run-aborted-wait"); - expect(run?.endedReason).toBe("subagent-killed"); - expect(run?.outcome?.status).toBe("error"); + await waitForFast(() => { + expect( + mod + .listSubagentRunsForRequester("agent:main:main") + .some((entry) => entry.runId === "run-aborted-wait"), + ).toBe(false); + }); }); it("reconciles stale active runs from persisted terminal session state during sweep", async () => { @@ -2448,7 +2647,1232 @@ describe("subagent registry seam flow", () => { }, "stale terminal run outcome", ); - expect(run?.cleanupCompletedAt).toBeTypeOf("number"); + await waitForFast(() => expect(run?.cleanupCompletedAt).toBeTypeOf("number")); + }); + + it("reconciles persisted completion before expiring a provisional kill", async () => { + const startedAt = Date.parse("2026-03-24T11:50:00Z"); + const killedAt = Date.parse("2026-03-24T11:55:00Z"); + const endedAt = Date.parse("2026-03-24T11:56:00Z"); + mocks.loadSessionStore.mockReturnValue({ + "agent:main:subagent:child": { + sessionId: "sess-child", + updatedAt: endedAt, + status: "done", + startedAt, + endedAt, + }, + }); + mod.addSubagentRunForTests({ + runId: "run-killed-with-persisted-completion", + childSessionKey: "agent:main:subagent:child", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "recover persisted completion", + cleanup: "keep", + createdAt: startedAt, + startedAt, + endedAt: killedAt, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt }, + cleanupHandled: true, + cleanupCompletedAt: killedAt, + }); + + await mod.testing.sweepOnceForTests(); + + await waitForFast(() => { + const run = mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === "run-killed-with-persisted-completion"); + expect(run?.endedReason).toBe(SUBAGENT_ENDED_REASON_COMPLETE); + expect(run?.outcome).toMatchObject({ status: "ok", startedAt, endedAt }); + expect(run?.archiveAtMs).toBeUndefined(); + }); + }); + + it.each([ + { + name: "repairs a provisional registry kill from a task-first completion", + taskStatus: "succeeded" as const, + expectedReason: SUBAGENT_ENDED_REASON_COMPLETE, + expectedOutcome: { status: "ok" }, + }, + { + name: "preserves an error reason when replaying a task-first failure", + taskStatus: "failed" as const, + taskError: "provider failed", + expectedReason: SUBAGENT_ENDED_REASON_ERROR, + expectedOutcome: { status: "error", error: "provider failed" }, + }, + ])("$name", async ({ taskStatus, taskError, expectedReason, expectedOutcome }) => { + resetTaskRegistryForTests({ persist: false }); + resetTaskFlowRegistryForTests({ persist: false }); + try { + const startedAt = Date.parse("2026-03-24T11:50:00Z"); + const killedAt = Date.parse("2026-03-24T11:55:00Z"); + const completedAt = Date.parse("2026-03-24T11:56:00Z"); + const runId = `run-task-first-${taskStatus}`; + const childSessionKey = `agent:main:subagent:task-first-${taskStatus}`; + expect( + createRunningTaskRun({ + runtime: "subagent", + sourceId: runId, + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey, + runId, + task: "repair task-first completion", + deliveryStatus: "pending", + startedAt, + lastEventAt: startedAt, + }), + ).not.toBeNull(); + expect( + finalizeTaskRunByRunId({ + runId, + runtime: "subagent", + sessionKey: childSessionKey, + status: taskStatus, + endedAt: completedAt, + lastEventAt: completedAt, + error: taskError, + progressSummary: "durable final result", + }), + ).toHaveLength(1); + mocks.loadSessionStore.mockReturnValue({}); + mod.addSubagentRunForTests({ + runId, + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "repair task-first completion", + cleanup: "keep", + createdAt: startedAt, + startedAt, + endedAt: killedAt, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt }, + cleanupHandled: true, + cleanupCompletedAt: killedAt, + }); + + await mod.testing.sweepOnceForTests(); + + await waitForFast(() => { + const run = mod + .listSubagentRunsForRequester("agent:main:main") + .find((candidate) => candidate.runId === runId); + expect(run).toMatchObject({ + endedAt: completedAt, + endedReason: expectedReason, + outcome: { ...expectedOutcome, startedAt, endedAt: completedAt }, + completion: { resultText: "durable final result", capturedAt: completedAt }, + }); + expect(run?.killReconciliation).toBeUndefined(); + }); + } finally { + resetTaskRegistryForTests({ persist: false }); + resetTaskFlowRegistryForTests({ persist: false }); + } + }); + + it("replays task-first completion against the current steer generation deadline", async () => { + resetTaskRegistryForTests({ persist: false }); + resetTaskFlowRegistryForTests({ persist: false }); + try { + const originalStartedAt = 1_000; + const replacementStartedAt = 100_000; + const killedAt = 104_000; + const completedAt = 105_000; + const taskRunId = "run-task-first-original"; + const runId = "run-task-first-replacement"; + const childSessionKey = "agent:main:subagent:task-first-steer"; + expect( + createRunningTaskRun({ + runtime: "subagent", + sourceId: taskRunId, + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey, + runId: taskRunId, + task: "finish after steer", + deliveryStatus: "pending", + startedAt: originalStartedAt, + lastEventAt: originalStartedAt, + }), + ).not.toBeNull(); + expect( + finalizeTaskRunByRunId({ + runId: taskRunId, + runtime: "subagent", + sessionKey: childSessionKey, + status: "succeeded", + endedAt: completedAt, + lastEventAt: completedAt, + progressSummary: "replacement completed", + }), + ).toHaveLength(1); + mocks.loadSessionStore.mockReturnValue({}); + mod.addSubagentRunForTests({ + runId, + taskRunId, + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "finish after steer", + cleanup: "keep", + generation: 2, + createdAt: replacementStartedAt, + startedAt: replacementStartedAt, + sessionStartedAt: originalStartedAt, + runTimeoutSeconds: 10, + endedAt: killedAt, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt }, + cleanupHandled: true, + cleanupCompletedAt: killedAt, + }); + + await mod.testing.sweepOnceForTests(); + + await waitForFast(() => { + const run = mod + .listSubagentRunsForRequester("agent:main:main") + .find((candidate) => candidate.runId === runId); + expect(run).toMatchObject({ + endedAt: completedAt, + endedReason: SUBAGENT_ENDED_REASON_COMPLETE, + outcome: { status: "ok", startedAt: replacementStartedAt, endedAt: completedAt }, + }); + }); + } finally { + resetTaskRegistryForTests({ persist: false }); + resetTaskFlowRegistryForTests({ persist: false }); + } + }); + + it("retains persisted completion evidence when task projection fails", async () => { + resetTaskRegistryForTests({ persist: false }); + resetTaskFlowRegistryForTests({ persist: false }); + try { + const startedAt = Date.parse("2026-03-24T11:50:00Z"); + const completedAt = Date.parse("2026-03-24T11:54:00Z"); + const killedAt = Date.parse("2026-03-24T11:55:00Z"); + const runId = "run-killed-projection-retry"; + const childSessionKey = "agent:main:subagent:projection-retry"; + const task = createRunningTaskRun({ + runtime: "subagent", + sourceId: runId, + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey, + runId, + task: "retry completion projection", + deliveryStatus: "pending", + startedAt, + lastEventAt: startedAt, + }); + expect(task).not.toBeNull(); + finalizeTaskRunByRunId({ + runId, + runtime: "subagent", + sessionKey: childSessionKey, + status: "cancelled", + endedAt: killedAt, + lastEventAt: killedAt, + error: "Cancelled by operator.", + }); + const runtime = getDetachedTaskLifecycleRuntime(); + setDetachedTaskLifecycleRuntime({ + ...runtime, + completeTaskRunByRunId: vi.fn(() => { + throw new Error("task projection unavailable"); + }), + }); + mocks.loadSessionStore.mockReturnValue({ + [childSessionKey]: { + sessionId: "sess-projection-retry", + updatedAt: completedAt, + status: "done", + startedAt, + endedAt: completedAt, + }, + }); + mod.addSubagentRunForTests({ + runId, + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "retry completion projection", + cleanup: "keep", + createdAt: startedAt, + startedAt, + endedAt: killedAt, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt }, + cleanupHandled: true, + cleanupCompletedAt: killedAt, + }); + + await mod.testing.sweepOnceForTests(); + + const run = mod + .listSubagentRunsForRequester("agent:main:main") + .find((candidate) => candidate.runId === runId); + expect(run).toMatchObject({ + endedAt: killedAt, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + killReconciliation: { killedAt }, + }); + expect(findTaskByRunIdForStatus(runId)).toMatchObject({ + status: "cancelled", + endedAt: killedAt, + }); + } finally { + resetDetachedTaskLifecycleRuntimeForTests(); + resetTaskRegistryForTests({ persist: false }); + resetTaskFlowRegistryForTests({ persist: false }); + } + }); + + it("lets an opaque runtime recover persisted completion before expiring a kill", async () => { + const startedAt = Date.parse("2026-03-24T11:50:00Z"); + const killedAt = Date.parse("2026-03-24T11:55:00Z"); + const endedAt = Date.parse("2026-03-24T11:56:00Z"); + const completeTaskRunByRunId = vi.fn(() => [{}] as never); + const opaqueRuntime = { + ...getDetachedTaskLifecycleRuntime(), + completeTaskRunByRunId, + }; + delete opaqueRuntime.findTaskRun; + setDetachedTaskLifecycleRuntime(opaqueRuntime); + mocks.loadSessionStore.mockReturnValue({ + "agent:main:subagent:opaque-child": { + sessionId: "sess-opaque-child", + updatedAt: endedAt, + status: "done", + startedAt, + endedAt, + }, + }); + mod.addSubagentRunForTests({ + runId: "run-opaque-killed-with-persisted-completion", + childSessionKey: "agent:main:subagent:opaque-child", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "recover opaque persisted completion", + cleanup: "keep", + createdAt: startedAt, + startedAt, + endedAt: killedAt, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt }, + cleanupHandled: true, + cleanupCompletedAt: killedAt, + }); + + await mod.testing.sweepOnceForTests(); + + await waitForFast(() => { + const run = mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === "run-opaque-killed-with-persisted-completion"); + expect(run).toMatchObject({ + endedAt, + endedReason: SUBAGENT_ENDED_REASON_COMPLETE, + outcome: { status: "ok", startedAt, endedAt }, + }); + expect(completeTaskRunByRunId).toHaveBeenCalledTimes(1); + }); + }); + + it("retires an opaque tombstone when persisted completion cannot update its task", async () => { + const startedAt = Date.parse("2026-03-24T11:50:00Z"); + const killedAt = Date.parse("2026-03-24T11:55:00Z"); + const completedAt = Date.parse("2026-03-24T11:56:00Z"); + const childSessionKey = "agent:main:subagent:opaque-finalizer-miss"; + const completeTaskRunByRunId = vi.fn(() => [] as never); + const finalizeTaskRunSpy = vi.fn(() => [] as never); + const opaqueRuntime = { + ...getDetachedTaskLifecycleRuntime(), + completeTaskRunByRunId, + finalizeTaskRunByRunId: finalizeTaskRunSpy, + }; + delete opaqueRuntime.findTaskRun; + setDetachedTaskLifecycleRuntime(opaqueRuntime); + mocks.loadSessionStore.mockReturnValue({ + [childSessionKey]: { + sessionId: "sess-opaque-finalizer-miss", + updatedAt: completedAt, + status: "done", + startedAt, + endedAt: completedAt, + }, + }); + mod.addSubagentRunForTests({ + runId: "run-opaque-finalizer-miss", + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "bound opaque finalizer failure", + cleanup: "keep", + expectsCompletionMessage: false, + createdAt: startedAt, + startedAt, + endedAt: killedAt, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt }, + cleanupHandled: true, + cleanupCompletedAt: killedAt, + }); + + await mod.testing.sweepOnceForTests(); + + const run = mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === "run-opaque-finalizer-miss"); + expect(run?.killReconciliation).toBeUndefined(); + expect(completeTaskRunByRunId).toHaveBeenCalled(); + expect(finalizeTaskRunSpy).toHaveBeenCalled(); + }); + + it("retires stable operator cancellation despite a late persisted completion", async () => { + resetTaskRegistryForTests({ persist: false }); + resetTaskFlowRegistryForTests({ persist: false }); + try { + const now = Date.parse("2026-03-24T12:00:00Z"); + const startedAt = now - 10_000; + const killedAt = now - 1_000; + const completedAt = now; + const runId = "run-killed-stable-cancellation"; + const childSessionKey = "agent:main:subagent:stable-cancellation"; + mocks.loadSessionStore.mockReturnValue({ + [childSessionKey]: { + sessionId: "sess-stable-cancellation", + updatedAt: completedAt, + status: "done", + startedAt, + endedAt: completedAt, + }, + }); + expect( + createRunningTaskRun({ + runtime: "subagent", + sourceId: runId, + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey, + runId, + task: "preserve operator cancellation", + deliveryStatus: "pending", + startedAt, + lastEventAt: startedAt, + }), + ).not.toBeNull(); + expect( + finalizeTaskRunByRunId({ + runId, + runtime: "subagent", + sessionKey: childSessionKey, + status: "cancelled", + endedAt: killedAt, + lastEventAt: killedAt, + error: "Cancelled by operator.", + }), + ).toHaveLength(1); + mod.addSubagentRunForTests({ + runId, + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "preserve operator cancellation", + cleanup: "delete", + expectsCompletionMessage: true, + createdAt: startedAt, + startedAt, + endedAt: killedAt, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt }, + cleanupHandled: true, + cleanupCompletedAt: killedAt, + archiveAtMs: Date.now(), + }); + + expect(killedAt + 5 * 60_000).toBeGreaterThan(Date.now()); + vi.setSystemTime(killedAt + 5 * 60_000); + + await mod.testing.sweepOnceForTests(); + + await waitForFast(() => { + expect( + mod + .listSubagentRunsForRequester("agent:main:main") + .some((entry) => entry.runId === runId), + ).toBe(false); + expect(mocks.callGateway).toHaveBeenCalledWith({ + method: "sessions.delete", + params: { + key: childSessionKey, + deleteTranscript: true, + emitLifecycleHooks: false, + }, + timeoutMs: 10_000, + }); + }); + expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled(); + } finally { + resetTaskRegistryForTests({ persist: false }); + resetTaskFlowRegistryForTests({ persist: false }); + } + }); + + it("restores an explicit timeout that predates stable operator cancellation", async () => { + resetTaskRegistryForTests({ persist: false }); + resetTaskFlowRegistryForTests({ persist: false }); + try { + const now = Date.parse("2026-03-24T12:00:00Z"); + const startedAt = now - 10_000; + const timeoutAt = now - 2_000; + const killedAt = now - 1_000; + const completedAt = now; + const runId = "run-completed-before-stable-cancellation"; + const childSessionKey = "agent:main:subagent:completed-before-cancellation"; + mocks.loadSessionStore.mockReturnValue({ + [childSessionKey]: { + sessionId: "sess-completed-before-cancellation", + updatedAt: completedAt, + status: "killed", + startedAt, + endedAt: completedAt, + }, + }); + createRunningTaskRun({ + runtime: "subagent", + sourceId: runId, + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey, + runId, + task: "preserve earlier completion", + deliveryStatus: "pending", + startedAt, + lastEventAt: startedAt, + }); + finalizeTaskRunByRunId({ + runId, + runtime: "subagent", + sessionKey: childSessionKey, + status: "cancelled", + endedAt: killedAt, + lastEventAt: killedAt, + error: "Cancelled by operator.", + suppressDelivery: true, + }); + mod.addSubagentRunForTests({ + runId, + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "preserve earlier completion", + cleanup: "keep", + expectsCompletionMessage: false, + createdAt: startedAt, + startedAt, + runTimeoutSeconds: 8, + endedAt: killedAt, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt }, + cleanupHandled: true, + cleanupCompletedAt: killedAt, + }); + + vi.setSystemTime(killedAt + 5 * 60_000); + await mod.testing.sweepOnceForTests(); + + await waitForFast(() => { + const run = mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === runId); + expect(run).toMatchObject({ + endedAt: timeoutAt, + endedReason: SUBAGENT_ENDED_REASON_COMPLETE, + outcome: { status: "timeout", startedAt, endedAt: timeoutAt }, + }); + const task = findTaskByRunIdForStatus(runId); + expect(task).toMatchObject({ + status: "timed_out", + endedAt: timeoutAt, + }); + expect(task?.error).toBeUndefined(); + }); + } finally { + resetTaskRegistryForTests({ persist: false }); + resetTaskFlowRegistryForTests({ persist: false }); + } + }); + + it("suppresses registry delivery when cancellation becomes durable during capture", async () => { + resetTaskRegistryForTests({ persist: false }); + resetTaskFlowRegistryForTests({ persist: false }); + try { + const now = Date.now(); + const killedAt = now - 5 * 60_000; + const startedAt = killedAt - 10_000; + const completedAt = killedAt + 1_000; + const runId = "run-cancelled-during-sweep-capture"; + const childSessionKey = "agent:main:subagent:cancelled-during-sweep-capture"; + mocks.loadSessionStore.mockReturnValue({ + [childSessionKey]: { + sessionId: "sess-cancelled-during-sweep-capture", + updatedAt: completedAt, + status: "done", + startedAt, + endedAt: completedAt, + }, + }); + vi.setSystemTime(startedAt); + createRunningTaskRun({ + runtime: "subagent", + sourceId: runId, + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey, + runId, + task: "cancel during result capture", + deliveryStatus: "pending", + startedAt, + lastEventAt: startedAt, + }); + vi.setSystemTime(now); + finalizeTaskRunByRunId({ + runId, + runtime: "subagent", + sessionKey: childSessionKey, + status: "cancelled", + endedAt: killedAt, + lastEventAt: killedAt, + error: SUBAGENT_KILL_TASK_ERROR, + }); + let finishCapture: ((value: string) => void) | undefined; + mocks.captureSubagentCompletionReply.mockImplementationOnce( + async () => + await new Promise((resolve) => { + finishCapture = resolve; + }), + ); + mod.addSubagentRunForTests({ + runId, + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "cancel during result capture", + cleanup: "keep", + expectsCompletionMessage: true, + createdAt: startedAt, + startedAt, + endedAt: killedAt, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt }, + cleanupHandled: true, + cleanupCompletedAt: killedAt, + }); + + const sweep = mod.testing.sweepOnceForTests(); + await vi.waitFor(() => expect(mocks.captureSubagentCompletionReply).toHaveBeenCalled()); + finalizeTaskRunByRunId({ + runId, + runtime: "subagent", + sessionKey: childSessionKey, + status: "cancelled", + endedAt: killedAt, + lastEventAt: Date.now(), + error: "Cancelled by operator.", + }); + expect(findTaskByRunIdForStatus(runId)).toMatchObject({ + status: "cancelled", + error: "Cancelled by operator.", + childSessionKey, + createdAt: startedAt, + }); + expect( + findDetachedTaskRun({ + runId, + runtime: "subagent", + sessionKey: childSessionKey, + createdAtOrAfter: startedAt, + }), + ).toMatchObject({ + lookup: "available", + task: { status: "cancelled", error: "Cancelled by operator." }, + }); + finishCapture?.("late provider result"); + await sweep; + await Promise.resolve(); + const remainingRun = mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === runId); + expect(remainingRun).toBeUndefined(); + + expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled(); + } finally { + resetTaskRegistryForTests({ persist: false }); + resetTaskFlowRegistryForTests({ persist: false }); + } + }); + + it("uses the kill time when reconciling a yielded run", async () => { + resetTaskRegistryForTests({ persist: false }); + resetTaskFlowRegistryForTests({ persist: false }); + try { + const startedAt = Date.parse("2026-03-24T11:50:00Z"); + const yieldedAt = Date.parse("2026-03-24T11:59:00Z"); + const completedAt = Date.parse("2026-03-24T11:59:30Z"); + const killedAt = Date.parse("2026-03-24T12:00:00Z"); + const runId = "run-yielded-before-kill"; + const childSessionKey = "agent:main:subagent:yielded-before-kill"; + mocks.loadSessionStore.mockReturnValue({ + [childSessionKey]: { + sessionId: "sess-yielded-before-kill", + updatedAt: completedAt, + status: "done", + startedAt, + endedAt: completedAt, + }, + }); + createRunningTaskRun({ + runtime: "subagent", + sourceId: runId, + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey, + runId, + task: "complete between yield and kill", + deliveryStatus: "pending", + startedAt, + lastEventAt: startedAt, + }); + mod.addSubagentRunForTests({ + runId, + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "complete between yield and kill", + cleanup: "keep", + expectsCompletionMessage: false, + createdAt: startedAt, + startedAt, + endedAt: yieldedAt, + pauseReason: "sessions_yield", + cleanupHandled: false, + }); + + vi.setSystemTime(killedAt); + expect(mod.markSubagentRunTerminated({ runId, reason: "manual kill" })).toBe(1); + expect(findTaskByRunIdForStatus(runId)).toMatchObject({ + status: "cancelled", + endedAt: killedAt, + }); + const killedRun = mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === runId); + expect(killedRun).toMatchObject({ + endedAt: yieldedAt, + cleanupCompletedAt: killedAt, + endedReason: SUBAGENT_ENDED_REASON_KILLED, + }); + + vi.setSystemTime(killedAt + 5 * 60_000); + await mod.testing.sweepOnceForTests(); + + await waitForFast(() => { + const run = mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === runId); + expect(run).toMatchObject({ + endedAt: completedAt, + endedReason: SUBAGENT_ENDED_REASON_COMPLETE, + outcome: { status: "ok", startedAt, endedAt: completedAt }, + }); + expect(findTaskByRunIdForStatus(runId)).toMatchObject({ + status: "succeeded", + endedAt: completedAt, + }); + }); + } finally { + resetTaskRegistryForTests({ persist: false }); + resetTaskFlowRegistryForTests({ persist: false }); + } + }); + + it("expires a tombstone instead of replaying persisted killed state", async () => { + const startedAt = Date.parse("2026-03-24T11:50:00Z"); + const endedAt = Date.parse("2026-03-24T11:55:00Z"); + mocks.loadSessionStore.mockReturnValue({ + "agent:main:subagent:child": { + sessionId: "sess-child", + updatedAt: endedAt, + status: "killed", + startedAt, + endedAt, + }, + }); + mod.addSubagentRunForTests({ + runId: "run-killed-with-persisted-kill", + childSessionKey: "agent:main:subagent:child", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "expire persisted kill", + cleanup: "keep", + expectsCompletionMessage: false, + createdAt: startedAt, + startedAt, + endedAt, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: endedAt }, + cleanupHandled: true, + cleanupCompletedAt: endedAt, + }); + + await mod.testing.sweepOnceForTests(); + + await waitForFast(() => { + expect( + mod + .listSubagentRunsForRequester("agent:main:main") + .some((entry) => entry.runId === "run-killed-with-persisted-kill"), + ).toBe(false); + }); + }); + + it("keeps requester stop delivery suppressed after kill reconciliation", async () => { + const killedAt = Date.now() - 5 * 60_000; + mod.addSubagentRunForTests({ + runId: "run-requester-stop-suppressed", + childSessionKey: "agent:main:subagent:requester-stop-suppressed", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "do not re-inject after stop", + cleanup: "keep", + expectsCompletionMessage: true, + createdAt: killedAt - 60_000, + endedAt: killedAt, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt, suppressTaskDelivery: true }, + cleanupHandled: true, + cleanupCompletedAt: killedAt, + }); + + await mod.testing.sweepOnceForTests(); + await waitForFast(() => { + expect( + mod + .listSubagentRunsForRequester("agent:main:main") + .some((entry) => entry.runId === "run-requester-stop-suppressed"), + ).toBe(false); + }); + + expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled(); + }); + + it("retires a superseded tombstone after its newer generation is released", async () => { + const killedAt = Date.now() - 5 * 60_000; + const childSessionKey = "agent:main:subagent:released-successor"; + mocks.callGateway.mockImplementation(async (request: { method?: string }) => + request.method === "agent.wait" ? { status: "pending" } : {}, + ); + mod.addSubagentRunForTests({ + runId: "run-released-successor-old", + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "retire only old ownership", + cleanup: "delete", + createdAt: killedAt - 60_000, + endedAt: killedAt, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt }, + cleanupHandled: true, + cleanupCompletedAt: killedAt, + }); + mod.registerSubagentRun({ + runId: "run-released-successor-new", + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "new generation", + cleanup: "keep", + }); + const oldRun = mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === "run-released-successor-old"); + expect(oldRun?.killReconciliation?.supersededAt).toBe(Date.now()); + mod.releaseSubagentRun("run-released-successor-new"); + + await mod.testing.sweepOnceForTests(); + + expect( + mod + .listSubagentRunsForRequester("agent:main:main") + .some((entry) => entry.runId === "run-released-successor-old"), + ).toBe(false); + expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled(); + expect( + mocks.callGateway.mock.calls.some( + ([request]) => (request as { method?: string } | undefined)?.method === "sessions.delete", + ), + ).toBe(false); + }); + + it("does not reconcile an old tombstone from a newer run completion", async () => { + const oldStartedAt = Date.parse("2026-03-24T11:50:00Z"); + const oldEndedAt = Date.parse("2026-03-24T11:55:00Z"); + const newStartedAt = Date.parse("2026-03-24T11:58:00Z"); + const newEndedAt = Date.parse("2026-03-24T11:59:00Z"); + mocks.loadSessionStore.mockReturnValue({ + "agent:main:subagent:reused": { + sessionId: "sess-reused", + updatedAt: newEndedAt, + status: "done", + startedAt: newStartedAt, + endedAt: newEndedAt, + }, + }); + mocks.getAgentRunContext.mockImplementation((runId: string) => + runId === "run-new-generation" ? ({} as never) : undefined, + ); + mocks.getGlobalHookRunner.mockReturnValue({ + hasHooks: (hookName: string) => hookName === "subagent_ended", + runSubagentEnded: mocks.runSubagentEnded, + } as never); + const attachmentsRootDir = await fs.mkdtemp( + path.join(os.tmpdir(), "openclaw-old-tombstone-attachments-"), + ); + const attachmentsDir = path.join(attachmentsRootDir, "child"); + await fs.mkdir(attachmentsDir, { recursive: true }); + await fs.writeFile(path.join(attachmentsDir, "artifact.txt"), "artifact"); + const oldTranscriptFile = "/tmp/internal-agent-runs/run-old-tombstone.jsonl"; + mod.addSubagentRunForTests({ + runId: "run-old-tombstone", + childSessionKey: "agent:main:subagent:reused", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "old generation", + cleanup: "delete", + createdAt: oldStartedAt, + startedAt: oldStartedAt, + sessionStartedAt: oldStartedAt, + endedAt: oldEndedAt, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: oldEndedAt }, + cleanupHandled: true, + cleanupCompletedAt: oldEndedAt, + archiveAtMs: Date.now(), + retainAttachmentsOnKeep: true, + attachmentsDir, + attachmentsRootDir, + execution: { + status: "terminal", + startedAt: oldStartedAt, + endedAt: oldEndedAt, + transcriptFile: oldTranscriptFile, + }, + }); + mod.addSubagentRunForTests({ + runId: "run-new-generation", + childSessionKey: "agent:main:subagent:reused", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "new generation", + cleanup: "keep", + createdAt: newStartedAt, + startedAt: newStartedAt, + sessionStartedAt: newStartedAt, + }); + + await mod.testing.sweepOnceForTests(); + + const runs = mod.listSubagentRunsForRequester("agent:main:main"); + expect(runs.some((entry) => entry.runId === "run-old-tombstone")).toBe(false); + const newRun = runs.find((entry) => entry.runId === "run-new-generation"); + expect(newRun).toBeDefined(); + expect(newRun?.endedAt).toBeUndefined(); + expect(newRun?.outcome).toBeUndefined(); + expect(mocks.runSubagentEnded).not.toHaveBeenCalled(); + expect( + mocks.onSubagentEnded.mock.calls.some( + ([params]) => params.childSessionKey === "agent:main:subagent:reused", + ), + ).toBe(false); + expect(mocks.removeInternalSessionEffectsTranscript).toHaveBeenCalledWith(oldTranscriptFile); + await expectPathMissing(attachmentsDir); + expect( + mocks.callGateway.mock.calls.some( + ([request]) => (request as { method?: string } | undefined)?.method === "sessions.delete", + ), + ).toBe(false); + }); + + it("checks the raw completion time before clamping an old run deadline", async () => { + resetTaskRegistryForTests({ persist: false }); + resetTaskFlowRegistryForTests({ persist: false }); + try { + const oldStartedAt = Date.parse("2026-03-24T11:50:00Z"); + const oldKilledAt = Date.parse("2026-03-24T11:55:00Z"); + const newStartedAt = Date.parse("2026-03-24T11:58:00Z"); + const newEndedAt = Date.parse("2026-03-24T11:59:00Z"); + const childSessionKey = "agent:main:subagent:reused-no-start"; + mocks.loadSessionStore.mockReturnValue({ + [childSessionKey]: { + sessionId: "sess-reused-no-start", + updatedAt: newEndedAt, + status: "done", + endedAt: newEndedAt, + }, + }); + createRunningTaskRun({ + runtime: "subagent", + sourceId: "run-old-no-start", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey, + runId: "run-old-no-start", + task: "keep old cancellation canonical", + deliveryStatus: "pending", + startedAt: oldStartedAt, + lastEventAt: oldStartedAt, + }); + finalizeTaskRunByRunId({ + runId: "run-old-no-start", + runtime: "subagent", + sessionKey: childSessionKey, + status: "cancelled", + endedAt: oldKilledAt, + lastEventAt: oldKilledAt, + error: SUBAGENT_KILL_TASK_ERROR, + }); + mod.addSubagentRunForTests({ + runId: "run-old-no-start", + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "keep old cancellation canonical", + cleanup: "keep", + createdAt: oldStartedAt, + startedAt: oldStartedAt, + endedAt: oldKilledAt, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: oldKilledAt }, + cleanupHandled: true, + cleanupCompletedAt: oldKilledAt, + runTimeoutSeconds: 60, + }); + mod.addSubagentRunForTests({ + runId: "run-new-no-start", + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "new generation without persisted start time", + cleanup: "keep", + createdAt: newStartedAt, + startedAt: newStartedAt, + generation: 2, + }); + + await mod.testing.sweepOnceForTests(); + + expect(findTaskByRunIdForStatus("run-old-no-start")).toMatchObject({ + status: "cancelled", + }); + expect(findTaskByRunIdForStatus("run-old-no-start")?.status).not.toBe("timed_out"); + expect( + mod + .listSubagentRunsForRequester("agent:main:main") + .some((entry) => entry.runId === "run-old-no-start"), + ).toBe(false); + } finally { + resetTaskRegistryForTests({ persist: false }); + resetTaskFlowRegistryForTests({ persist: false }); + } + }); + + it("does not restore session-write ownership after a successor is released", async () => { + const childSessionKey = "agent:main:subagent:released-timing-owner"; + mocks.callGateway.mockImplementation(async (request: { method?: string }) => + request.method === "agent.wait" ? { status: "pending" } : {}, + ); + mocks.loadSessionStore.mockReturnValue({ + [childSessionKey]: { sessionId: "sess-released-timing-owner", updatedAt: 1 }, + }); + let releaseTimingWrite: (() => void) | undefined; + let timingWriteStarted: (() => void) | undefined; + const timingWriteStartedPromise = new Promise((resolve) => { + timingWriteStarted = resolve; + }); + const timingWriteFinished = new Promise((resolveFinished) => { + mocks.patchSessionEntry.mockImplementationOnce(async (scope, update) => { + timingWriteStarted?.(); + await new Promise((resolve) => { + releaseTimingWrite = resolve; + }); + const store = mocks.loadSessionStore(scope.storePath, { clone: false }) as Record< + string, + SessionEntry + >; + const current = store[scope.sessionKey]; + const patch = await update(current, { existingEntry: { ...current } }); + if (patch) { + mocks.updateSessionStore(scope.storePath, () => {}); + } + resolveFinished(); + return patch ? { ...current, ...patch } : current; + }); + }); + + mod.registerSubagentRun({ + runId: "run-released-timing-old", + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "old timing owner", + cleanup: "keep", + }); + expect( + mod.markSubagentRunTerminated({ + runId: "run-released-timing-old", + reason: "manual kill", + }), + ).toBe(1); + await timingWriteStartedPromise; + + mod.registerSubagentRun({ + runId: "run-released-timing-new", + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "new timing owner", + cleanup: "keep", + }); + mod.releaseSubagentRun("run-released-timing-new"); + releaseTimingWrite?.(); + await timingWriteFinished; + + const oldRun = mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === "run-released-timing-old"); + expect(oldRun?.killReconciliation?.supersededAt).toBeTypeOf("number"); + expect(mocks.updateSessionStore).not.toHaveBeenCalled(); + }); + + it("reconciles an old completion without touching the newer session generation", async () => { + const oldStartedAt = Date.parse("2026-03-24T11:50:00Z"); + const oldKilledAt = Date.parse("2026-03-24T11:55:00Z"); + const oldCompletedAt = Date.parse("2026-03-24T11:56:00Z"); + const newStartedAt = Date.parse("2026-03-24T11:58:00Z"); + const childSessionKey = "agent:main:subagent:reused-completed"; + mocks.loadSessionStore.mockReturnValue({ + [childSessionKey]: { + sessionId: "sess-reused-completed", + updatedAt: oldCompletedAt, + status: "done", + startedAt: oldStartedAt, + endedAt: oldCompletedAt, + }, + }); + mocks.getAgentRunContext.mockImplementation((runId: string) => + runId === "run-new-generation-after-completion" ? ({} as never) : undefined, + ); + mocks.getGlobalHookRunner.mockReturnValue({ + hasHooks: (hookName: string) => hookName === "subagent_ended", + runSubagentEnded: mocks.runSubagentEnded, + } as never); + mod.addSubagentRunForTests({ + runId: "run-old-completed-tombstone", + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "old completed generation", + cleanup: "delete", + createdAt: oldStartedAt, + startedAt: oldStartedAt, + sessionStartedAt: oldStartedAt, + endedAt: oldKilledAt, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: oldKilledAt }, + cleanupHandled: true, + cleanupCompletedAt: oldKilledAt, + }); + mod.addSubagentRunForTests({ + runId: "run-new-generation-after-completion", + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "new generation", + cleanup: "keep", + createdAt: newStartedAt, + startedAt: newStartedAt, + sessionStartedAt: newStartedAt, + }); + + await mod.testing.sweepOnceForTests(); + + const runs = mod.listSubagentRunsForRequester("agent:main:main"); + expect(runs.some((entry) => entry.runId === "run-old-completed-tombstone")).toBe(false); + const newRun = runs.find((entry) => entry.runId === "run-new-generation-after-completion"); + expect(newRun).toBeDefined(); + expect(newRun?.endedAt).toBeUndefined(); + expect(newRun?.outcome).toBeUndefined(); + expect( + mocks.callGateway.mock.calls.some( + ([request]) => (request as { method?: string } | undefined)?.method === "sessions.delete", + ), + ).toBe(false); + expect( + mocks.patchSessionEntry.mock.calls.some( + ([scope]) => (scope as SessionAccessScope).sessionKey === childSessionKey, + ), + ).toBe(false); + expect( + mocks.emitSessionLifecycleEvent.mock.calls.some( + ([event]) => (event as { sessionKey?: string }).sessionKey === childSessionKey, + ), + ).toBe(false); + expect(mocks.runSubagentEnded).not.toHaveBeenCalled(); + expect( + mocks.onSubagentEnded.mock.calls.some( + ([params]) => params.childSessionKey === childSessionKey, + ), + ).toBe(false); }); it("uses session-store start time when sweeping stale explicit-timeout runs", async () => { @@ -2618,7 +4042,39 @@ describe("subagent registry seam flow", () => { "updated child session store entry", ); - expect(mocks.persistSubagentRunsToDisk).toHaveBeenCalledTimes(6); + expect(mocks.persistSubagentRunsToDisk).toHaveBeenCalledTimes(4); + expect(mocks.persistSubagentRunsToDiskOrThrow).toHaveBeenCalledTimes(2); + }); + + it("retries completion after a transient durable registry write failure", async () => { + mocks.persistSubagentRunsToDiskOrThrow + .mockImplementationOnce(() => {}) + .mockImplementationOnce(() => { + throw new Error("transient disk error"); + }) + .mockImplementation(() => {}); + + mod.registerSubagentRun({ + runId: "run-retry-durable-completion", + childSessionKey: "agent:main:subagent:retry-durable-completion", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "retry durable completion", + cleanup: "keep", + expectsCompletionMessage: false, + }); + + await waitForFast(() => { + const run = mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === "run-retry-durable-completion"); + expect(run).toMatchObject({ + endedAt: 222, + endedReason: SUBAGENT_ENDED_REASON_COMPLETE, + outcome: { status: "ok", startedAt: 111, endedAt: 222 }, + }); + expect(mocks.persistSubagentRunsToDiskOrThrow).toHaveBeenCalledTimes(3); + }); }); it("throws and removes the entry when the initial durable registry write fails", () => { @@ -2644,6 +4100,110 @@ describe("subagent registry seam flow", () => { ).toBeUndefined(); }); + it("retains an already-running replacement when its durable write fails", () => { + mocks.callGateway.mockImplementation(async (request: { method?: string }) => + request.method === "agent.wait" ? { status: "pending" } : {}, + ); + mod.registerSubagentRun({ + runId: "run-replacement-persist-old", + childSessionKey: "agent:main:subagent:replacement-persist", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "keep live successor tracked", + cleanup: "keep", + }); + mocks.persistSubagentRunsToDiskOrThrow.mockImplementationOnce(() => { + throw new Error("disk full"); + }); + + expect( + mod.replaceSubagentRunAfterSteer({ + previousRunId: "run-replacement-persist-old", + nextRunId: "run-replacement-persist-new", + }), + ).toBe(true); + + const runs = mod.listSubagentRunsForRequester("agent:main:main"); + expect(runs.some((entry) => entry.runId === "run-replacement-persist-old")).toBe(false); + expect(runs).toEqual([ + expect.objectContaining({ + runId: "run-replacement-persist-new", + taskRunId: "run-replacement-persist-old", + }), + ]); + expect(mocks.persistSubagentRunsToDisk).toHaveBeenCalled(); + }); + + it("rolls back an older kill ownership boundary when registration persistence fails", () => { + const childSessionKey = "agent:main:subagent:registration-rollback"; + mod.addSubagentRunForTests({ + runId: "run-registration-rollback-old", + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "preserve old ownership", + cleanup: "keep", + createdAt: Date.now() - 1_000, + endedAt: Date.now() - 500, + endedReason: "subagent-killed", + suppressAnnounceReason: "killed", + killReconciliation: { killedAt: Date.now() - 500 }, + }); + mocks.persistSubagentRunsToDiskOrThrow.mockImplementationOnce(() => { + throw new Error("disk full"); + }); + + expect(() => + mod.registerSubagentRun({ + runId: "run-registration-rollback-new", + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "new generation", + cleanup: "keep", + }), + ).toThrowError("disk full"); + + const oldRun = mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === "run-registration-rollback-old"); + expect(oldRun?.killReconciliation).toEqual({ killedAt: Date.now() - 500 }); + expect( + mod + .listSubagentRunsForRequester("agent:main:main") + .some((entry) => entry.runId === "run-registration-rollback-new"), + ).toBe(false); + }); + + it("rolls back a killed tombstone when its durable registry write fails", () => { + mocks.callGateway.mockImplementation(async (request: { method?: string }) => + request.method === "agent.wait" ? { status: "pending" } : {}, + ); + const runId = "run-kill-persist-failure"; + mod.registerSubagentRun({ + runId, + childSessionKey: "agent:main:subagent:kill-persist-failure", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "keep kill state atomic", + cleanup: "keep", + }); + mocks.persistSubagentRunsToDiskOrThrow.mockImplementationOnce(() => { + throw new Error("disk full"); + }); + + expect(() => mod.markSubagentRunTerminated({ runId, reason: "manual kill" })).toThrowError( + "disk full", + ); + + const run = mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === runId); + expect(run?.endedAt).toBeUndefined(); + expect(run?.endedReason).toBeUndefined(); + expect(findTaskByRunIdForStatus(runId)).toMatchObject({ status: "running" }); + }); + it("continues completion announce cleanup when lifecycle cleanup fails", async () => { mocks.cleanupBrowserSessionsForLifecycleEnd.mockRejectedValueOnce( new Error("browser cleanup unavailable"), @@ -2752,7 +4312,7 @@ describe("subagent registry seam flow", () => { expect(run?.outcome?.status).toBe("error"); }); - it("announces aborted lifecycle end events as killed subagent failures", async () => { + it("publishes aborted lifecycle end events only after killed reconciliation", async () => { mocks.callGateway.mockImplementation(async (request: { method?: string }) => { if (request.method === "agent.wait") { return { status: "pending" }; @@ -2800,8 +4360,16 @@ describe("subagent registry seam flow", () => { }); await waitForFast(() => { - expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledTimes(1); + const run = mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === "run-aborted-end"); + expect(run?.endedReason).toBe("subagent-killed"); + expect(run?.suppressAnnounceReason).toBe("killed"); }); + expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled(); + + await mod.testing.sweepOnceForTests(); + await waitForFast(() => expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledTimes(1)); const announceParams = expectRecordFields( getMockCallArg(mocks.runSubagentAnnounceFlow, 0, 0, "aborted announce"), { childRunId: "run-aborted-end" }, @@ -2819,17 +4387,19 @@ describe("subagent registry seam flow", () => { "aborted announce outcome", ); - const run = mod - .listSubagentRunsForRequester("agent:main:main") - .find((entry) => entry.runId === "run-aborted-end"); - expect(run?.endedReason).toBe("subagent-killed"); - expect(run?.outcome?.status).toBe("error"); + await waitForFast(() => { + expect( + mod + .listSubagentRunsForRequester("agent:main:main") + .some((entry) => entry.runId === "run-aborted-end"), + ).toBe(false); + }); await vi.advanceTimersByTimeAsync(20_000); expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledTimes(1); }); - it("announces restart lifecycle end events as killed subagent failures", async () => { + it("publishes restart lifecycle end events only after killed reconciliation", async () => { mocks.callGateway.mockImplementation(async (request: { method?: string }) => { if (request.method === "agent.wait") { return { status: "pending" }; @@ -2871,12 +4441,15 @@ describe("subagent registry seam flow", () => { .find((entry) => entry.runId === "run-restart-end"); expect(run?.endedReason).toBe("subagent-killed"); expect(run?.outcome?.status).toBe("error"); - expect(run?.cleanupCompletedAt).toBeTypeOf("number"); - expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledTimes(1); + expect(run?.suppressAnnounceReason).toBe("killed"); }); + expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled(); + + await mod.testing.sweepOnceForTests(); + await waitForFast(() => expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledTimes(1)); }); - it("announces restart lifecycle error events as killed subagent failures", async () => { + it("publishes restart lifecycle error events only after killed reconciliation", async () => { mocks.callGateway.mockImplementation(async (request: { method?: string }) => { if (request.method === "agent.wait") { return { status: "pending" }; @@ -2919,21 +4492,24 @@ describe("subagent registry seam flow", () => { .find((entry) => entry.runId === "run-restart-error"); expect(run?.endedReason).toBe("subagent-killed"); expect(run?.outcome?.status).toBe("error"); - expect(run?.cleanupCompletedAt).toBeTypeOf("number"); - expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledTimes(1); + expect(run?.suppressAnnounceReason).toBe("killed"); }); + expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled(); + + await mod.testing.sweepOnceForTests(); + await waitForFast(() => expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledTimes(1)); }); - it("resumes ended cleanup when lifecycle killed completion rejects before cleanup", async () => { + it("finishes canonical killed cleanup when its best-effort hook fails", async () => { mocks.callGateway.mockImplementation(async (request: { method?: string }) => { if (request.method === "agent.wait") { return { status: "pending" }; } return {}; }); - mocks.ensureRuntimePluginsLoaded - .mockRejectedValueOnce(new Error("runtime unavailable before cleanup")) - .mockRejectedValueOnce(new Error("runtime still unavailable before cleanup")); + mocks.ensureRuntimePluginsLoaded.mockRejectedValueOnce( + new Error("runtime unavailable during killed hook"), + ); mod.registerSubagentRun({ runId: "run-killed-recovery", @@ -2974,10 +4550,20 @@ describe("subagent registry seam flow", () => { const run = mod .listSubagentRunsForRequester("agent:main:main") .find((entry) => entry.runId === "run-killed-recovery"); - expect(mocks.ensureRuntimePluginsLoaded).toHaveBeenCalledTimes(2); expect(run?.outcome?.status).toBe("error"); expect(run?.endedReason).toBe("subagent-killed"); - expect(run?.cleanupCompletedAt).toBeTypeOf("number"); + expect(run?.suppressAnnounceReason).toBe("killed"); + }); + expect(mocks.ensureRuntimePluginsLoaded).not.toHaveBeenCalled(); + + await mod.testing.sweepOnceForTests(); + await waitForFast(() => { + expect(mocks.ensureRuntimePluginsLoaded).toHaveBeenCalled(); + expect( + mod + .listSubagentRunsForRequester("agent:main:main") + .some((entry) => entry.runId === "run-killed-recovery"), + ).toBe(false); }); expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled(); }); @@ -3023,7 +4609,7 @@ describe("subagent registry seam flow", () => { }); await waitForFast(() => { - expect(mocks.ensureRuntimePluginsLoaded).toHaveBeenCalledTimes(2); + expect(mocks.ensureRuntimePluginsLoaded.mock.calls.length).toBeGreaterThanOrEqual(2); const run = mod .listSubagentRunsForRequester("agent:main:main") .find((entry) => entry.runId === "run-hook-retry"); @@ -3098,6 +4684,62 @@ describe("subagent registry seam flow", () => { expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledTimes(1); }); + it("emits the canonical ended hook when plugin loading overlaps a newer completion", async () => { + mocks.callGateway.mockImplementation(async (request: { method?: string }) => + request.method === "agent.wait" ? { status: "pending" } : {}, + ); + mocks.getGlobalHookRunner.mockReturnValue({ + hasHooks: (hookName: string) => hookName === "subagent_ended", + runSubagentEnded: mocks.runSubagentEnded, + } as never); + let releaseOldPluginLoad: (() => void) | undefined; + const oldPluginLoad = new Promise((resolve) => { + releaseOldPluginLoad = resolve; + }); + mocks.ensureRuntimePluginsLoaded.mockImplementationOnce(async () => { + await oldPluginLoad; + }); + + mod.registerSubagentRun({ + runId: "run-hook-timeout-then-ok", + childSessionKey: "agent:main:subagent:hook-timeout-then-ok", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "publish only the canonical hook", + cleanup: "keep", + expectsCompletionMessage: false, + }); + const lifecycleHandler = mocks.onAgentEvent.mock.calls.at(-1)?.[0] as + | ((evt: { runId: string; stream: string; data: Record }) => void) + | undefined; + expect(lifecycleHandler).toBeTypeOf("function"); + + lifecycleHandler?.({ + runId: "run-hook-timeout-then-ok", + stream: "lifecycle", + data: { phase: "end", startedAt: 100, endedAt: 200, aborted: true }, + }); + await vi.advanceTimersByTimeAsync(15_000); + await waitForFast(() => expect(mocks.ensureRuntimePluginsLoaded).toHaveBeenCalledTimes(1)); + + lifecycleHandler?.({ + runId: "run-hook-timeout-then-ok", + stream: "lifecycle", + data: { phase: "end", startedAt: 100, endedAt: 250 }, + }); + await waitForFast(() => expect(mocks.runSubagentEnded).toHaveBeenCalledTimes(1)); + releaseOldPluginLoad?.(); + await Promise.resolve(); + await Promise.resolve(); + + expect(mocks.runSubagentEnded).toHaveBeenCalledTimes(1); + expectRecordFields( + getMockCallArg(mocks.runSubagentEnded, 0, 0, "canonical ended hook"), + { reason: "subagent-complete", outcome: "ok", error: undefined }, + "canonical ended hook", + ); + }); + it("deletes delete-mode completion runs when announce cleanup gives up after retry limit", async () => { mocks.runSubagentAnnounceFlow.mockResolvedValue(false); const endedAt = Date.parse("2026-03-24T12:00:00Z"); @@ -3428,7 +5070,13 @@ describe("subagent registry seam flow", () => { ); }); - it("loads runtime plugins before emitting killed subagent ended hooks", async () => { + it("defers the killed hook until the provisional result reconciles", async () => { + mocks.callGateway.mockImplementation(async (request: { method?: string }) => { + if (request.method === "agent.wait") { + return { status: "pending" }; + } + return {}; + }); const endedHookRunner = { hasHooks: (hookName: string) => hookName === "subagent_ended", runSubagentEnded: mocks.runSubagentEnded, @@ -3446,6 +5094,7 @@ describe("subagent registry seam flow", () => { requesterOrigin: { channel: "quietchat", accountId: "acct-1" }, task: "kill after init", cleanup: "keep", + expectsCompletionMessage: false, workspaceDir: "/tmp/killed-workspace", }); @@ -3466,6 +5115,11 @@ describe("subagent registry seam flow", () => { endedAt: killedAt, elapsedMs: 0, }); + expect(mocks.runSubagentEnded).not.toHaveBeenCalled(); + mocks.ensureRuntimePluginsLoaded.mockClear(); + + vi.setSystemTime(killedAt + 5 * 60_000); + await mod.testing.sweepOnceForTests(); await waitForFast(() => { expect(mocks.ensureRuntimePluginsLoaded).toHaveBeenCalledWith({ config: { @@ -3476,6 +5130,7 @@ describe("subagent registry seam flow", () => { allowGatewaySubagentBinding: true, }); }); + await waitForFast(() => expect(mocks.runSubagentEnded).toHaveBeenCalled()); expectRecordFields( getMockCallArg(mocks.runSubagentEnded, 0, 0, "subagent ended hook"), { @@ -3499,7 +5154,7 @@ describe("subagent registry seam flow", () => { ); }); - it("deletes killed delete-mode runs and notifies deleted cleanup", async () => { + it("keeps killed delete-mode runs as reconciliation tombstones", async () => { mod.registerSubagentRun({ runId: "run-killed-delete", childSessionKey: "agent:main:subagent:killed-delete", @@ -3520,14 +5175,134 @@ describe("subagent registry seam flow", () => { mod .listSubagentRunsForRequester("agent:main:main") .find((entry) => entry.runId === "run-killed-delete"), - ).toBeUndefined(); - await waitForFast(() => { - expect(mocks.onSubagentEnded).toHaveBeenCalledWith({ - childSessionKey: "agent:main:subagent:killed-delete", - reason: "deleted", - workspaceDir: "/tmp/killed-delete-workspace", - }); + ).toMatchObject({ + cleanup: "delete", + cleanupHandled: true, + endedReason: "subagent-killed", + outcome: { status: "error", error: "manual kill" }, }); + expect( + mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === "run-killed-delete")?.archiveAtMs, + ).toBeUndefined(); + expect(findTaskByRunIdForStatus("run-killed-delete")).toMatchObject({ + status: "cancelled", + error: SUBAGENT_KILL_TASK_ERROR, + deliveryStatus: "pending", + }); + expect(mocks.onSubagentEnded).not.toHaveBeenCalled(); + }); + + it("does not replace durable task completion when a provisional kill is replayed", () => { + const runId = "run-repeated-kill-after-task-success"; + const childSessionKey = "agent:main:subagent:repeated-kill-after-task-success"; + mod.registerSubagentRun({ + runId, + childSessionKey, + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "finish before registry reconciliation persists", + cleanup: "keep", + }); + + expect(mod.markSubagentRunTerminated({ runId, reason: "manual kill" })).toBe(1); + expect( + finalizeTaskRunByRunId({ + runId, + runtime: "subagent", + sessionKey: childSessionKey, + status: "succeeded", + endedAt: Date.now() + 1, + lastEventAt: Date.now() + 1, + progressSummary: "durable provider completion", + }), + ).toHaveLength(1); + + // Simulate task-first completion followed by a failed registry commit and + // a repeated admin kill against the retained reconciliation tombstone. + expect(mod.markSubagentRunTerminated({ runId, reason: "manual kill" })).toBe(1); + expect(findTaskByRunIdForStatus(runId)).toMatchObject({ + status: "succeeded", + progressSummary: "durable provider completion", + }); + }); + + it("suppresses task delivery immediately when requester teardown kills a run", () => { + mod.registerSubagentRun({ + runId: "run-requester-teardown", + childSessionKey: "agent:main:subagent:requester-teardown", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "stop without reinjecting", + cleanup: "keep", + }); + + expect( + mod.markSubagentRunTerminated({ + runId: "run-requester-teardown", + reason: "manual kill", + suppressTaskDelivery: true, + }), + ).toBe(1); + + expect(findTaskByRunIdForStatus("run-requester-teardown")).toMatchObject({ + status: "cancelled", + error: SUBAGENT_KILL_TASK_ERROR, + deliveryStatus: "not_applicable", + }); + }); + + it("emits only the canonical completion hook when completion beats a provisional kill", async () => { + mocks.callGateway.mockImplementation(async (request: { method?: string }) => { + if (request.method === "agent.wait") { + return { status: "pending" }; + } + return {}; + }); + mocks.getGlobalHookRunner.mockReturnValue({ + hasHooks: (hookName: string) => hookName === "subagent_ended", + runSubagentEnded: mocks.runSubagentEnded, + } as never); + mod.registerSubagentRun({ + runId: "run-killed-hook-race", + childSessionKey: "agent:main:subagent:killed-hook-race", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "finish while kill hook is publishing", + cleanup: "keep", + expectsCompletionMessage: false, + }); + const lifecycleHandler = mocks.onAgentEvent.mock.calls.at(-1)?.[0] as + | ((evt: { runId: string; stream: string; data: Record }) => void) + | undefined; + expect(lifecycleHandler).toBeTypeOf("function"); + + expect( + mod.markSubagentRunTerminated({ + runId: "run-killed-hook-race", + reason: "manual kill", + }), + ).toBe(1); + expect(mocks.runSubagentEnded).not.toHaveBeenCalled(); + + lifecycleHandler?.({ + runId: "run-killed-hook-race", + stream: "lifecycle", + data: { phase: "end", startedAt: 100, endedAt: 200 }, + }); + await waitForFast(() => { + const run = mod + .listSubagentRunsForRequester("agent:main:main") + .find((entry) => entry.runId === "run-killed-hook-race"); + expect(run?.endedReason).toBe("subagent-complete"); + }); + expect(mocks.runSubagentEnded).toHaveBeenCalledTimes(1); + expectRecordFields( + getMockCallArg(mocks.runSubagentEnded, 0, 0, "exactly-once completion hook"), + { reason: "subagent-complete", outcome: "ok", error: undefined }, + "exactly-once completion hook", + ); }); it("removes attachments for killed delete-mode runs", async () => { diff --git a/src/agents/subagent-registry.ts b/src/agents/subagent-registry.ts index 97db556cc93..fb8f89bf372 100644 --- a/src/agents/subagent-registry.ts +++ b/src/agents/subagent-registry.ts @@ -14,6 +14,10 @@ import { createSubsystemLogger } from "../logging/subsystem.js"; import { formatBlockedLivenessError, isBlockedLivenessState } from "../shared/agent-liveness.js"; import { createLazyImportLoader, createLazyPromiseLoader } from "../shared/lazy-promise.js"; import { importRuntimeModule } from "../shared/runtime-import.js"; +import { SUBAGENT_KILL_TASK_ERROR } from "../tasks/detached-task-runtime-contract.js"; +import { finalizeTaskRunByRunId, findDetachedTaskRun } from "../tasks/detached-task-runtime.js"; +import { isProvisionalSubagentKillTask } from "../tasks/task-cancellation-state.js"; +import type { TaskRecord } from "../tasks/task-registry.types.js"; import { normalizeDeliveryContext } from "../utils/delivery-context.shared.js"; import type { DeliveryContext } from "../utils/delivery-context.types.js"; import { @@ -35,6 +39,7 @@ import { isDeliverySuspended, } from "./subagent-delivery-state.js"; import { + SUBAGENT_ENDED_OUTCOME_KILLED, SUBAGENT_ENDED_REASON_COMPLETE, SUBAGENT_ENDED_REASON_ERROR, SUBAGENT_ENDED_REASON_KILLED, @@ -47,6 +52,7 @@ import { import { ANNOUNCE_EXPIRY_MS, MAX_ANNOUNCE_RETRY_COUNT, + PROVISIONAL_KILL_RECONCILIATION_MS, reconcileOrphanedRestoredRuns, reconcileOrphanedRun, resolveAnnounceRetryDelayMs, @@ -81,6 +87,11 @@ import { } from "./subagent-registry-state.js"; import { configureSubagentRegistrySteerRuntime } from "./subagent-registry-steer-runtime.js"; import type { SubagentRunRecord } from "./subagent-registry.types.js"; +import { compareSubagentRunGeneration } from "./subagent-run-generation.js"; +import { + resolveSubagentRunDeadlineMs, + resolveSubagentRunEffectiveEndedAt, +} from "./subagent-run-timeout.js"; import { loadSubagentSessionEntry, resolveCompletionFromSessionEntry, @@ -121,7 +132,9 @@ type SubagentRegistryDeps = { restoreSubagentRunsFromDisk: typeof restoreSubagentRunsFromDisk; runSubagentAnnounceFlow: SubagentAnnounceModule["runSubagentAnnounceFlow"]; ensureContextEnginesInitialized?: () => void; - ensureRuntimePluginsLoaded?: typeof ensureRuntimePluginsLoadedFn; + ensureRuntimePluginsLoaded?: ( + params: Parameters[0], + ) => void | Promise; resolveContextEngine?: ( cfg?: OpenClawConfig, options?: ResolveContextEngineOptions, @@ -252,7 +265,7 @@ async function ensureSubagentRegistryPluginRuntimeLoaded(params: { }) { const ensureRuntimePluginsLoaded = subagentRegistryDeps.ensureRuntimePluginsLoaded; if (ensureRuntimePluginsLoaded) { - ensureRuntimePluginsLoaded(params); + await ensureRuntimePluginsLoaded(params); return; } (await loadRuntimePluginsModule()).ensureRuntimePluginsLoaded(params); @@ -281,6 +294,78 @@ function persistSubagentRunsOrThrow() { subagentRegistryDeps.persistSubagentRunsToDiskOrThrow(subagentRuns); } +function findSubagentTaskForRun(entry: SubagentRunRecord) { + const nextRunCreatedAt = findNextSubagentRunCreatedAt(entry); + const generationStartedAt = entry.sessionStartedAt ?? entry.createdAt; + return findDetachedTaskRun({ + runId: entry.taskRunId ?? entry.runId, + runtime: "subagent", + sessionKey: entry.childSessionKey, + createdAtOrAfter: generationStartedAt, + createdBefore: nextRunCreatedAt, + // Steer/wake replaces the registry run ID while retaining the original + // task row. Only those continuations may adopt a session-scoped task. + allowSessionFallback: + entry.taskRunId === undefined && + typeof entry.sessionStartedAt === "number" && + entry.sessionStartedAt < entry.createdAt, + }); +} + +function findNextSubagentRunCreatedAt(entry: SubagentRunRecord): number | undefined { + let nextCreatedAt = entry.killReconciliation?.supersededAt; + for (const candidate of subagentRuns.values()) { + if ( + candidate.runId === entry.runId || + candidate.childSessionKey !== entry.childSessionKey || + compareSubagentRunGeneration(candidate, entry) <= 0 + ) { + continue; + } + nextCreatedAt = Math.min(nextCreatedAt ?? candidate.createdAt, candidate.createdAt); + } + return nextCreatedAt; +} + +function resolveCompletionFromTerminalTask( + task: TaskRecord | undefined, + entry: SubagentRunRecord, +): + | { + startedAt?: number; + endedAt: number; + outcome: SubagentRunOutcome; + reason: SubagentLifecycleEndedReason; + completionSnapshot: { resultText: string | null; capturedAt: number }; + } + | undefined { + if ( + !task || + typeof task.endedAt !== "number" || + (task.status !== "succeeded" && task.status !== "failed" && task.status !== "timed_out") + ) { + return undefined; + } + const outcome: SubagentRunOutcome = + task.status === "succeeded" + ? { status: "ok" } + : task.status === "timed_out" + ? { status: "timeout" } + : { status: "error", error: task.error }; + return { + // A steer continuation keeps the original task row but owns a new timeout + // window. Replay against the current registry generation, not task history. + startedAt: entry.startedAt ?? task.startedAt, + endedAt: task.endedAt, + outcome, + reason: task.status === "failed" ? SUBAGENT_ENDED_REASON_ERROR : SUBAGENT_ENDED_REASON_COMPLETE, + completionSnapshot: { + resultText: task.progressSummary ?? task.terminalSummary ?? null, + capturedAt: task.endedAt, + }, + }; +} + export function scheduleSubagentOrphanRecovery(params?: { delayMs?: number; maxRetries?: number }) { const now = Date.now(); if (now - lastOrphanRecoveryScheduleAt < ORPHAN_RECOVERY_DEBOUNCE_MS) { @@ -362,6 +447,7 @@ type CompleteSubagentRunParams = { accountId?: string; triggerCleanup: boolean; startedAt?: number; + suppressSessionEffects?: boolean; }; async function completeSubagentRunWithRecovery(params: CompleteSubagentRunParams, source: string) { @@ -379,12 +465,7 @@ async function completeSubagentRunWithRecovery(params: CompleteSubagentRunParams } const current = subagentRuns.get(params.runId); - if ( - !current || - typeof current.endedAt !== "number" || - typeof current.cleanupCompletedAt === "number" || - current.pauseReason === "sessions_yield" - ) { + if (!current) { return; } @@ -401,6 +482,12 @@ async function completeSubagentRunWithRecovery(params: CompleteSubagentRunParams } const latest = subagentRuns.get(params.runId); + if (latest && typeof latest.endedAt !== "number") { + // The durable write rolled the in-memory entry back. Preserve the original + // completion through the normal persisted-session recovery path. + scheduleSubagentOrphanRecovery({ delayMs: 1_000 }); + return; + } if ( !latest || typeof latest.endedAt !== "number" || @@ -557,6 +644,7 @@ async function emitSubagentEndedHookForRun(params: { reason?: SubagentLifecycleEndedReason; sendFarewell?: boolean; accountId?: string; + isCurrent?: () => boolean; }) { if (params.entry.endedHookEmittedAt) { return; @@ -567,8 +655,17 @@ async function emitSubagentEndedHookForRun(params: { workspaceDir: params.entry.workspaceDir, allowGatewaySubagentBinding: true, }); - const reason = params.reason ?? params.entry.endedReason ?? SUBAGENT_ENDED_REASON_COMPLETE; - const outcome = resolveLifecycleOutcomeFromRunOutcome(params.entry.outcome); + if (params.entry.endedHookEmittedAt || params.isCurrent?.() === false) { + return; + } + // Plugin loading yields after the terminal lock is released. Resolve the + // event from the canonical row only after that boundary so an older callback + // cannot claim the exactly-once hook with a superseded timeout or error. + const reason = params.entry.endedReason ?? params.reason ?? SUBAGENT_ENDED_REASON_COMPLETE; + const outcome = + reason === SUBAGENT_ENDED_REASON_KILLED + ? SUBAGENT_ENDED_OUTCOME_KILLED + : resolveLifecycleOutcomeFromRunOutcome(params.entry.outcome); const error = params.entry.outcome?.status === "error" ? params.entry.outcome.error : undefined; await emitSubagentEndedHookOnce({ entry: params.entry, @@ -587,12 +684,15 @@ const subagentLifecycleController = createSubagentRegistryLifecycleController({ resumedRuns, subagentAnnounceTimeoutMs: SUBAGENT_ANNOUNCE_TIMEOUT_MS, persist: persistSubagentRuns, + persistOrThrow: persistSubagentRunsOrThrow, clearPendingLifecycleError, countPendingDescendantRuns, suppressAnnounceForSteerRestart, + resolveSubagentTask: findSubagentTaskForRun, shouldEmitEndedHookForRun, emitSubagentEndedHookForRun, notifyContextEngineSubagentEnded, + retireSupersededRun: retireSupersededSubagentRun, resumeSubagentRun, callGateway: (request) => subagentRegistryDeps.callGateway(request), captureSubagentCompletionReply: (sessionKey, options) => @@ -675,6 +775,12 @@ function resumeSubagentRun(runId: string) { } if (typeof entry.endedAt === "number" && entry.endedAt > 0) { + if (entry.killReconciliation) { + // Restored kills remain reconciliation tombstones; only the sweeper may + // accept late provider completion or stabilize their task cancellation. + resumedRuns.add(runId); + return; + } const orphanReason = resolveSubagentRunOrphanReason({ entry }); if (orphanReason) { if ( @@ -862,6 +968,21 @@ async function discardSuspendedPendingFinalDelivery( } } +async function retireSupersededSubagentRun(runId: string, entry: SubagentRunRecord): Promise { + const transcriptFile = entry.execution?.transcriptFile; + clearPendingLifecycleError(runId); + subagentRuns.delete(runId); + const transcriptStillOwned = Array.from(subagentRuns.values()).some( + (candidate) => candidate.execution?.transcriptFile === transcriptFile, + ); + if (transcriptFile && !transcriptStillOwned) { + await removeInternalSessionEffectsTranscript(transcriptFile); + } + if (entry.cleanup === "delete" || !entry.retainAttachmentsOnKeep) { + await safeRemoveAttachmentsDir(entry); + } +} + async function sweepSubagentRuns() { if (sweepInProgress) { return; @@ -979,6 +1100,226 @@ async function sweepSubagentRuns() { } } + if (entry.killReconciliation) { + const killReconciliation = entry.killReconciliation; + const taskResolutionBeforeReconciliation = findSubagentTaskForRun(entry); + const taskBeforeReconciliation = taskResolutionBeforeReconciliation.task; + const nextRunCreatedAt = findNextSubagentRunCreatedAt(entry); + const hasStableTaskCancellation = + taskBeforeReconciliation?.status === "cancelled" && + !isProvisionalSubagentKillTask(taskBeforeReconciliation); + const killedAt = killReconciliation.killedAt; + const taskCompletion = + nextRunCreatedAt === undefined + ? resolveCompletionFromTerminalTask(taskBeforeReconciliation, entry) + : undefined; + if (taskCompletion) { + // Provider reconciliation commits the non-publishing task ledger first. + // If the registry write was interrupted, replay that durable projection + // before the provisional kill can age into a contradictory cancellation. + await completeSubagentRunWithRecovery( + { + runId, + ...taskCompletion, + sendFarewell: true, + accountId: entry.requesterOrigin?.accountId, + triggerCleanup: true, + }, + "sweeper-provisional-kill-task-completion", + ); + const current = subagentRuns.get(runId); + if (current !== entry || current.killReconciliation !== killReconciliation) { + continue; + } + // A failed registry retry must preserve the replayable task evidence. + continue; + } + const reconcileAtMs = killedAt + PROVISIONAL_KILL_RECONCILIATION_MS; + if (reconcileAtMs > now) { + // Even durable cancellation keeps the evidence window open: a + // provider result persisted before killedAt remains canonical. + continue; + } + const sessionEntry = loadSubagentSessionEntry({ + childSessionKey: entry.childSessionKey, + storeCache, + }); + const completion = resolveCompletionFromSessionEntry(sessionEntry, now, { + notBeforeMs: entry.startedAt ?? entry.createdAt, + }); + const completionEndedAt = completion + ? resolveSubagentRunEffectiveEndedAt(entry, completion.endedAt, completion.startedAt) + : undefined; + const completionDeadline = completion + ? resolveSubagentRunDeadlineMs(entry, completion.startedAt) + : undefined; + const killedSnapshotExpiredDeadline = + completion?.reason === SUBAGENT_ENDED_REASON_KILLED && + completionDeadline !== undefined && + completion.endedAt > completionDeadline + ? completionDeadline + : undefined; + const completionCanOverrideCancellation = + !hasStableTaskCancellation || (completionEndedAt ?? Number.POSITIVE_INFINITY) < killedAt; + const completionBelongsToGeneration = + nextRunCreatedAt === undefined || + (completion != null && completion.endedAt < nextRunCreatedAt); + if ( + completion && + completionEndedAt !== undefined && + completionCanOverrideCancellation && + completionBelongsToGeneration && + (completion.reason !== SUBAGENT_ENDED_REASON_KILLED || + killedSnapshotExpiredDeadline !== undefined) + ) { + const hasNewerGeneration = nextRunCreatedAt !== undefined; + await completeSubagentRunWithRecovery( + { + runId, + startedAt: completion.startedAt, + endedAt: killedSnapshotExpiredDeadline ?? completion.endedAt, + outcome: + killedSnapshotExpiredDeadline !== undefined + ? { status: "timeout" } + : completion.outcome, + reason: + killedSnapshotExpiredDeadline !== undefined + ? SUBAGENT_ENDED_REASON_COMPLETE + : completion.reason, + sendFarewell: true, + accountId: entry.requesterOrigin?.accountId, + triggerCleanup: !hasNewerGeneration, + suppressSessionEffects: hasNewerGeneration, + }, + "sweeper-provisional-kill-completion", + ); + if ( + hasNewerGeneration && + subagentRuns.get(runId) === entry && + entry.endedReason !== SUBAGENT_ENDED_REASON_KILLED + ) { + await retireSupersededSubagentRun(runId, entry); + mutated = true; + continue; + } + if ( + subagentRuns.get(runId) !== entry || + entry.endedReason !== SUBAGENT_ENDED_REASON_KILLED || + entry.killReconciliation !== killReconciliation + ) { + continue; + } + const taskResolutionAfterCompletion = findSubagentTaskForRun(entry); + const taskAfterCompletion = taskResolutionAfterCompletion.task; + const stableCancellationWonDuringCompletion = + taskAfterCompletion?.status === "cancelled" && + !isProvisionalSubagentKillTask(taskAfterCompletion) && + completionEndedAt >= killedAt; + if ( + !stableCancellationWonDuringCompletion && + taskResolutionAfterCompletion.lookup !== "unavailable" + ) { + // The attempted completion did not commit. Keep both durable + // sources unless a newer stable cancellation won during capture. + continue; + } + } + // Completion capture yields. Revalidate both owners before promoting a + // provisional marker into a sticky operator cancellation. + if ( + subagentRuns.get(runId) !== entry || + entry.endedReason !== SUBAGENT_ENDED_REASON_KILLED || + entry.killReconciliation !== killReconciliation + ) { + continue; + } + const taskResolutionBefore = findSubagentTaskForRun(entry); + const taskBefore = taskResolutionBefore.task; + const stableTaskCancellationAfterReconciliation = + taskBefore?.status === "cancelled" && !isProvisionalSubagentKillTask(taskBefore); + const taskNeedsStabilization = + taskResolutionBefore.lookup === "unavailable" || + (taskBefore !== undefined && + (taskBefore.status === "queued" || + taskBefore.status === "running" || + isProvisionalSubagentKillTask(taskBefore))); + if (taskNeedsStabilization) { + const observedError = + entry.outcome?.status === "error" ? entry.outcome.error?.trim() : undefined; + try { + // The live callback may be lost across restart. Make the provisional + // task state stable before its last reconciliation record is deleted. + const finalizedTasks = finalizeTaskRunByRunId({ + runId: taskBefore?.runId ?? entry.taskRunId ?? runId, + runtime: "subagent", + sessionKey: taskBefore?.childSessionKey ?? entry.childSessionKey, + status: "cancelled", + endedAt: killedAt, + lastEventAt: killedAt, + error: + observedError && observedError !== SUBAGENT_KILL_TASK_ERROR + ? observedError + : "Subagent run cancellation finalized.", + suppressDelivery: true, + }); + if (finalizedTasks.length === 0) { + const taskAfterResolution = findSubagentTaskForRun(entry); + const taskAfter = taskAfterResolution.task; + if ( + taskAfterResolution.lookup === "available" && + taskAfter !== undefined && + (taskAfter.status === "queued" || + taskAfter.status === "running" || + isProvisionalSubagentKillTask(taskAfter)) + ) { + log.warn("killed task was not stabilized during sweep", { + runId, + childSessionKey: entry.childSessionKey, + }); + continue; + } + if (taskAfterResolution.lookup === "unavailable") { + // Legacy custom runtimes cannot distinguish missing from + // opaque task state. After the bounded window and one finalizer + // attempt, do not leak the registry/session tombstone forever. + log.warn("retiring killed tombstone after opaque task finalization", { + runId, + childSessionKey: entry.childSessionKey, + }); + } + } + } catch (error) { + log.warn("failed to finalize provisional killed task during sweep", { + error, + runId, + childSessionKey: entry.childSessionKey, + }); + continue; + } + } + if (findNextSubagentRunCreatedAt(entry) !== undefined) { + // A newer generation owns this session key. Retire only the old run; + // session-scoped hooks or context cleanup would tear down the live owner. + await retireSupersededSubagentRun(runId, entry); + mutated = true; + continue; + } + // Re-enter the normal cleanup owner only after cancellation is canonical. + // It publishes the final failure once, then applies keep/delete semantics. + entry.suppressCompletionDelivery = + killReconciliation.suppressTaskDelivery === true || + hasStableTaskCancellation || + stableTaskCancellationAfterReconciliation + ? true + : undefined; + entry.suppressAnnounceReason = undefined; + entry.killReconciliation = undefined; + entry.cleanupHandled = false; + entry.cleanupCompletedAt = undefined; + mutated = true; + startSubagentAnnounceCleanupFlow(runId, entry); + continue; + } if (!entry.archiveAtMs && entry.cleanup === "keep" && entry.spawnMode !== "session") { continue; } @@ -1196,16 +1537,10 @@ function ensureListener() { const subagentRunManager = createSubagentRunManager({ runs: subagentRuns, resumedRuns, - endedHookInFlightRunIds, persist: persistSubagentRuns, persistOrThrow: persistSubagentRunsOrThrow, callGateway: (request) => subagentRegistryDeps.callGateway(request), getRuntimeConfig: () => subagentRegistryDeps.getRuntimeConfig(), - ensureRuntimePluginsLoaded: (args: { - config: OpenClawConfig; - workspaceDir?: string; - allowGatewaySubagentBinding?: boolean; - }) => ensureSubagentRegistryPluginRuntimeLoaded(args), ensureListener, startSweeper, stopSweeper, @@ -1219,6 +1554,7 @@ const subagentRunManager = createSubagentRunManager({ notifyContextEngineSubagentEnded, completeCleanupBookkeeping, completeSubagentRun, + resolveSubagentTask: findSubagentTaskForRun, }); configureSubagentRegistrySteerRuntime({ @@ -1387,6 +1723,7 @@ export function markSubagentRunTerminated(params: { runId?: string; childSessionKey?: string; reason?: string; + suppressTaskDelivery?: boolean; }): number { return subagentRunManager.markSubagentRunTerminated(params); } @@ -1526,7 +1863,7 @@ export function getLatestSubagentRunByChildSessionKey( if (entry.childSessionKey !== key) { continue; } - if (!latest || entry.createdAt > latest.createdAt) { + if (!latest || compareSubagentRunGeneration(entry, latest) > 0) { latest = entry; } } diff --git a/src/agents/subagent-registry.types.ts b/src/agents/subagent-registry.types.ts index 317cc1832b9..6de31683baa 100644 --- a/src/agents/subagent-registry.types.ts +++ b/src/agents/subagent-registry.types.ts @@ -84,8 +84,19 @@ export type SubagentCompletionDeliveryState = { | "waiting_for_requester_turn"; }; +export type SubagentKillReconciliationState = { + /** Actual cancellation time; a yielded run may have an older execution end. */ + killedAt: number; + /** Requester aborts must not re-inject a delayed completion after queues are cleared. */ + suppressTaskDelivery?: boolean; + /** Durable ownership boundary even after the newer registry row is released. */ + supersededAt?: number; +}; + export type SubagentRunRecord = { runId: string; + /** Detached task owner; steer/restart changes runId but continues the same task. */ + taskRunId?: string; childSessionKey: string; controllerSessionKey?: string; requesterSessionKey: string; @@ -100,6 +111,8 @@ export type SubagentRunRecord = { workspaceDir?: string; runTimeoutSeconds?: number; spawnMode?: SpawnSubagentMode; + /** Monotonic ownership generation within one child session. */ + generation?: number; createdAt: number; startedAt?: number; sessionStartedAt?: number; @@ -110,6 +123,10 @@ export type SubagentRunRecord = { cleanupCompletedAt?: number; cleanupHandled?: boolean; suppressAnnounceReason?: "steer-restart" | "killed"; + /** Present only while a current-version killed run awaits bounded reconciliation. */ + killReconciliation?: SubagentKillReconciliationState; + /** Durable requester-stop policy until silent completion cleanup finishes. */ + suppressCompletionDelivery?: boolean; expectsCompletionMessage?: boolean; endedReason?: SubagentLifecycleEndedReason; pauseReason?: "sessions_yield"; @@ -120,6 +137,8 @@ export type SubagentRunRecord = { endedHookEmittedAt?: number; /** Set after cleanupBrowserSessionsForLifecycleEnd has been dispatched once. */ browserCleanupDispatchedAt?: number; + /** Set immediately before irreversible sessions.delete cleanup is dispatched. */ + deleteCleanupDispatchedAt?: number; /** Durable outbox marker for parent/external completion delivery. */ delivery?: SubagentCompletionDeliveryState; attachmentsDir?: string; diff --git a/src/agents/subagent-run-generation.ts b/src/agents/subagent-run-generation.ts new file mode 100644 index 00000000000..8490161000f --- /dev/null +++ b/src/agents/subagent-run-generation.ts @@ -0,0 +1,45 @@ +type ComparableSubagentRun = { + runId: string; + createdAt: number; + generation?: number; +}; + +type GenerationalSubagentRun = ComparableSubagentRun & { + childSessionKey?: string; +}; + +function normalizeGeneration(entry: ComparableSubagentRun): number { + return typeof entry.generation === "number" && Number.isFinite(entry.generation) + ? entry.generation + : 0; +} + +/** Orders runs that share a child session, including legacy rows without a generation. */ +export function compareSubagentRunGeneration( + left: ComparableSubagentRun, + right: ComparableSubagentRun, +): number { + const generationDelta = normalizeGeneration(left) - normalizeGeneration(right); + if (generationDelta !== 0) { + return generationDelta; + } + const createdAtDelta = left.createdAt - right.createdAt; + if (createdAtDelta !== 0) { + return createdAtDelta; + } + return left.runId.localeCompare(right.runId); +} + +/** Allocates a durable monotonic generation within one child session. */ +export function nextSubagentRunGeneration( + runs: Iterable, + childSessionKey: string, +): number { + let generation = 0; + for (const entry of runs) { + if (entry.childSessionKey === childSessionKey) { + generation = Math.max(generation, normalizeGeneration(entry)); + } + } + return generation + 1; +} diff --git a/src/agents/subagent-run-timeout.test.ts b/src/agents/subagent-run-timeout.test.ts index 410a6e979b2..52e8d931847 100644 --- a/src/agents/subagent-run-timeout.test.ts +++ b/src/agents/subagent-run-timeout.test.ts @@ -5,6 +5,7 @@ import { MAX_TIMER_TIMEOUT_MS } from "../shared/number-coercion.js"; import { resolveSubagentRunDeadlineMs, resolveSubagentRunDurationMs, + resolveSubagentRunEffectiveEndedAt, resolveSubagentRunTimerDelayMs, } from "./subagent-run-timeout.js"; @@ -30,6 +31,15 @@ describe("subagent run timeout helpers", () => { expect(resolveSubagentRunDurationMs(thirtyDaysSeconds)).toBeGreaterThan(MAX_TIMER_TIMEOUT_MS); }); + it("clamps delayed terminal observations to the explicit deadline", () => { + expect( + resolveSubagentRunEffectiveEndedAt( + { createdAt: 1_000, startedAt: 2_000, runTimeoutSeconds: 3 }, + 6_000, + ), + ).toBe(5_000); + }); + it("ignores invalid timeout seconds and invalid start timestamps", () => { expect(resolveSubagentRunDurationMs(Number.NaN)).toBeUndefined(); expect(resolveSubagentRunDurationMs(0)).toBeUndefined(); diff --git a/src/agents/subagent-run-timeout.ts b/src/agents/subagent-run-timeout.ts index 4ea982b8376..5bc593903ff 100644 --- a/src/agents/subagent-run-timeout.ts +++ b/src/agents/subagent-run-timeout.ts @@ -51,3 +51,13 @@ export function resolveSubagentRunDeadlineMs( ? deadlineMs : undefined; } + +/** Clamp a reported terminal time to the run's explicit timeout deadline. */ +export function resolveSubagentRunEffectiveEndedAt( + entry: Pick, + endedAt: number, + observedStartedAt?: number, +): number { + const deadlineMs = resolveSubagentRunDeadlineMs(entry, observedStartedAt); + return deadlineMs !== undefined && endedAt > deadlineMs ? deadlineMs : endedAt; +} diff --git a/src/auto-reply/reply/abort.test.ts b/src/auto-reply/reply/abort.test.ts index c26422bc0b0..40c6859e062 100644 --- a/src/auto-reply/reply/abort.test.ts +++ b/src/auto-reply/reply/abort.test.ts @@ -681,6 +681,7 @@ describe("abort detection", () => { childSessionKey: childKey, reason: "killed", runId: "slow-child-run", + suppressTaskDelivery: true, }); expect(getFollowupQueueDepth(sessionKey)).toBe(0); expectSessionLaneCleared(sessionKey); @@ -1205,6 +1206,43 @@ describe("abort detection", () => { expectSessionLaneCleared(childKey); }); + it("continues stopping siblings when one termination persistence write fails", () => { + subagentRegistryMocks.markSubagentRunTerminated.mockClear(); + const sessionKey = "telegram:persistence-failure-parent"; + const firstChildKey = "agent:main:subagent:persistence-failure-first"; + const secondChildKey = "agent:main:subagent:persistence-failure-second"; + const run = (runId: string, childSessionKey: string): SubagentRunRecord => ({ + runId, + childSessionKey, + requesterSessionKey: sessionKey, + requesterDisplayKey: sessionKey, + task: "stop despite persistence failure", + cleanup: "keep", + createdAt: Date.now(), + }); + subagentRegistryMocks.listSubagentRunsForRequester + .mockReturnValueOnce([ + run("run-persistence-failure-first", firstChildKey), + run("run-persistence-failure-second", secondChildKey), + ]) + .mockReturnValue([]); + subagentRegistryMocks.markSubagentRunTerminated + .mockImplementationOnce(() => { + throw new Error("sqlite busy"); + }) + .mockReturnValue(1); + + expect( + stopSubagentsForRequester({ + cfg: {} as OpenClawConfig, + requesterSessionKey: sessionKey, + }), + ).toEqual({ stopped: 2 }); + expect(subagentRegistryMocks.markSubagentRunTerminated).toHaveBeenCalledTimes(2); + expectSessionLaneCleared(firstChildKey); + expectSessionLaneCleared(secondChildKey); + }); + it("cascade stop kills depth-2 children when stopping depth-1 agent", async () => { const sessionKey = "telegram:parent"; const depth1Key = "agent:main:subagent:child-1"; @@ -1261,6 +1299,43 @@ describe("abort detection", () => { expectSessionLaneCleared(depth2Key); }); + it("stops a subagent that is paused after yielding", () => { + subagentRegistryMocks.listSubagentRunsForRequester.mockClear(); + subagentRegistryMocks.markSubagentRunTerminated.mockClear(); + const sessionKey = "telegram:yield-parent"; + const childKey = "agent:main:subagent:yield-child"; + const now = Date.now(); + subagentRegistryMocks.listSubagentRunsForRequester + .mockReturnValueOnce([ + { + runId: "run-yield-child", + childSessionKey: childKey, + requesterSessionKey: sessionKey, + requesterDisplayKey: sessionKey, + task: "paused worker", + cleanup: "keep", + createdAt: now - 1_000, + endedAt: now - 500, + pauseReason: "sessions_yield", + }, + ]) + .mockReturnValueOnce([]); + + const result = stopSubagentsForRequester({ + cfg: {} as OpenClawConfig, + requesterSessionKey: sessionKey, + }); + + expect(result).toEqual({ stopped: 1 }); + expectSessionLaneCleared(childKey); + expect(subagentRegistryMocks.markSubagentRunTerminated).toHaveBeenCalledWith({ + runId: "run-yield-child", + childSessionKey: childKey, + reason: "killed", + suppressTaskDelivery: true, + }); + }); + it("cascade stop traverses ended depth-1 parents to stop active depth-2 children", async () => { subagentRegistryMocks.listSubagentRunsForRequester.mockClear(); subagentRegistryMocks.markSubagentRunTerminated.mockClear(); diff --git a/src/auto-reply/reply/abort.ts b/src/auto-reply/reply/abort.ts index 717e74f5cfc..0f7b46d6a73 100644 --- a/src/auto-reply/reply/abort.ts +++ b/src/auto-reply/reply/abort.ts @@ -218,6 +218,21 @@ function normalizeRequesterSessionKey( return resolveInternalSessionKey({ key: cleaned, alias, mainKey }); } +function markSubagentRunTerminatedBestEffort( + params: Parameters[0], +): number { + try { + return abortDeps.markSubagentRunTerminated(params); + } catch (error) { + // The runtime abort already happened. Keep stopping siblings and descendants; + // durable reconciliation can retry the rolled-back registry transition later. + logVerbose( + `abort: failed to persist killed subagent ${params.runId ?? params.childSessionKey ?? "unknown"}: ${formatErrorMessage(error)}`, + ); + return 0; + } +} + export function stopSubagentsForRequester(params: { cfg: OpenClawConfig; requesterSessionKey?: string; @@ -266,7 +281,7 @@ export function stopSubagentsForRequester(params: { } seenChildKeys.add(childKey); - if (!run.endedAt) { + if (!run.endedAt || run.pauseReason === "sessions_yield") { const cleared = clearSessionQueues([childKey]); const parsed = parseAgentSessionKey(childKey); const storePath = resolveStorePath(params.cfg.session?.store, { agentId: parsed?.agentId }); @@ -282,10 +297,11 @@ export function stopSubagentsForRequester(params: { const abortRejected = abortOutcome.active && !abortOutcome.aborted; const markedTerminated = abortRejected ? false - : abortDeps.markSubagentRunTerminated({ + : markSubagentRunTerminatedBestEffort({ runId: run.runId, childSessionKey: childKey, reason: "killed", + suppressTaskDelivery: true, }) > 0; if ( diff --git a/src/tasks/detached-task-runtime-contract.ts b/src/tasks/detached-task-runtime-contract.ts index 5dfc0e1c7a1..2d325e495d9 100644 --- a/src/tasks/detached-task-runtime-contract.ts +++ b/src/tasks/detached-task-runtime-contract.ts @@ -11,6 +11,10 @@ import type { TaskTerminalOutcome, } from "./task-registry.types.js"; +// A killed subagent can still report a completion that raced the kill marker. +// Task cancellation replaces this marker once the operator request is accepted. +export const SUBAGENT_KILL_TASK_ERROR = "Subagent run killed."; + export type DetachedTaskCreateParams = { runtime: TaskRuntime; taskKind?: string; @@ -66,6 +70,7 @@ export type DetachedTaskCompleteParams = { progressSummary?: string | null; terminalSummary?: string | null; terminalOutcome?: TaskTerminalOutcome | null; + suppressDelivery?: boolean; }; export type DetachedTaskFailParams = { @@ -78,6 +83,7 @@ export type DetachedTaskFailParams = { error?: string; progressSummary?: string | null; terminalSummary?: string | null; + suppressDelivery?: boolean; }; export type DetachedTaskFinalizeParams = { @@ -91,8 +97,14 @@ export type DetachedTaskFinalizeParams = { progressSummary?: string | null; terminalSummary?: string | null; terminalOutcome?: TaskTerminalOutcome | null; + suppressDelivery?: boolean; }; +export type DetachedTaskTerminalState = Omit< + DetachedTaskFinalizeParams, + "runId" | "runtime" | "sessionKey" +>; + export type DetachedTaskDeliveryStatusParams = { runId: string; runtime?: TaskRuntime; @@ -125,6 +137,19 @@ export type DetachedTaskRecoveryAttemptResult = { recovered: boolean; }; +export type DetachedTaskFindParams = { + runId: string; + runtime: TaskRuntime; + sessionKey: string; + createdAtOrAfter: number; + createdBefore?: number; + allowSessionFallback?: boolean; +}; + +export type DetachedTaskFindResult = + | { lookup: "available"; task?: TaskRecord } + | { lookup: "unavailable"; task?: undefined }; + export type DetachedTaskLifecycleRuntime = { createQueuedTaskRun: (params: DetachedTaskCreateParams) => TaskRecord | null; createRunningTaskRun: (params: DetachedRunningTaskCreateParams) => TaskRecord | null; @@ -134,6 +159,11 @@ export type DetachedTaskLifecycleRuntime = { completeTaskRunByRunId: (params: DetachedTaskCompleteParams) => TaskRecord[]; failTaskRunByRunId: (params: DetachedTaskFailParams) => TaskRecord[]; setDetachedTaskDeliveryStatusByRunId: (params: DetachedTaskDeliveryStatusParams) => TaskRecord[]; + /** + * Resolve the task owned by one run generation. Custom runtimes should + * implement this when their records are not mirrored into core task state. + */ + findTaskRun?: (params: DetachedTaskFindParams) => TaskRecord | undefined; /** * Return `found: false` when this runtime does not own the task so core can * fall back to the legacy detached-task cancel path. diff --git a/src/tasks/detached-task-runtime.test.ts b/src/tasks/detached-task-runtime.test.ts index 892e11379e9..8ece1e46881 100644 --- a/src/tasks/detached-task-runtime.test.ts +++ b/src/tasks/detached-task-runtime.test.ts @@ -6,6 +6,7 @@ import { createQueuedTaskRun, createRunningTaskRun, failTaskRunByRunId, + findDetachedTaskRun, finalizeTaskRunByRunId, getDetachedTaskLifecycleRuntime, getDetachedTaskLifecycleRuntimeRegistration, @@ -19,9 +20,12 @@ import { } from "./detached-task-runtime.js"; import type { TaskRecord } from "./task-registry.types.js"; -const { mockLogWarn } = vi.hoisted(() => ({ - mockLogWarn: vi.fn(), -})); +const { mockFindTaskByRunIdForStatus, mockListTasksForSessionKeyForStatus, mockLogWarn } = + vi.hoisted(() => ({ + mockFindTaskByRunIdForStatus: vi.fn(), + mockListTasksForSessionKeyForStatus: vi.fn(() => [] as TaskRecord[]), + mockLogWarn: vi.fn(), + })); vi.mock("../logging/subsystem.js", () => ({ createSubsystemLogger: () => ({ subsystem: "tasks/detached-runtime", @@ -37,6 +41,11 @@ vi.mock("../logging/subsystem.js", () => ({ }), })); +vi.mock("./task-status-access.js", () => ({ + findTaskByRunIdForStatus: mockFindTaskByRunIdForStatus, + listTasksForSessionKeyForStatus: mockListTasksForSessionKeyForStatus, +})); + function createFakeTaskRecord(overrides?: Partial): TaskRecord { return { taskId: "task-fake", @@ -77,9 +86,123 @@ function requireFirstCallArg( describe("detached-task-runtime", () => { afterEach(() => { resetDetachedTaskLifecycleRuntimeForTests(); + mockFindTaskByRunIdForStatus.mockReset(); + mockListTasksForSessionKeyForStatus.mockReset(); + mockListTasksForSessionKeyForStatus.mockReturnValue([]); mockLogWarn.mockClear(); }); + it("finds a replacement task within the requested session generation", () => { + const expected = createFakeTaskRecord({ + taskId: "task-expected", + runtime: "subagent", + runId: "run-shared", + childSessionKey: "agent:main:subagent:expected", + createdAt: 30, + }); + mockFindTaskByRunIdForStatus.mockReturnValue( + createFakeTaskRecord({ + taskId: "task-other-generation", + runtime: "subagent", + runId: "run-shared", + childSessionKey: "agent:main:subagent:other", + createdAt: 10, + }), + ); + mockListTasksForSessionKeyForStatus.mockReturnValue([ + createFakeTaskRecord({ + taskId: "task-next-generation", + runtime: "subagent", + runId: "run-next", + childSessionKey: "agent:main:subagent:expected", + createdAt: 40, + }), + expected, + ]); + + expect( + findDetachedTaskRun({ + runId: "run-shared", + runtime: "subagent", + sessionKey: "agent:main:subagent:expected", + createdAtOrAfter: 15, + createdBefore: 40, + allowSessionFallback: true, + }), + ).toEqual({ lookup: "available", task: expected }); + }); + + it("uses an exact task owner even when its timestamps predate the current run", () => { + const expected = createFakeTaskRecord({ + taskId: "task-original-owner", + runtime: "subagent", + runId: "run-original-owner", + childSessionKey: "agent:main:subagent:steered", + createdAt: 10, + }); + mockFindTaskByRunIdForStatus.mockReturnValue(expected); + + expect( + findDetachedTaskRun({ + runId: "run-original-owner", + runtime: "subagent", + sessionKey: "agent:main:subagent:steered", + createdAtOrAfter: 20, + createdBefore: 20, + }), + ).toEqual({ lookup: "available", task: expected }); + expect(mockListTasksForSessionKeyForStatus).not.toHaveBeenCalled(); + }); + + it("does not adopt a session task for an unchanged run ID", () => { + mockListTasksForSessionKeyForStatus.mockReturnValue([ + createFakeTaskRecord({ + taskId: "task-unrelated", + runtime: "subagent", + runId: "run-unrelated", + childSessionKey: "agent:main:subagent:expected", + createdAt: 30, + }), + ]); + + expect( + findDetachedTaskRun({ + runId: "run-current", + runtime: "subagent", + sessionKey: "agent:main:subagent:expected", + createdAtOrAfter: 15, + createdBefore: 40, + }), + ).toEqual({ lookup: "available", task: undefined }); + expect(mockListTasksForSessionKeyForStatus).not.toHaveBeenCalled(); + }); + + it("contains failures from custom task lookup hooks", () => { + setDetachedTaskLifecycleRuntime({ + ...getDetachedTaskLifecycleRuntime(), + findTaskRun: () => { + throw new Error("lookup unavailable"); + }, + }); + + expect( + findDetachedTaskRun({ + runId: "run-lookup-failure", + runtime: "subagent", + sessionKey: "agent:main:subagent:lookup-failure", + createdAtOrAfter: 1, + }), + ).toEqual({ lookup: "unavailable" }); + expect(mockLogWarn).toHaveBeenCalledWith( + "Detached task lookup failed", + expect.objectContaining({ + runtime: "subagent", + runId: "run-lookup-failure", + error: expect.any(Error), + }), + ); + }); + it("dispatches lifecycle operations through the installed runtime", async () => { const defaultRuntime = getDetachedTaskLifecycleRuntime(); const queuedTask = createFakeTaskRecord({ @@ -102,6 +225,7 @@ describe("detached-task-runtime", () => { completeTaskRunByRunId: vi.fn(() => updatedTasks), failTaskRunByRunId: vi.fn(() => updatedTasks), setDetachedTaskDeliveryStatusByRunId: vi.fn(() => updatedTasks), + findTaskRun: vi.fn(() => runningTask), cancelDetachedTaskRunById: vi.fn(async () => ({ found: true, cancelled: true, @@ -141,6 +265,14 @@ describe("detached-task-runtime", () => { runId: "run-running", deliveryStatus: "delivered", }); + expect( + findDetachedTaskRun({ + runId: "run-running", + runtime: "cli", + sessionKey: "agent:main:main", + createdAtOrAfter: 1, + }), + ).toEqual({ lookup: "available", task: runningTask }); await cancelDetachedTaskRunById({ cfg: {} as never, taskId: runningTask.taskId, @@ -182,6 +314,12 @@ describe("detached-task-runtime", () => { .calls[0]?.[0]; expect(deliveryArgs?.runId).toBe("run-running"); expect(deliveryArgs?.deliveryStatus).toBe("delivered"); + expect(fakeRuntime.findTaskRun).toHaveBeenCalledWith({ + runId: "run-running", + runtime: "cli", + sessionKey: "agent:main:main", + createdAtOrAfter: 1, + }); expect(fakeRuntime.cancelDetachedTaskRunById).toHaveBeenCalledWith({ cfg: {} as never, taskId: runningTask.taskId, @@ -232,6 +370,21 @@ describe("detached-task-runtime", () => { expect(failArgs.endedAt).toBe(20); }); + it("reports unavailable lookup for an opaque legacy runtime", () => { + const legacyRuntime = { ...getDetachedTaskLifecycleRuntime() }; + delete legacyRuntime.findTaskRun; + setDetachedTaskLifecycleRuntime(legacyRuntime); + + expect( + findDetachedTaskRun({ + runId: "run-not-mirrored", + runtime: "subagent", + sessionKey: "agent:main:subagent:not-mirrored", + createdAtOrAfter: 1, + }), + ).toEqual({ lookup: "unavailable" }); + }); + describe("tryRecoverTaskBeforeMarkLost", () => { it("returns recovered when hook returns recovered true", async () => { const task = createFakeTaskRecord({ taskId: "task-recover", runtime: "subagent" }); diff --git a/src/tasks/detached-task-runtime.ts b/src/tasks/detached-task-runtime.ts index b95d5ff6f3b..0c609d153ba 100644 --- a/src/tasks/detached-task-runtime.ts +++ b/src/tasks/detached-task-runtime.ts @@ -3,6 +3,8 @@ import { createSubsystemLogger } from "../logging/subsystem.js"; import type { DetachedTaskRecoveryAttemptParams, DetachedTaskRecoveryAttemptResult, + DetachedTaskFindParams, + DetachedTaskFindResult, DetachedTaskFinalizeParams, DetachedTaskLifecycleRuntime, DetachedTaskLifecycleRuntimeRegistration, @@ -25,10 +27,37 @@ import { startTaskRunByRunId as startTaskRunByRunIdFromExecutor, } from "./task-executor.js"; import type { TaskRecord } from "./task-registry.types.js"; +import { findTaskByRunIdForStatus, listTasksForSessionKeyForStatus } from "./task-status-access.js"; const log = createSubsystemLogger("tasks/detached-runtime"); const DETACHED_TASK_RECOVERY_WARN_MS = 5_000; +function taskMatchesFindScope(task: TaskRecord, params: DetachedTaskFindParams): boolean { + return ( + task.runtime === params.runtime && + task.childSessionKey === params.sessionKey && + task.createdAt >= params.createdAtOrAfter && + (params.createdBefore === undefined || task.createdAt < params.createdBefore) + ); +} + +function taskMatchesFindIdentity(task: TaskRecord, params: DetachedTaskFindParams): boolean { + return task.runtime === params.runtime && task.childSessionKey === params.sessionKey; +} + +function findCoreTaskRun(params: DetachedTaskFindParams): TaskRecord | undefined { + const direct = findTaskByRunIdForStatus(params.runId); + if (direct && taskMatchesFindIdentity(direct, params)) { + return direct; + } + if (params.allowSessionFallback !== true) { + return undefined; + } + return listTasksForSessionKeyForStatus(params.sessionKey).find((task) => + taskMatchesFindScope(task, params), + ); +} + export type { DetachedTaskLifecycleRuntime, DetachedTaskLifecycleRuntimeRegistration }; // Default runtime keeps detached task APIs usable before plugins install custom lifecycle hooks. @@ -41,6 +70,7 @@ const DEFAULT_DETACHED_TASK_LIFECYCLE_RUNTIME: DetachedTaskLifecycleRuntime = { completeTaskRunByRunId: completeTaskRunByRunIdFromExecutor, failTaskRunByRunId: failTaskRunByRunIdFromExecutor, setDetachedTaskDeliveryStatusByRunId: setDetachedTaskDeliveryStatusByRunIdFromExecutor, + findTaskRun: findCoreTaskRun, cancelDetachedTaskRunById: cancelDetachedTaskRunByIdInCore, }; @@ -125,6 +155,26 @@ export function setDetachedTaskDeliveryStatusByRunId( return getDetachedTaskLifecycleRuntime().setDetachedTaskDeliveryStatusByRunId(...args); } +export function findDetachedTaskRun(params: DetachedTaskFindParams): DetachedTaskFindResult { + const runtime = getDetachedTaskLifecycleRuntime(); + if (runtime.findTaskRun) { + try { + return { lookup: "available", task: runtime.findTaskRun(params) }; + } catch (error) { + log.warn("Detached task lookup failed", { + runtime: params.runtime, + runId: params.runId, + error, + }); + return { lookup: "unavailable" }; + } + } + const coreTask = findCoreTaskRun(params); + // Older custom runtimes may mirror records into core. When they do not, an + // empty fallback cannot prove that the runtime-owned task is absent. + return coreTask ? { lookup: "available", task: coreTask } : { lookup: "unavailable" }; +} + export function cancelDetachedTaskRunById( ...args: Parameters ): ReturnType { diff --git a/src/tasks/task-cancellation-state.ts b/src/tasks/task-cancellation-state.ts new file mode 100644 index 00000000000..7dbd31a269b --- /dev/null +++ b/src/tasks/task-cancellation-state.ts @@ -0,0 +1,21 @@ +// Defines task states that still participate in flow cancellation. +import { SUBAGENT_KILL_TASK_ERROR } from "./detached-task-runtime-contract.js"; +import type { TaskRecord } from "./task-registry.types.js"; + +export function isProvisionalSubagentKillTask( + task: Pick, +): boolean { + return ( + task.runtime === "subagent" && + task.status === "cancelled" && + task.error === SUBAGENT_KILL_TASK_ERROR + ); +} + +export function isTaskFlowCancellationPending( + task: Pick, +): boolean { + return ( + task.status === "queued" || task.status === "running" || isProvisionalSubagentKillTask(task) + ); +} diff --git a/src/tasks/task-executor-policy.test.ts b/src/tasks/task-executor-policy.test.ts index f3567f7632c..5a18ce22210 100644 --- a/src/tasks/task-executor-policy.test.ts +++ b/src/tasks/task-executor-policy.test.ts @@ -1,5 +1,6 @@ // Verifies task executor delivery policy and terminal message formatting. import { describe, expect, it } from "vitest"; +import { SUBAGENT_KILL_TASK_ERROR } from "./detached-task-runtime-contract.js"; import { formatTaskBlockedFollowupMessage, formatTaskStateChangeMessage, @@ -71,6 +72,11 @@ describe("task-executor-policy", () => { expect(formatTaskBlockedFollowupMessage(blockedTask)).toBe( "Task needs follow-up: ACP import (run run-1234). Needs login.", ); + expect( + formatTaskTerminalMessage( + createTask({ runtime: "subagent", status: "cancelled", runId: "run-34567890" }), + ), + ).toBe("Background task cancellation requested: Subagent task (run run-3456)."); expect(formatTaskStateChangeMessage(blockedTask, progressEvent)).toBe( "Background task update: ACP import. No output for 60s.", ); @@ -152,6 +158,26 @@ describe("task-executor-policy", () => { }), ), ).toBe(false); + expect( + shouldAutoDeliverTaskTerminalUpdate( + createTask({ + runtime: "subagent", + status: "cancelled", + error: SUBAGENT_KILL_TASK_ERROR, + deliveryStatus: "pending", + }), + ), + ).toBe(false); + expect( + shouldAutoDeliverTaskTerminalUpdate( + createTask({ + runtime: "subagent", + status: "cancelled", + error: "Cancelled by operator.", + deliveryStatus: "pending", + }), + ), + ).toBe(true); expect( shouldAutoDeliverTaskStateChange( createTask({ @@ -179,6 +205,17 @@ describe("task-executor-policy", () => { preferredTaskId: "task-2", }), ).toBe(true); + expect( + shouldSuppressDuplicateTerminalDelivery({ + task: createTask({ + runtime: "subagent", + status: "cancelled", + runId: "run-duplicate", + }), + preferredTaskId: "task-1", + peerDeliveryCovered: true, + }), + ).toBe(true); expect( shouldSuppressDuplicateTerminalDelivery({ task: createTask({ @@ -197,6 +234,16 @@ describe("task-executor-policy", () => { preferredTaskId: undefined, }), ).toBe(false); + expect( + shouldSuppressDuplicateTerminalDelivery({ + task: createTask({ + runtime: "subagent", + status: "cancelled", + runId: "run-duplicate", + }), + preferredTaskId: "task-2", + }), + ).toBe(true); expect( shouldUseParentReviewTaskTerminalMessage( createTask({ diff --git a/src/tasks/task-executor-policy.ts b/src/tasks/task-executor-policy.ts index 33d0d6564b5..13c440746ad 100644 --- a/src/tasks/task-executor-policy.ts +++ b/src/tasks/task-executor-policy.ts @@ -1,4 +1,5 @@ // Decides task executor delivery, terminal update, and follow-up message policy. +import { SUBAGENT_KILL_TASK_ERROR } from "./detached-task-runtime-contract.js"; import type { TaskEventRecord, TaskRecord, TaskStatus } from "./task-registry.types.js"; import { formatTaskStatusTitleText, sanitizeTaskStatusText } from "./task-status.js"; @@ -62,6 +63,11 @@ export function formatTaskTerminalMessage( return `Background task lost: ${title}${runLabel}. ${error || fallbackSummary || "Backing session disappeared."}`; } if (task.status === "cancelled") { + if (task.runtime === "subagent") { + // A final reply can win the kill race and reconcile this row to success. + // Report the operator action without claiming an irreversible outcome. + return `Background task cancellation requested: ${title}${runLabel}.`; + } return `Background task cancelled: ${title}${runLabel}.`; } const error = sanitizeTaskStatusText(task.error, { errorContext: true }); @@ -114,6 +120,15 @@ export function shouldAutoDeliverTaskTerminalUpdate(task: TaskRecord): boolean { return false; } if (task.runtime === "subagent" && task.status !== "cancelled") { + // Subagent lifecycle owns provider-result publication. + return false; + } + if ( + task.runtime === "subagent" && + task.status === "cancelled" && + task.error === SUBAGENT_KILL_TASK_ERROR + ) { + // A direct kill is provisional until lifecycle reconciliation settles. return false; } if (!isTerminalTaskStatus(task.status)) { @@ -133,9 +148,19 @@ export function shouldAutoDeliverTaskStateChange(task: TaskRecord): boolean { export function shouldSuppressDuplicateTerminalDelivery(params: { task: TaskRecord; preferredTaskId?: string; + peerDeliveryCovered?: boolean; }): boolean { - if (params.task.runtime !== "acp" || !params.task.runId?.trim()) { + if (!params.task.runId?.trim()) { return false; } + const sharesRunDelivery = + params.task.runtime === "acp" || + (params.task.runtime === "subagent" && params.task.status === "cancelled"); + if (!sharesRunDelivery) { + return false; + } + if (params.task.runtime === "subagent" && params.peerDeliveryCovered) { + return true; + } return Boolean(params.preferredTaskId && params.preferredTaskId !== params.task.taskId); } diff --git a/src/tasks/task-executor.test.ts b/src/tasks/task-executor.test.ts index e30ee65d8a8..edeea01631a 100644 --- a/src/tasks/task-executor.test.ts +++ b/src/tasks/task-executor.test.ts @@ -5,6 +5,7 @@ import { resetHeartbeatWakeStateForTests } from "../infra/heartbeat-wake.js"; import { resetSystemEventsForTest } from "../infra/system-events.js"; import { withStateDirEnv } from "../test-helpers/state-dir-env.js"; import { captureEnv } from "../test-utils/env.js"; +import { SUBAGENT_KILL_TASK_ERROR } from "./detached-task-runtime-contract.js"; import { getDetachedTaskLifecycleRuntime, resetDetachedTaskLifecycleRuntimeForTests, @@ -362,6 +363,58 @@ describe("task-executor", () => { }); }); + it("promotes a provisional kill in an already-cancelled one-task flow", async () => { + await withTaskExecutorStateDir(async () => { + const child = createRunningTaskRun({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:worker:subagent:mirrored-kill", + runId: "run-mirrored-provisional-kill", + task: "Stop mirrored child", + startedAt: 10, + deliveryStatus: "pending", + }); + const flowId = expectParentFlowId(child); + failTaskRunByRunId({ + runId: child.runId!, + runtime: "subagent", + sessionKey: child.childSessionKey, + status: "cancelled", + endedAt: 20, + error: SUBAGENT_KILL_TASK_ERROR, + }); + hoisted.killSubagentRunAdminMock.mockResolvedValueOnce({ + found: true, + killed: false, + runId: child.runId!, + sessionKey: child.childSessionKey!, + targetState: { + state: "terminal", + task: { status: "cancelled", endedAt: 20, error: SUBAGENT_KILL_TASK_ERROR }, + }, + }); + expect(getTaskFlowById(flowId)?.status).toBe("cancelled"); + + const cancelled = await cancelFlowById({ cfg: {} as never, flowId }); + completeTaskRunByRunId({ + runId: child.runId!, + runtime: "subagent", + sessionKey: child.childSessionKey, + endedAt: 30, + terminalSummary: "completed too late", + }); + + expect(cancelled).toMatchObject({ found: true, cancelled: true }); + expect(getTaskFlowById(flowId)?.status).toBe("cancelled"); + expect(getTaskById(child.taskId)).toMatchObject({ + status: "cancelled", + endedAt: 20, + error: "Cancelled by operator.", + }); + }); + }); + it("does not auto-create one-task flows for non-returning bookkeeping runs", async () => { await withTaskExecutorStateDir(async () => { const created = createRunningTaskRun({ @@ -421,6 +474,61 @@ describe("task-executor", () => { }); }); + it("promotes provisional subagent kills before cancelling a managed TaskFlow", async () => { + await withTaskExecutorStateDir(async () => { + const flow = createManagedTaskFlow({ + ownerKey: "agent:main:main", + controllerId: "tests/managed-flow", + goal: "Cancel a killed child", + }); + const created = runTaskInFlow({ + flowId: flow.flowId, + runtime: "subagent", + childSessionKey: "agent:worker:subagent:flow-killed", + runId: "run-flow-provisional-kill", + task: "Stop the child", + status: "running", + startedAt: 10, + }); + const child = requireCreatedFlowTask(created); + failTaskRunByRunId({ + runId: child.runId!, + runtime: "subagent", + sessionKey: child.childSessionKey, + status: "cancelled", + endedAt: 20, + error: SUBAGENT_KILL_TASK_ERROR, + }); + hoisted.killSubagentRunAdminMock.mockResolvedValueOnce({ + found: true, + killed: false, + runId: child.runId!, + sessionKey: child.childSessionKey!, + targetState: { + state: "terminal", + task: { status: "cancelled", endedAt: 20, error: SUBAGENT_KILL_TASK_ERROR }, + }, + }); + + const cancelled = await cancelFlowById({ cfg: {} as never, flowId: flow.flowId }); + completeTaskRunByRunId({ + runId: child.runId!, + runtime: "subagent", + sessionKey: child.childSessionKey, + endedAt: 30, + terminalSummary: "completed too late", + }); + + expect(cancelled).toMatchObject({ found: true, cancelled: true }); + expect(getTaskFlowById(flow.flowId)?.status).toBe("cancelled"); + expect(getTaskById(child.taskId)).toMatchObject({ + status: "cancelled", + endedAt: 20, + error: "Cancelled by operator.", + }); + }); + }); + it("runs child tasks under managed TaskFlows", async () => { await withTaskExecutorStateDir(async () => { const flow = createManagedTaskFlow({ diff --git a/src/tasks/task-executor.ts b/src/tasks/task-executor.ts index 1757c23a1a9..4846cbe3ec4 100644 --- a/src/tasks/task-executor.ts +++ b/src/tasks/task-executor.ts @@ -19,6 +19,10 @@ import { recordTaskProgressByRunId, setTaskRunDeliveryStatusByRunId, } from "./runtime-internal.js"; +import { + isProvisionalSubagentKillTask, + isTaskFlowCancellationPending, +} from "./task-cancellation-state.js"; import { getTaskFlowByIdForOwner } from "./task-flow-owner-access.js"; import type { TaskFlowRecord } from "./task-flow-registry.types.js"; import { @@ -174,6 +178,7 @@ export function completeTaskRunByRunId(params: { progressSummary?: string | null; terminalSummary?: string | null; terminalOutcome?: TaskTerminalOutcome | null; + suppressDelivery?: boolean; }) { return finalizeTaskRunByRunId({ ...params, @@ -195,6 +200,7 @@ export function failTaskRunByRunId(params: { error?: string; progressSummary?: string | null; terminalSummary?: string | null; + suppressDelivery?: boolean; }) { return finalizeTaskRunByRunId({ ...params, @@ -228,10 +234,6 @@ type RunTaskInFlowResult = { task?: TaskRecord; }; -function isActiveTaskStatus(status: TaskStatus): boolean { - return status === "queued" || status === "running"; -} - function isTerminalFlowStatus(status: TaskFlowRecord["status"]): boolean { return ( status === "succeeded" || status === "failed" || status === "cancelled" || status === "lost" @@ -475,6 +477,33 @@ export async function cancelFlowById(params: { }; } if (isTerminalFlowStatus(flow.status)) { + const provisionalTasks = listTasksForFlowId(flow.flowId).filter(isProvisionalSubagentKillTask); + if (flow.status === "cancelled" && provisionalTasks.length > 0) { + for (const task of provisionalTasks) { + await cancelDetachedTaskRunById({ cfg: params.cfg, taskId: task.taskId }); + } + const tasks = listTasksForFlowId(flow.flowId); + if (tasks.some(isProvisionalSubagentKillTask)) { + return { + found: true, + cancelled: false, + reason: "One or more child tasks remain provisionally cancelled.", + flow: getTaskFlowById(flow.flowId) ?? flow, + tasks, + }; + } + const refreshedFlow = getTaskFlowById(flow.flowId) ?? flow; + return { + found: true, + cancelled: refreshedFlow.status === "cancelled", + reason: + refreshedFlow.status === "cancelled" + ? undefined + : `Flow is already ${refreshedFlow.status}.`, + flow: refreshedFlow, + tasks, + }; + } return { found: true, cancelled: false, @@ -494,7 +523,7 @@ export async function cancelFlowById(params: { }; } const linkedTasks = listTasksForFlowId(flow.flowId); - const activeTasks = linkedTasks.filter((task) => isActiveTaskStatus(task.status)); + const activeTasks = linkedTasks.filter(isTaskFlowCancellationPending); for (const task of activeTasks) { await cancelDetachedTaskRunById({ cfg: params.cfg, @@ -502,7 +531,7 @@ export async function cancelFlowById(params: { }); } const refreshedTasks = listTasksForFlowId(flow.flowId); - const remainingActive = refreshedTasks.filter((task) => isActiveTaskStatus(task.status)); + const remainingActive = refreshedTasks.filter(isTaskFlowCancellationPending); if (remainingActive.length > 0) { return { found: true, diff --git a/src/tasks/task-flow-registry.audit.test.ts b/src/tasks/task-flow-registry.audit.test.ts index f5dd86cd6cf..90a3aa21fd1 100644 --- a/src/tasks/task-flow-registry.audit.test.ts +++ b/src/tasks/task-flow-registry.audit.test.ts @@ -2,7 +2,11 @@ import { afterEach, describe, expect, it } from "vitest"; import { captureEnv } from "../test-utils/env.js"; import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; -import { createRunningTaskRun as createRunningTaskRunOrNull } from "./task-executor.js"; +import { SUBAGENT_KILL_TASK_ERROR } from "./detached-task-runtime-contract.js"; +import { + createRunningTaskRun as createRunningTaskRunOrNull, + finalizeTaskRunByRunId, +} from "./task-executor.js"; import { listTaskFlowAuditFindings, type TaskFlowAuditCode, @@ -10,6 +14,7 @@ import { } from "./task-flow-registry.audit.js"; import { createManagedTaskFlow as createManagedTaskFlowOrNull, + requestFlowCancel, resetTaskFlowRegistryForTests, setFlowWaiting, } from "./task-flow-registry.js"; @@ -242,4 +247,50 @@ describe("task-flow-registry audit", () => { expect(requireFinding(findings, "cancel_stuck", flow.flowId).flow?.flowId).toBe(flow.flowId); }); }); + + it("counts provisional subagent cancellation as active during audit", async () => { + await withTaskFlowAuditStateDir(async () => { + const now = Date.now(); + const flow = createManagedTaskFlow({ + ownerKey: "agent:main:main", + controllerId: "tests/task-flow-audit", + goal: "Cancel subagent work", + status: "running", + createdAt: now - 6 * 60_000, + updatedAt: now - 6 * 60_000, + }); + const runId = "run-provisional-cancel-audit"; + const task = createRunningTaskRun({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + parentFlowId: flow.flowId, + childSessionKey: "agent:main:subagent:provisional-cancel", + runId, + task: "Wait for kill reconciliation", + startedAt: now - 6 * 60_000, + lastEventAt: now - 6 * 60_000, + }); + expect(task.runId).toBe(runId); + requestFlowCancel({ + flowId: flow.flowId, + expectedRevision: flow.revision, + cancelRequestedAt: now - 6 * 60_000, + updatedAt: now - 6 * 60_000, + }); + finalizeTaskRunByRunId({ + runId, + runtime: "subagent", + status: "cancelled", + endedAt: now - 6 * 60_000, + error: SUBAGENT_KILL_TASK_ERROR, + }); + + expect( + listTaskFlowAuditFindings({ now }).find( + (finding) => finding.code === "cancel_stuck" && finding.flow?.flowId === flow.flowId, + ), + ).toBeUndefined(); + }); + }); }); diff --git a/src/tasks/task-flow-registry.audit.ts b/src/tasks/task-flow-registry.audit.ts index 0e9537380c6..67165be7a82 100644 --- a/src/tasks/task-flow-registry.audit.ts +++ b/src/tasks/task-flow-registry.audit.ts @@ -1,5 +1,6 @@ // Produces task-flow registry audit summaries for diagnostics and maintenance. import { listTasksForFlowId } from "./runtime-internal.js"; +import { isTaskFlowCancellationPending } from "./task-cancellation-state.js"; import { getTaskFlowRegistryRestoreFailure, listTaskFlowRecords } from "./task-flow-registry.js"; import type { TaskFlowRecord } from "./task-flow-registry.types.js"; import type { TaskRecord } from "./task-registry.types.js"; @@ -164,9 +165,7 @@ export function listTaskFlowAuditFindings( const referenceAt = getReferenceAt(flow); const ageMs = Math.max(0, now - referenceAt); const linkedTasks = getLinkedTasks(flow.flowId); - const activeTasks = linkedTasks.filter( - (task) => task.status === "queued" || task.status === "running", - ); + const activeTasks = linkedTasks.filter((task) => isTaskFlowCancellationPending(task)); if (flow.status === "running" && ageMs >= staleRunningMs) { findings.push( diff --git a/src/tasks/task-flow-registry.maintenance.test.ts b/src/tasks/task-flow-registry.maintenance.test.ts index 45d0011daf4..020d09a6d19 100644 --- a/src/tasks/task-flow-registry.maintenance.test.ts +++ b/src/tasks/task-flow-registry.maintenance.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { captureEnv } from "../test-utils/env.js"; import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; +import { SUBAGENT_KILL_TASK_ERROR } from "./detached-task-runtime-contract.js"; import { createRunningTaskRun as createRunningTaskRunOrNull } from "./task-executor.js"; import { createFlowRecord as createFlowRecordOrNull, @@ -18,6 +19,7 @@ import { } from "./task-flow-registry.maintenance.js"; import type { TaskFlowRecord } from "./task-flow-registry.types.js"; import { + finalizeTaskRunByRunId, resetTaskRegistryDeliveryRuntimeForTests, resetTaskRegistryForTests, } from "./task-registry.js"; @@ -229,6 +231,56 @@ describe("task-flow-registry maintenance", () => { }); }); + it("does not finalize cancel-requested flows while a child kill is provisional", async () => { + await withTaskFlowMaintenanceStateDir(async () => { + const flow = createManagedTaskFlow({ + ownerKey: "agent:main:main", + controllerId: "tests/task-flow-maintenance", + goal: "Wait for child kill reconciliation", + status: "running", + createdAt: 1, + updatedAt: 100, + }); + const child = createRunningTaskRun({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + parentFlowId: flow.flowId, + childSessionKey: "agent:main:subagent:provisional-kill", + runId: "run-provisional-kill", + task: "Finish while cancellation races", + startedAt: 100, + lastEventAt: 100, + }); + finalizeTaskRunByRunId({ + runId: child.runId!, + runtime: "subagent", + sessionKey: child.childSessionKey, + status: "cancelled", + endedAt: 110, + error: SUBAGENT_KILL_TASK_ERROR, + }); + const currentFlow = getTaskFlowById(flow.flowId); + if (!currentFlow) { + throw new Error("Expected provisional child flow to remain registered"); + } + const cancelResult = requestFlowCancel({ + flowId: currentFlow.flowId, + expectedRevision: currentFlow.revision, + cancelRequestedAt: 120, + updatedAt: 120, + }); + expect(cancelResult.applied).toBe(true); + + expect(previewTaskFlowRegistryMaintenance()).toEqual({ reconciled: 0, pruned: 0 }); + expect(await runTaskFlowRegistryMaintenance()).toEqual({ reconciled: 0, pruned: 0 }); + expect(getTaskFlowById(flow.flowId)).toMatchObject({ + status: "running", + cancelRequestedAt: 120, + }); + }); + }); + it("prunes many old terminal flows while keeping fresh and active ones", async () => { await withTaskFlowMaintenanceStateDir(async () => { const now = Date.now(); diff --git a/src/tasks/task-flow-registry.maintenance.ts b/src/tasks/task-flow-registry.maintenance.ts index ee976fdacfe..2c3552b8476 100644 --- a/src/tasks/task-flow-registry.maintenance.ts +++ b/src/tasks/task-flow-registry.maintenance.ts @@ -1,5 +1,6 @@ // Reconciles stale task-flow records with their child task state. import { listTasksForFlowId } from "./runtime-internal.js"; +import { isTaskFlowCancellationPending } from "./task-cancellation-state.js"; import { listTaskFlowAuditFindings, summarizeTaskFlowAuditFindings, @@ -32,9 +33,7 @@ function isTerminalFlow(flow: TaskFlowRecord): boolean { } function hasActiveLinkedTasks(flowId: string): boolean { - return listTasksForFlowId(flowId).some( - (task) => task.status === "queued" || task.status === "running", - ); + return listTasksForFlowId(flowId).some(isTaskFlowCancellationPending); } function resolveTerminalAt(flow: TaskFlowRecord): number { diff --git a/src/tasks/task-registry-control.types.ts b/src/tasks/task-registry-control.types.ts index d95714ff572..f8a835f23d4 100644 --- a/src/tasks/task-registry-control.types.ts +++ b/src/tasks/task-registry-control.types.ts @@ -1,5 +1,10 @@ // Defines task control runtime contracts exposed to command surfaces. import type { OpenClawConfig } from "../config/types.openclaw.js"; +import type { DetachedTaskTerminalState } from "./detached-task-runtime-contract.js"; + +type KillSubagentTargetState = + | { state: "finalizing" } + | { state: "terminal"; task: DetachedTaskTerminalState }; /** Admin cancellation hook for ACP sessions owned by task records. */ export type CancelAcpSessionAdmin = (params: { @@ -8,14 +13,17 @@ export type CancelAcpSessionAdmin = (params: { reason: string; }) => Promise; -export type KillSubagentRunAdminResult = { - found: boolean; - killed: boolean; - runId?: string; - sessionKey?: string; - cascadeKilled?: number; - cascadeLabels?: string[]; -}; +export type KillSubagentRunAdminResult = + | { found: false; killed: false } + | { + found: true; + killed: boolean; + targetState?: KillSubagentTargetState; + runId: string; + sessionKey: string; + cascadeKilled: number; + cascadeLabels?: string[]; + }; export type KillSubagentRunAdmin = (params: { cfg: OpenClawConfig; diff --git a/src/tasks/task-registry.test.ts b/src/tasks/task-registry.test.ts index c79ca1e6509..b78702ef48e 100644 --- a/src/tasks/task-registry.test.ts +++ b/src/tasks/task-registry.test.ts @@ -18,10 +18,12 @@ import type { ParsedAgentSessionKey } from "../routing/session-key.js"; import { withTempDir } from "../test-helpers/temp-dir.js"; import { withEnvAsync } from "../test-utils/env.js"; import { registerActiveCronTaskRun, resetActiveCronTaskRunsForTests } from "./cron-task-cancel.js"; +import { SUBAGENT_KILL_TASK_ERROR } from "./detached-task-runtime-contract.js"; import { createTaskFlowForTask as createTaskFlowForTaskOrNull, createManagedTaskFlow as createManagedTaskFlowOrNull, getTaskFlowById, + requestFlowCancel, resetTaskFlowRegistryForTests, } from "./task-flow-registry.js"; import { configureTaskFlowRegistryRuntime } from "./task-flow-registry.store.js"; @@ -524,6 +526,47 @@ describe("task-registry", () => { }); }); + it("keeps subagent abort lifecycle projections provisional", async () => { + await withTaskRegistryTempDir(async () => { + resetTaskRegistryMemoryForTest(); + createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:main:subagent:abort-race", + runId: "run-subagent-abort-race", + task: "Finish while aborting", + status: "running", + deliveryStatus: "not_applicable", + startedAt: 100, + }); + + emitAgentEvent({ + runId: "run-subagent-abort-race", + stream: "lifecycle", + data: { phase: "end", stopReason: "aborted", endedAt: 200 }, + }); + expectRecordFields(requireTaskByRunId("run-subagent-abort-race"), { + status: "cancelled", + error: SUBAGENT_KILL_TASK_ERROR, + }); + + finalizeTaskRunByRunId({ + runId: "run-subagent-abort-race", + runtime: "subagent", + status: "succeeded", + endedAt: 201, + terminalSummary: "finished", + }); + expectRecordFields(requireTaskByRunId("run-subagent-abort-race"), { + status: "succeeded", + endedAt: 201, + error: undefined, + terminalSummary: "finished", + }); + }); + }); + it("reuses an ACP run task when a derived flow id is linked before a duplicate create", async () => { await withTaskRegistryTempDir(async () => { resetTaskRegistryMemoryForTest({ persist: false }); @@ -654,6 +697,79 @@ describe("task-registry", () => { }); }); + it("recovers only direct subagent kill markers when completion wins the race", async () => { + await withTaskRegistryTempDir(async () => { + for (const entry of [ + { + runId: "run-subagent-late-success", + runtime: "subagent" as const, + childSessionKey: "agent:main:subagent:late-success", + error: SUBAGENT_KILL_TASK_ERROR, + terminalStatus: "succeeded" as const, + terminalError: undefined, + expectedError: undefined, + }, + { + runId: "run-subagent-late-failure", + runtime: "subagent" as const, + childSessionKey: "agent:main:subagent:late-failure", + error: SUBAGENT_KILL_TASK_ERROR, + terminalStatus: "failed" as const, + terminalError: "provider failed", + expectedError: "provider failed", + }, + { + runId: "run-subagent-late-timeout", + runtime: "subagent" as const, + childSessionKey: "agent:main:subagent:late-timeout", + error: SUBAGENT_KILL_TASK_ERROR, + terminalStatus: "timed_out" as const, + terminalError: undefined, + expectedError: undefined, + }, + { + runId: "run-acp-late-success", + runtime: "acp" as const, + childSessionKey: "agent:main:acp:late-success", + error: "Task cancellation requested.", + terminalStatus: "succeeded" as const, + terminalError: undefined, + expectedError: "Task cancellation requested.", + }, + ]) { + createTaskRecord({ + runtime: entry.runtime, + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: entry.childSessionKey, + runId: entry.runId, + task: entry.runId, + status: "running", + deliveryStatus: "not_applicable", + }); + finalizeTaskRunByRunId({ + runId: entry.runId, + runtime: entry.runtime, + status: "cancelled", + endedAt: 200, + error: entry.error, + }); + finalizeTaskRunByRunId({ + runId: entry.runId, + runtime: entry.runtime, + status: entry.terminalStatus, + endedAt: 201, + error: entry.terminalError, + terminalSummary: "completed", + }); + + const task = requireTaskByRunId(entry.runId); + expect(task.status).toBe(entry.runtime === "acp" ? "cancelled" : entry.terminalStatus); + expect(task.error).toBe(entry.expectedError); + } + }); + }); + it("keeps stronger run-scoped terminal states when a late success arrives", async () => { await withTaskRegistryTempDir(async () => { resetTaskRegistryMemoryForTest(); @@ -1223,6 +1339,47 @@ describe("task-registry", () => { }); }); + it("keeps managed cancellation pending while a child kill is provisional", async () => { + await withTaskRegistryTempDir(async () => { + const flow = createManagedTaskFlow({ + ownerKey: "agent:main:main", + controllerId: "tests/task-registry", + goal: "Wait for canonical child state", + }); + const task = createTaskRecord({ + runtime: "subagent", + ownerKey: flow.ownerKey, + scopeKind: "session", + parentFlowId: flow.flowId, + childSessionKey: "agent:worker:subagent:provisional-flow", + runId: "run-provisional-managed-flow", + task: "Resolve cancellation race", + status: "running", + deliveryStatus: "not_applicable", + }); + expect( + requestFlowCancel({ + flowId: flow.flowId, + expectedRevision: getTaskFlowById(flow.flowId)!.revision, + cancelRequestedAt: 100, + }).applied, + ).toBe(true); + + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "cancelled", + endedAt: 200, + error: SUBAGENT_KILL_TASK_ERROR, + }); + + expect(getTaskFlowById(flow.flowId)).toMatchObject({ + status: "queued", + cancelRequestedAt: 100, + }); + }); + }); + it("rejects parent flow links for terminal flows", async () => { await withTaskRegistryTempDir(async () => { resetTaskRegistryMemoryForTest({ persist: false }); @@ -3733,11 +3890,17 @@ describe("task-registry", () => { it("cancels subagent-backed tasks through subagent control", async () => { await withTaskRegistryTempDir(async () => { - hoisted.killSubagentRunAdminMock.mockResolvedValue({ - found: true, - killed: true, + const silentTask = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:worker:subagent:child", + runId: "run-cancel-subagent", + task: "Silent projection", + status: "running", + deliveryStatus: "not_applicable", + notifyPolicy: "silent", }); - const task = createTaskRecord({ runtime: "subagent", ownerKey: "agent:main:main", @@ -3752,6 +3915,30 @@ describe("task-registry", () => { status: "running", deliveryStatus: "pending", }); + const peerTask = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + requesterOrigin: { + channel: "notifychat", + to: "notifychat:123", + }, + childSessionKey: "agent:worker:subagent:child", + runId: "run-cancel-subagent", + task: "Peer projection", + status: "running", + deliveryStatus: "pending", + }); + hoisted.killSubagentRunAdminMock.mockImplementationOnce(async () => { + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "cancelled", + endedAt: 200, + error: SUBAGENT_KILL_TASK_ERROR, + }); + return { found: true, killed: true }; + }); const result = await cancelTaskById({ cfg: {} as never, @@ -3772,13 +3959,755 @@ describe("task-registry", () => { status: "cancelled", error: "Cancelled by operator.", }); + expectRecordFields(getTaskById(peerTask.taskId), { + status: "cancelled", + error: "Cancelled by operator.", + }); + expectRecordFields(getTaskById(silentTask.taskId), { + status: "cancelled", + deliveryStatus: "not_applicable", + error: "Cancelled by operator.", + }); await waitForAssertion(() => expectRecordFields(sentMessageCall(), { channel: "notifychat", to: "notifychat:123", - content: "Background task cancelled: Subagent task (run run-canc).", + content: "Background task cancellation requested: Subagent task (run run-canc).", }), ); + expect(hoisted.sendMessageMock).toHaveBeenCalledTimes(1); + }); + }); + + it("promotes a provisional subagent kill that races task cancellation", async () => { + await withTaskRegistryTempDir(async () => { + const task = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:worker:subagent:concurrent-kill", + runId: "run-subagent-concurrent-kill", + task: "Cancel during teardown", + status: "running", + deliveryStatus: "not_applicable", + }); + hoisted.killSubagentRunAdminMock.mockImplementationOnce(async () => { + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "cancelled", + endedAt: 200, + error: SUBAGENT_KILL_TASK_ERROR, + }); + return { found: true, killed: false }; + }); + + const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId }); + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "succeeded", + endedAt: 201, + terminalSummary: "completed too late", + }); + + expectRecordFields(result, { found: true, cancelled: true }); + expectRecordFields(getTaskById(task.taskId), { + status: "cancelled", + endedAt: 200, + error: "Cancelled by operator.", + terminalSummary: undefined, + }); + }); + }); + + it("reconciles an already-provisional kill before making cancellation sticky", async () => { + await withTaskRegistryTempDir(async () => { + const task = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:worker:subagent:provisional-completion", + runId: "run-subagent-provisional-completion", + task: "Finish before explicit cancellation", + status: "running", + deliveryStatus: "not_applicable", + }); + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "cancelled", + endedAt: 200, + error: SUBAGENT_KILL_TASK_ERROR, + }); + hoisted.killSubagentRunAdminMock.mockResolvedValueOnce({ + found: true, + killed: false, + runId: task.runId!, + sessionKey: task.childSessionKey!, + cascadeKilled: 0, + targetState: { + state: "terminal", + task: { + status: "succeeded", + endedAt: 201, + terminalSummary: "completed", + }, + }, + }); + + const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId }); + + expect(hoisted.killSubagentRunAdminMock).toHaveBeenCalledOnce(); + expectRecordFields(result, { + found: true, + cancelled: false, + reason: "Subagent completed while cancellation was in progress.", + }); + expectRecordFields(getTaskById(task.taskId), { + status: "succeeded", + endedAt: 201, + error: undefined, + terminalSummary: "completed", + }); + }); + }); + + it("preserves subagent success that completes during cancellation", async () => { + await withTaskRegistryTempDir(async () => { + const task = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:worker:subagent:cancel-race", + runId: "run-subagent-cancel-race", + task: "Finish during cancellation", + status: "running", + deliveryStatus: "not_applicable", + }); + const peerTask = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:worker:subagent:cancel-race", + runId: "run-subagent-cancel-race", + task: "Peer projection", + status: "running", + deliveryStatus: "not_applicable", + }); + hoisted.killSubagentRunAdminMock.mockImplementationOnce(async () => { + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "cancelled", + endedAt: 200, + error: SUBAGENT_KILL_TASK_ERROR, + }); + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "succeeded", + endedAt: 201, + terminalSummary: "completed", + }); + return { found: true, killed: true }; + }); + + const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId }); + + expectRecordFields(result, { + found: true, + cancelled: false, + reason: "Subagent completed while cancellation was in progress.", + }); + expectRecordFields(result.task, { + status: "succeeded", + error: undefined, + terminalSummary: "completed", + }); + expectRecordFields(getTaskById(peerTask.taskId), { + status: "succeeded", + error: undefined, + terminalSummary: "completed", + }); + }); + }); + + it("does not cancel a lagging task projection after subagent completion wins", async () => { + await withTaskRegistryTempDir(async () => { + const task = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:worker:subagent:lagging-projection", + runId: "run-subagent-lagging-projection", + task: "Finish before task projection", + status: "running", + deliveryStatus: "not_applicable", + }); + hoisted.killSubagentRunAdminMock.mockResolvedValueOnce({ + found: true, + killed: false, + runId: task.runId!, + sessionKey: task.childSessionKey!, + cascadeKilled: 0, + targetState: { + state: "terminal", + task: { + status: "succeeded", + endedAt: 200, + progressSummary: "final answer", + terminalSummary: "final answer", + terminalOutcome: "blocked", + }, + }, + }); + + const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId }); + + expectRecordFields(result, { + found: true, + cancelled: false, + reason: "Subagent completed while cancellation was in progress.", + }); + expectRecordFields(result.task, { + status: "succeeded", + endedAt: 200, + progressSummary: "final answer", + terminalSummary: "final answer", + terminalOutcome: "blocked", + }); + }); + }); + + it("reconciles a replacement run cancellation into the original task scope", async () => { + await withTaskRegistryTempDir(async () => { + const task = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:worker:subagent:replacement-run", + runId: "run-subagent-before-replacement", + task: "Cancel after recovery", + status: "running", + deliveryStatus: "not_applicable", + }); + hoisted.killSubagentRunAdminMock.mockResolvedValueOnce({ + found: true, + killed: true, + runId: "run-subagent-after-replacement", + sessionKey: task.childSessionKey!, + cascadeKilled: 0, + targetState: { + state: "terminal", + task: { + status: "cancelled", + endedAt: 200, + error: SUBAGENT_KILL_TASK_ERROR, + }, + }, + }); + + const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId }); + + expectRecordFields(result, { found: true, cancelled: true }); + expectRecordFields(getTaskById(task.taskId), { + status: "cancelled", + endedAt: 200, + error: "Cancelled by operator.", + }); + }); + }); + + it("ignores a stale killed snapshot after canonical completion persists", async () => { + await withTaskRegistryTempDir(async () => { + const task = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:worker:subagent:stale-kill-snapshot", + runId: "run-subagent-stale-kill-snapshot", + task: "Complete while admin kill unwinds", + status: "running", + deliveryStatus: "not_applicable", + }); + hoisted.killSubagentRunAdminMock.mockImplementationOnce(async () => { + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "succeeded", + endedAt: 201, + terminalSummary: "completed", + }); + return { + found: true, + killed: false, + runId: task.runId!, + sessionKey: task.childSessionKey!, + cascadeKilled: 0, + targetState: { + state: "terminal" as const, + task: { + status: "cancelled" as const, + endedAt: 200, + error: SUBAGENT_KILL_TASK_ERROR, + terminalSummary: null, + }, + }, + }; + }); + + const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId }); + + expectRecordFields(result, { + found: true, + cancelled: false, + reason: "Subagent completed while cancellation was in progress.", + }); + expectRecordFields(getTaskById(task.taskId), { + status: "succeeded", + endedAt: 201, + error: undefined, + terminalSummary: "completed", + }); + }); + }); + + it("promotes an already-killed run projection during task cancellation", async () => { + await withTaskRegistryTempDir(async () => { + const task = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:worker:subagent:killed-projection", + runId: "run-subagent-killed-projection", + task: "Repair and cancel killed projection", + status: "running", + deliveryStatus: "not_applicable", + }); + hoisted.killSubagentRunAdminMock.mockResolvedValueOnce({ + found: true, + killed: false, + runId: task.runId!, + sessionKey: task.childSessionKey!, + cascadeKilled: 0, + targetState: { + state: "terminal", + task: { + status: "cancelled", + endedAt: 200, + error: SUBAGENT_KILL_TASK_ERROR, + terminalSummary: null, + }, + }, + }); + + const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId }); + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "succeeded", + endedAt: 201, + terminalSummary: "completed too late", + }); + + expectRecordFields(result, { found: true, cancelled: true }); + expectRecordFields(getTaskById(task.taskId), { + status: "cancelled", + endedAt: 200, + error: "Cancelled by operator.", + terminalSummary: undefined, + }); + }); + }); + + it("reports when terminal reconciliation cannot be persisted", async () => { + await withTaskRegistryTempDir(async () => { + const store = createInMemoryTaskRegistryStore(); + configureTaskRegistryRuntime({ store }); + const task = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:worker:subagent:persist-failure", + runId: "run-subagent-persist-failure", + task: "Finish before persistence fails", + status: "running", + deliveryStatus: "not_applicable", + }); + configureTaskRegistryRuntime({ + store: { + ...store, + upsertTaskWithDeliveryState: () => { + throw new Error("task store unavailable"); + }, + }, + }); + hoisted.killSubagentRunAdminMock.mockResolvedValueOnce({ + found: true, + killed: false, + runId: task.runId!, + sessionKey: task.childSessionKey!, + cascadeKilled: 0, + targetState: { + state: "terminal", + task: { status: "succeeded", endedAt: 200, terminalSummary: "done" }, + }, + }); + + const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId }); + + expectRecordFields(result, { + found: true, + cancelled: false, + reason: "Subagent became terminal, but task state reconciliation failed to persist.", + }); + expectRecordFields(result.task, { status: "running" }); + }); + }); + + it("returns a subagent failure that wins during cancellation", async () => { + await withTaskRegistryTempDir(async () => { + const task = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:worker:subagent:failed-race", + runId: "run-subagent-failed-race", + task: "Fail during cancellation", + status: "running", + deliveryStatus: "not_applicable", + }); + hoisted.killSubagentRunAdminMock.mockImplementationOnce(async () => { + return { + found: true, + killed: false, + runId: task.runId!, + sessionKey: task.childSessionKey!, + cascadeKilled: 0, + targetState: { + state: "terminal", + task: { + status: "failed", + endedAt: 200, + error: "provider failed", + progressSummary: "partial work", + terminalSummary: null, + }, + }, + }; + }); + + const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId }); + + expectRecordFields(result, { + found: true, + cancelled: false, + reason: "Subagent became failed while cancellation was in progress.", + }); + expectRecordFields(result.task, { + status: "failed", + endedAt: 200, + error: "provider failed", + progressSummary: "partial work", + }); + }); + }); + + it("defers cancellation while canonical subagent completion is still finalizing", async () => { + await withTaskRegistryTempDir(async () => { + const task = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:worker:subagent:finalizing-race", + runId: "run-subagent-finalizing-race", + task: "Capture final result", + status: "running", + deliveryStatus: "not_applicable", + }); + hoisted.killSubagentRunAdminMock.mockResolvedValueOnce({ + found: true, + killed: false, + runId: task.runId!, + sessionKey: task.childSessionKey!, + cascadeKilled: 0, + targetState: { state: "finalizing" }, + }); + + const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId }); + + expectRecordFields(result, { + found: true, + cancelled: false, + reason: "Subagent completion is still being finalized.", + }); + expectRecordFields(result.task, { status: "running" }); + }); + }); + + it("keeps subagent cancellation terminal when success arrives after cancellation", async () => { + await withTaskRegistryTempDir(async () => { + const task = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:worker:subagent:late-success", + runId: "run-subagent-late-success", + task: "Finish after cancellation", + status: "running", + deliveryStatus: "not_applicable", + }); + hoisted.killSubagentRunAdminMock.mockImplementationOnce(async () => { + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "cancelled", + endedAt: 200, + error: SUBAGENT_KILL_TASK_ERROR, + }); + return { found: true, killed: true }; + }); + + const result = await cancelTaskById({ + cfg: {} as never, + taskId: task.taskId, + reason: SUBAGENT_KILL_TASK_ERROR, + }); + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "succeeded", + endedAt: 201, + terminalSummary: "completed too late", + }); + + expectRecordFields(result, { found: true, cancelled: true }); + expectRecordFields(getTaskById(task.taskId), { + status: "cancelled", + error: "Cancelled by operator.", + terminalSummary: undefined, + }); + }); + }); + + it("accepts subagent success that completed before cancellation", async () => { + await withTaskRegistryTempDir(async () => { + const task = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:worker:subagent:earlier-success", + runId: "run-subagent-earlier-success", + task: "Finish before cancellation", + status: "running", + deliveryStatus: "not_applicable", + }); + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "cancelled", + endedAt: 200, + error: "Cancelled by operator.", + }); + + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "succeeded", + endedAt: 199, + terminalSummary: "completed before cancellation", + }); + + expectRecordFields(getTaskById(task.taskId), { + status: "succeeded", + endedAt: 199, + error: undefined, + terminalSummary: "completed before cancellation", + }); + }); + }); + + it("does not let a repeated kill restore the provisional marker after cancellation", async () => { + await withTaskRegistryTempDir(async () => { + const task = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:worker:subagent:repeated-kill", + runId: "run-subagent-repeated-kill", + task: "Stay cancelled", + status: "running", + deliveryStatus: "not_applicable", + }); + for (const error of [ + SUBAGENT_KILL_TASK_ERROR, + "Cancelled by operator.", + SUBAGENT_KILL_TASK_ERROR, + ]) { + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "cancelled", + endedAt: 200, + error, + }); + } + + expectRecordFields(getTaskById(task.taskId), { + status: "cancelled", + error: "Cancelled by operator.", + }); + }); + }); + + it("promotes an existing subagent kill marker to operator cancellation", async () => { + await withTaskRegistryTempDir(async () => { + const task = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:worker:subagent:already-killed", + runId: "run-subagent-already-killed", + task: "Promote killed task", + status: "running", + deliveryStatus: "not_applicable", + }); + const peerTask = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:worker:subagent:already-killed", + runId: "run-subagent-already-killed", + task: "Peer projection", + status: "running", + deliveryStatus: "not_applicable", + }); + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "cancelled", + endedAt: 200, + error: SUBAGENT_KILL_TASK_ERROR, + }); + hoisted.killSubagentRunAdminMock.mockClear(); + hoisted.killSubagentRunAdminMock.mockResolvedValueOnce({ + found: true, + killed: false, + runId: task.runId!, + sessionKey: task.childSessionKey!, + cascadeKilled: 0, + targetState: { + state: "terminal", + task: { + status: "cancelled", + endedAt: 200, + error: SUBAGENT_KILL_TASK_ERROR, + }, + }, + }); + + const result = await cancelTaskById({ cfg: {} as never, taskId: task.taskId }); + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "succeeded", + endedAt: 201, + terminalSummary: "completed too late", + }); + + expect(hoisted.killSubagentRunAdminMock).toHaveBeenCalledOnce(); + expectRecordFields(result, { found: true, cancelled: true }); + for (const taskId of [task.taskId, peerTask.taskId]) { + expectRecordFields(getTaskById(taskId), { + status: "cancelled", + endedAt: 200, + error: "Cancelled by operator.", + terminalSummary: undefined, + }); + } + }); + }); + + it("suppresses terminal delivery when teardown finalizes a killed task", async () => { + await withTaskRegistryTempDir(async () => { + hoisted.sendMessageMock.mockClear(); + const task = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + requesterOrigin: { channel: "notifychat", to: "notifychat:123" }, + childSessionKey: "agent:worker:subagent:teardown", + runId: "run-subagent-teardown", + task: "Stop silently", + status: "running", + deliveryStatus: "pending", + }); + + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "cancelled", + endedAt: 200, + error: SUBAGENT_KILL_TASK_ERROR, + suppressDelivery: true, + }); + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "cancelled", + endedAt: 201, + error: SUBAGENT_KILL_TASK_ERROR, + }); + await Promise.resolve(); + + expectRecordFields(getTaskById(task.taskId), { + status: "cancelled", + error: SUBAGENT_KILL_TASK_ERROR, + deliveryStatus: "not_applicable", + }); + expect(hoisted.sendMessageMock).not.toHaveBeenCalled(); + }); + }); + + it("stops a pending terminal notifier when teardown suppresses delivery", async () => { + await withTaskRegistryTempDir(async () => { + hoisted.sendMessageMock.mockClear(); + const task = createTaskRecord({ + runtime: "subagent", + ownerKey: "agent:main:main", + scopeKind: "session", + requesterOrigin: { channel: "notifychat", to: "notifychat:123" }, + childSessionKey: "agent:worker:subagent:pending-teardown", + runId: "run-subagent-pending-teardown", + task: "Stop pending delivery", + status: "running", + deliveryStatus: "pending", + }); + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "cancelled", + endedAt: 200, + error: SUBAGENT_KILL_TASK_ERROR, + }); + + const pendingDelivery = maybeDeliverTaskTerminalUpdate(task.taskId); + finalizeTaskRunByRunId({ + runId: task.runId!, + runtime: "subagent", + status: "cancelled", + endedAt: 201, + error: SUBAGENT_KILL_TASK_ERROR, + suppressDelivery: true, + }); + await pendingDelivery; + + expect(hoisted.sendMessageMock).not.toHaveBeenCalled(); + expectRecordFields(getTaskById(task.taskId), { + status: "cancelled", + deliveryStatus: "not_applicable", + }); }); }); diff --git a/src/tasks/task-registry.ts b/src/tasks/task-registry.ts index 8b6348300bb..f44b5d18d5c 100644 --- a/src/tasks/task-registry.ts +++ b/src/tasks/task-registry.ts @@ -19,7 +19,9 @@ import { createLazyPromiseLoader } from "../shared/lazy-runtime.js"; import { normalizeDeliveryContext } from "../utils/delivery-context.shared.js"; import { isDeliverableMessageChannel } from "../utils/message-channel.js"; import { cancelActiveCronTaskRun } from "./cron-task-cancel.js"; +import { SUBAGENT_KILL_TASK_ERROR } from "./detached-task-runtime-contract.js"; import { isChildlessNativeSubagentTask } from "./native-subagent-task.js"; +import { isTaskFlowCancellationPending } from "./task-cancellation-state.js"; import { formatTaskBlockedFollowupMessage, formatTaskStateChangeMessage, @@ -496,8 +498,25 @@ function normalizeTaskTerminalOutcome( function shouldApplyRunScopedStatusUpdate(params: { currentStatus: TaskStatus; + currentRuntime: TaskRuntime; + currentChildSessionKey?: string; + currentError?: string; + currentEndedAt?: number; nextStatus: TaskStatus; + nextError?: string; + nextEndedAt?: number; }): boolean { + if ( + params.currentRuntime === "subagent" && + params.nextStatus === "cancelled" && + params.nextError === SUBAGENT_KILL_TASK_ERROR && + isTerminalTaskStatus(params.currentStatus) && + !(params.currentStatus === "cancelled" && params.currentError === SUBAGENT_KILL_TASK_ERROR) + ) { + // The kill marker is provisional. It may refresh only its own tombstone; + // canonical completion or operator cancellation already won this race. + return false; + } if (params.currentStatus === params.nextStatus) { return true; } @@ -507,6 +526,26 @@ function shouldApplyRunScopedStatusUpdate(params: { if (!isTerminalTaskStatus(params.nextStatus)) { return false; } + // Direct subagent termination is provisional. An operator cancellation is + // sticky only against outcomes that completed at or after cancellation. + if ( + params.currentStatus === "cancelled" && + (params.nextStatus === "succeeded" || + params.nextStatus === "failed" || + params.nextStatus === "timed_out") + ) { + const canonicalOutcomePredatesCancellation = + params.currentRuntime === "subagent" && + params.currentEndedAt !== undefined && + params.nextEndedAt !== undefined && + params.nextEndedAt < params.currentEndedAt; + return ( + canonicalOutcomePredatesCancellation || + (params.currentRuntime === "subagent" && + Boolean(params.currentChildSessionKey?.trim()) && + params.currentError === SUBAGENT_KILL_TASK_ERROR) + ); + } return params.currentStatus === "succeeded" && params.nextStatus !== "lost"; } @@ -541,6 +580,18 @@ function mapAgentRunTerminalOutcomeToTaskStatus( } } +function resolveTaskLifecycleTerminalError(params: { + runtime: TaskRuntime; + status: TaskStatus; + error?: string; +}): string | undefined { + // A runner abort can race either an accepted task cancellation or a real + // completion. Keep it provisional until the task-control owner decides. + return params.runtime === "subagent" && params.status === "cancelled" + ? SUBAGENT_KILL_TASK_ERROR + : params.error; +} + function buildTaskLifecycleTerminalOutcome(params: { phase: "end" | "error"; data?: Record; @@ -1044,7 +1095,7 @@ function syncManagedFlowCancellationFromTask(task: TaskRecord): void { ) { return; } - if (listTasksForFlowId(flowId).some((candidate) => isActiveTaskStatus(candidate.status))) { + if (listTasksForFlowId(flowId).some(isTaskFlowCancellationPending)) { return; } const endedAt = task.endedAt ?? task.lastEventAt ?? Date.now(); @@ -1073,7 +1124,7 @@ function syncManagedFlowCancellationFromTask(task: TaskRecord): void { ) { return; } - if (listTasksForFlowId(flowId).some((candidate) => isActiveTaskStatus(candidate.status))) { + if (listTasksForFlowId(flowId).some(isTaskFlowCancellationPending)) { return; } } @@ -1360,11 +1411,27 @@ export async function maybeDeliverTaskTerminalUpdate(taskId: string): Promise shouldAutoDeliverTaskTerminalUpdate(candidate)) + : peers, + ); + const peerDeliveryCovered = + isSubagentCancellation && + peers.some( + (candidate) => + candidate.taskId !== latest.taskId && + (candidate.deliveryStatus === "delivered" || + candidate.deliveryStatus === "session_queued"), + ); if ( - shouldSuppressDuplicateTerminalDelivery({ task: latest, preferredTaskId: preferred?.taskId }) + shouldSuppressDuplicateTerminalDelivery({ + task: latest, + preferredTaskId: preferred?.taskId, + peerDeliveryCovered, + }) ) { return updateTask(taskId, { deliveryStatus: "not_applicable", @@ -1413,6 +1480,10 @@ export async function maybeDeliverTaskTerminalUpdate(taskId: string): Promise candidate.taskId === task.taskId); + if (!reconciled) { + return { + found: true, + cancelled: false, + reason: "Subagent became terminal, but task state reconciliation failed to persist.", + task: cloneTaskRecord(tasks.get(task.taskId) ?? task), + }; + } + if ( + result.targetState.task.status === "cancelled" && + result.targetState.task.error === SUBAGENT_KILL_TASK_ERROR + ) { + isProvisionalSubagentKill = true; + } else { + const reason = + result.targetState.task.status === "succeeded" + ? "Subagent completed while cancellation was in progress." + : `Subagent became ${result.targetState.task.status} while cancellation was in progress.`; + return { + found: true, + cancelled: false, + reason, + task: cloneTaskRecord(reconciled), + }; + } + } + if (result.found && result.targetState?.state === "finalizing") { + return { + found: true, + cancelled: false, + reason: "Subagent completion is still being finalized.", + task: cloneTaskRecord(current ?? task), + }; + } + if ((!result.found || !result.killed) && !isProvisionalSubagentKill) { return { found: true, cancelled: false, reason: result.found ? "Subagent was not running." : "Subagent task not found.", - task: cloneTaskRecord(task), + task: cloneTaskRecord(current ?? task), }; } } else { @@ -2163,24 +2360,25 @@ export async function cancelTaskById(params: { }; } } - const endedAt = Date.now(); - const error = params.reason?.trim() || "Cancelled by operator."; + const eventAt = Date.now(); + const current = tasks.get(task.taskId) ?? task; + const endedAt = isProvisionalSubagentKill ? (current.endedAt ?? eventAt) : eventAt; const updated = - task.runtime === "acp" && task.runId?.trim() + (task.runtime === "acp" || task.runtime === "subagent") && task.runId?.trim() ? (updateTaskStateByRunId({ runId: task.runId, - runtime: "acp", + runtime: task.runtime, sessionKey: childSessionKey, status: "cancelled", endedAt, - lastEventAt: endedAt, - error, + lastEventAt: eventAt, + error: cancellationError, }).find((record) => record.taskId === task.taskId) ?? null) : updateTask(task.taskId, { status: "cancelled", endedAt, - lastEventAt: endedAt, - error, + lastEventAt: eventAt, + error: cancellationError, }); if (!updated) { return {