mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 09:39:25 +00:00
refactor: await durable subagent announcement cleanup (#161273)
Persist ordinary cleanup, retry, delivery and retirement through the existing state worker and staged publication owner. Keep source and run-generation authority current across awaited work before dependent effects. Move descendant checks to the prepared reader, remove the unused synchronous tree-reader chain, and retain real registered completion, deferred delivery and publication-race coverage. Related: #150153
This commit is contained in:
parent
1a4c5d9620
commit
969ae095cc
66 changed files with 4193 additions and 1773 deletions
|
|
@ -508,7 +508,10 @@ async function runSweepSample(childCount: number): Promise<Sample> {
|
|||
},
|
||||
resumeRequesterSettleWake: () => {},
|
||||
startSubagentAnnounceCleanupFlow: () => true,
|
||||
completeCleanupBookkeeping: () => {},
|
||||
completeCleanupBookkeeping: async () => {},
|
||||
isEndedHookOwnerCurrent: (runId, entry) => runs.get(runId) === entry || !runs.has(runId),
|
||||
sessionEffectsHostCurrent: (entry) => entry.execution.suppressSessionEffects !== true,
|
||||
shouldSuppressSessionEffects: async (entry) => entry.execution.suppressSessionEffects === true,
|
||||
discardTerminalDelivery: () => {},
|
||||
shouldEmitEndedHookForRun: () => false,
|
||||
emitSubagentEndedHookForRun: async () => {},
|
||||
|
|
|
|||
|
|
@ -241,7 +241,7 @@ it("injects complete lifecycle results into requester prompts and acknowledges o
|
|||
publication.onCommitted?.();
|
||||
},
|
||||
clearPendingLifecycleError: vi.fn(),
|
||||
countPendingDescendantRuns: () => 0,
|
||||
countPendingDescendantRuns: async () => 0,
|
||||
getLatestRunForChildSession: () => null,
|
||||
suppressAnnounceForSteerRestart: () => false,
|
||||
shouldEmitEndedHookForRun: () => false,
|
||||
|
|
|
|||
|
|
@ -63,6 +63,26 @@ export function createSubagentPersistenceMock(
|
|||
};
|
||||
return {
|
||||
onSubagentRegistryPersisted: (listener: () => void) => registerListener(listeners, listener),
|
||||
// Policy fixtures supply retained rows in memory; worker custody uses the real state owner.
|
||||
withSubagentRunReadSnapshot: (async (runs, select, consume) => {
|
||||
await Promise.resolve();
|
||||
const selected = select(new Map(runs));
|
||||
const runIds = new Set(selected.runIds);
|
||||
const sessionKeys = new Set(selected.sessionKeys);
|
||||
return consume(
|
||||
selected,
|
||||
new Map(
|
||||
[...runs].filter(
|
||||
([runId, entry]) =>
|
||||
runIds.has(runId) ||
|
||||
sessionKeys.has(entry.requesterSessionKey.trim()) ||
|
||||
Boolean(
|
||||
entry.controllerSessionKey && sessionKeys.has(entry.controllerSessionKey.trim()),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
}) satisfies typeof RegistryPersistence.withSubagentRunReadSnapshot,
|
||||
persistSubagentRunsToDisk: publishAfter(methods.persistSubagentRunsToDisk),
|
||||
persistSubagentRunsToDiskOrThrow: publishAfter(methods.persistSubagentRunsToDiskOrThrow),
|
||||
restoreSubagentRunsFromDisk: async (
|
||||
|
|
|
|||
|
|
@ -147,9 +147,7 @@ const { subagentRegistryMock } = vi.hoisted(() => ({
|
|||
subagentRegistryMock: {
|
||||
isSubagentSessionRunActive: vi.fn(() => true),
|
||||
shouldIgnorePostCompletionAnnounceForSession: vi.fn((_sessionKey: string) => false),
|
||||
countActiveDescendantRuns: vi.fn((_sessionKey: string) => 0),
|
||||
countPendingDescendantRuns: vi.fn((_sessionKey: string) => 0),
|
||||
hasDescendantRunAwaitingSettle: vi.fn((_sessionKey: string, _excludeRunId?: string) => false),
|
||||
getLatestSubagentRunByChildSessionKey: vi.fn(
|
||||
(_childSessionKey: string): MockSubagentRun | undefined => undefined,
|
||||
),
|
||||
|
|
@ -514,18 +512,7 @@ describe("subagent announce formatting", () => {
|
|||
subagentRegistryMock.shouldIgnorePostCompletionAnnounceForSession
|
||||
.mockClear()
|
||||
.mockReturnValue(false);
|
||||
subagentRegistryMock.countActiveDescendantRuns.mockClear().mockReturnValue(0);
|
||||
subagentRegistryMock.countPendingDescendantRuns
|
||||
.mockClear()
|
||||
.mockImplementation((sessionKey: string) =>
|
||||
subagentRegistryMock.countActiveDescendantRuns(sessionKey),
|
||||
);
|
||||
subagentRegistryMock.hasDescendantRunAwaitingSettle
|
||||
.mockClear()
|
||||
.mockImplementation(
|
||||
(sessionKey: string, _excludeRunId?: string) =>
|
||||
subagentRegistryMock.countPendingDescendantRuns(sessionKey) > 0,
|
||||
);
|
||||
subagentRegistryMock.countPendingDescendantRuns.mockReset().mockReturnValue(0);
|
||||
subagentRegistryMock.getLatestSubagentRunByChildSessionKey
|
||||
.mockClear()
|
||||
.mockReturnValue(undefined);
|
||||
|
|
@ -1284,7 +1271,7 @@ describe("subagent announce formatting", () => {
|
|||
chatHistoryMock.mockResolvedValueOnce({
|
||||
messages: [{ role: "assistant", content: [{ type: "text", text: "final answer: 2" }] }],
|
||||
});
|
||||
subagentRegistryMock.countActiveDescendantRuns.mockImplementation((sessionKey: string) =>
|
||||
subagentRegistryMock.countPendingDescendantRuns.mockImplementation((sessionKey: string) =>
|
||||
sessionKey === "agent:main:main" ? 1 : 0,
|
||||
);
|
||||
|
||||
|
|
@ -1321,7 +1308,7 @@ describe("subagent announce formatting", () => {
|
|||
chatHistoryMock.mockResolvedValueOnce({
|
||||
messages: [{ role: "assistant", content: [{ type: "text", text: "bound answer: 2" }] }],
|
||||
});
|
||||
subagentRegistryMock.countActiveDescendantRuns.mockImplementation((sessionKey: string) =>
|
||||
subagentRegistryMock.countPendingDescendantRuns.mockImplementation((sessionKey: string) =>
|
||||
sessionKey === "agent:main:main" ? 1 : 0,
|
||||
);
|
||||
registerBoundSubagent({
|
||||
|
|
@ -1405,7 +1392,7 @@ describe("subagent announce formatting", () => {
|
|||
};
|
||||
|
||||
// Simulate active sibling runs so non-bound paths would normally coordinate via agent().
|
||||
subagentRegistryMock.countActiveDescendantRuns.mockImplementation((sessionKey: string) =>
|
||||
subagentRegistryMock.countPendingDescendantRuns.mockImplementation((sessionKey: string) =>
|
||||
sessionKey === "agent:main:main" ? 2 : 0,
|
||||
);
|
||||
registerSessionBindingAdapter({
|
||||
|
|
@ -1501,7 +1488,7 @@ describe("subagent announce formatting", () => {
|
|||
chatHistoryMock.mockResolvedValueOnce({
|
||||
messages: [{ role: "assistant", content: [{ type: "text", text: "matrix bound answer" }] }],
|
||||
});
|
||||
subagentRegistryMock.countActiveDescendantRuns.mockImplementation((sessionKey: string) =>
|
||||
subagentRegistryMock.countPendingDescendantRuns.mockImplementation((sessionKey: string) =>
|
||||
sessionKey === "agent:main:main" ? 1 : 0,
|
||||
);
|
||||
registerBoundSubagent({
|
||||
|
|
@ -2509,7 +2496,7 @@ describe("subagent announce formatting", () => {
|
|||
});
|
||||
|
||||
it("does not include batching guidance when sibling subagents are still active", async () => {
|
||||
subagentRegistryMock.countActiveDescendantRuns.mockImplementation((sessionKey: string) =>
|
||||
subagentRegistryMock.countPendingDescendantRuns.mockImplementation((sessionKey: string) =>
|
||||
sessionKey === "agent:main:main" ? 2 : 0,
|
||||
);
|
||||
|
||||
|
|
@ -2592,7 +2579,6 @@ describe("subagent announce formatting", () => {
|
|||
|
||||
it("announces completion immediately when no descendants are pending", async () => {
|
||||
subagentRegistryMock.countPendingDescendantRuns.mockReturnValue(0);
|
||||
subagentRegistryMock.countActiveDescendantRuns.mockReturnValue(0);
|
||||
|
||||
const didAnnounce = await runSubagentAnnounceFlow({
|
||||
...defaultOutcomeAnnounce,
|
||||
|
|
@ -3700,6 +3686,7 @@ describe("subagent announce formatting", () => {
|
|||
expect(second).toBe("delivered");
|
||||
expect(subagentRegistryMock.countPendingDescendantRuns).toHaveBeenCalledWith(
|
||||
"agent:main:subagent:parent-gated",
|
||||
expect.any(Function),
|
||||
);
|
||||
expect(agentSpy).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import {
|
|||
} from "../registry/subagent-registry-queries.js";
|
||||
import {
|
||||
registryRuntimeMock,
|
||||
readDescendantFacts,
|
||||
wakeParams,
|
||||
} from "./subagent-announce.requester-settle-fixture.test-support.js";
|
||||
import { maybeWakeRequesterAfterAllChildrenSettled } from "./subagent-announce.requester-settle-wake.js";
|
||||
|
|
@ -69,12 +70,24 @@ it.each([
|
|||
registryRuntimeMock.listSubagentRunsForRequester.mockImplementation((key) =>
|
||||
[...runs.values()].filter((entry) => entry.requesterSessionKey === key),
|
||||
);
|
||||
registryRuntimeMock.hasDescendantRunAwaitingSettle.mockImplementation((...args) =>
|
||||
hasDescendantRunAwaitingSettleFromRuns(runs, ...args),
|
||||
);
|
||||
registryRuntimeMock.countActiveDescendantRuns.mockImplementation((...args) =>
|
||||
countActiveDescendantRunsFromRuns(runs, ...args),
|
||||
);
|
||||
readDescendantFacts.mockImplementation(async (params) => ({
|
||||
unsettled: hasDescendantRunAwaitingSettleFromRuns(
|
||||
runs,
|
||||
params.requesterSessionKey,
|
||||
params.settledEntry.runId,
|
||||
params.requesterAgentId,
|
||||
params.requesterStorePath,
|
||||
params.settledBefore,
|
||||
params.rootRunIds,
|
||||
),
|
||||
active: countActiveDescendantRunsFromRuns(
|
||||
runs,
|
||||
params.requesterSessionKey,
|
||||
params.requesterAgentId,
|
||||
params.requesterStorePath,
|
||||
params.rootRunIds,
|
||||
),
|
||||
}));
|
||||
const early = scenario === "older batch";
|
||||
expect(await maybeWakeRequesterAfterAllChildrenSettled(wakeParams({ settledEntry: short }))).toBe(
|
||||
early,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,85 @@
|
|||
import { subagentRuns } from "../registry/subagent-registry-memory.js";
|
||||
import {
|
||||
countActiveDescendantRunsFromRuns,
|
||||
hasDescendantRunAwaitingSettleFromRuns,
|
||||
} from "../registry/subagent-registry-queries.js";
|
||||
import { listSubagentRunsForRequester } from "../registry/subagent-registry-read.js";
|
||||
import { withSubagentRunReadSnapshot } from "../registry/subagent-registry-state.js";
|
||||
import type { SubagentRunRecord } from "../registry/subagent-registry.types.js";
|
||||
import { collectSubagentSessionReadKeys } from "../registry/subagent-session-read-scope.js";
|
||||
|
||||
export function createRequesterDescendantReader(params: {
|
||||
requesterSessionKey: string;
|
||||
requesterAgentId?: string;
|
||||
requesterStorePath: string | null;
|
||||
settledEntry: SubagentRunRecord;
|
||||
settledBefore: number;
|
||||
rootRunIds?: ReadonlySet<string>;
|
||||
signal?: AbortSignal;
|
||||
isSourceCurrent: () => boolean;
|
||||
}) {
|
||||
const generation = params.settledEntry.generation;
|
||||
const rearmGeneration = params.settledEntry.requesterSettleWake?.rearmGeneration;
|
||||
const isCurrent = () => {
|
||||
const wake = params.settledEntry.requesterSettleWake;
|
||||
return (
|
||||
!params.signal?.aborted &&
|
||||
params.isSourceCurrent() &&
|
||||
params.settledEntry.generation === generation &&
|
||||
wake !== undefined &&
|
||||
wake.rearmGeneration === rearmGeneration &&
|
||||
listSubagentRunsForRequester(params.requesterSessionKey, {
|
||||
requesterAgentId: params.requesterAgentId,
|
||||
requesterStorePath: params.requesterStorePath,
|
||||
}).includes(params.settledEntry)
|
||||
);
|
||||
};
|
||||
return async () => {
|
||||
if (!isCurrent()) {
|
||||
return undefined;
|
||||
}
|
||||
const result = await withSubagentRunReadSnapshot(
|
||||
subagentRuns,
|
||||
(snapshot) => {
|
||||
if (!isCurrent()) {
|
||||
return { runIds: [], sessionKeys: [] };
|
||||
}
|
||||
const sessionKeys = collectSubagentSessionReadKeys(
|
||||
[params.requesterSessionKey],
|
||||
snapshot.values(),
|
||||
);
|
||||
return {
|
||||
runIds: [...snapshot.values()]
|
||||
.filter((entry) => sessionKeys.has(entry.childSessionKey.trim()))
|
||||
.map((entry) => entry.runId),
|
||||
sessionKeys: [],
|
||||
};
|
||||
},
|
||||
(_selection, runs) => {
|
||||
if (!isCurrent()) {
|
||||
return undefined;
|
||||
}
|
||||
const snapshot = new Map(runs);
|
||||
return {
|
||||
unsettled: hasDescendantRunAwaitingSettleFromRuns(
|
||||
snapshot,
|
||||
params.requesterSessionKey,
|
||||
params.settledEntry.runId,
|
||||
params.requesterAgentId,
|
||||
params.requesterStorePath,
|
||||
params.settledBefore,
|
||||
params.rootRunIds,
|
||||
),
|
||||
active: countActiveDescendantRunsFromRuns(
|
||||
snapshot,
|
||||
params.requesterSessionKey,
|
||||
params.requesterAgentId,
|
||||
params.requesterStorePath,
|
||||
params.rootRunIds,
|
||||
),
|
||||
};
|
||||
},
|
||||
);
|
||||
return isCurrent() ? result : undefined;
|
||||
};
|
||||
}
|
||||
|
|
@ -31,6 +31,7 @@ import { createEmbeddedRunLaneController } from "../../embedded-agent-runner/run
|
|||
import type { RunEmbeddedAgentParams } from "../../embedded-agent-runner/run/params.js";
|
||||
import { MAIN_SESSION_RECOVERY_WORK_ADMISSION_OWNER } from "../../main-session-recovery/main-session-recovery-admission.js";
|
||||
import { resolveAgentTimeoutMs } from "../../timeout.js";
|
||||
import type { countPendingDescendantRuns } from "../registry/subagent-registry-read.js";
|
||||
import type { SubagentRunRecord } from "../registry/subagent-registry.types.js";
|
||||
import {
|
||||
registerRequesterFinalAttachment,
|
||||
|
|
@ -38,14 +39,34 @@ import {
|
|||
} from "../requester-final-attachment.js";
|
||||
import { sendSubagentAnnounceDirectly } from "./subagent-announce-direct-delivery.js";
|
||||
import { setSubagentAnnounceDeliveryDepsForTest } from "./subagent-announce-overrides.test-support.js";
|
||||
import type { createRequesterDescendantReader } from "./subagent-announce.requester-settle-descendants.js";
|
||||
|
||||
const readDescendantFacts = vi.hoisted(() =>
|
||||
vi.fn<
|
||||
(
|
||||
params: Parameters<typeof createRequesterDescendantReader>[0],
|
||||
) => ReturnType<ReturnType<typeof createRequesterDescendantReader>>
|
||||
>(async () => ({ unsettled: false, active: 0 })),
|
||||
);
|
||||
|
||||
vi.mock("./subagent-announce.requester-settle-descendants.js", () => ({
|
||||
createRequesterDescendantReader:
|
||||
(params: Parameters<typeof createRequesterDescendantReader>[0]) => () =>
|
||||
readDescendantFacts(params),
|
||||
}));
|
||||
|
||||
const startTurn = vi.hoisted(() => vi.fn());
|
||||
const deliver = vi.hoisted(() => vi.fn());
|
||||
const registryRead = vi.hoisted(() => ({
|
||||
countPendingDescendantRuns: vi.fn<typeof countPendingDescendantRuns>(
|
||||
async (_key, assertCurrent) => {
|
||||
assertCurrent();
|
||||
return 0;
|
||||
},
|
||||
),
|
||||
getLatestLiveSubagentRunByChildSessionKey: vi.fn<() => SubagentRunRecord | undefined>(
|
||||
() => undefined,
|
||||
),
|
||||
hasDescendantRunAwaitingSettle: vi.fn(() => false),
|
||||
listSubagentRunsForRequester: vi.fn<() => SubagentRunRecord[]>(() => []),
|
||||
getLatestSubagentRunByChildSessionKey: vi.fn(() => undefined),
|
||||
}));
|
||||
|
|
@ -151,7 +172,7 @@ describe("requester settle dispatch deadline", () => {
|
|||
resetCommandQueueStateForTest();
|
||||
startTurn.mockReset();
|
||||
deliver.mockReset();
|
||||
registryRead.hasDescendantRunAwaitingSettle.mockReset().mockReturnValue(false);
|
||||
readDescendantFacts.mockReset().mockResolvedValue({ unsettled: false, active: 0 });
|
||||
registryRead.getLatestLiveSubagentRunByChildSessionKey.mockReset().mockReturnValue(undefined);
|
||||
registryRead.getLatestSubagentRunByChildSessionKey.mockReset().mockReturnValue(undefined);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,9 +1,6 @@
|
|||
import { beforeEach, vi } from "vitest";
|
||||
import type {
|
||||
countActiveDescendantRuns,
|
||||
hasDescendantRunAwaitingSettle,
|
||||
} from "../registry/subagent-registry-read.js";
|
||||
import type { SubagentRunRecord } from "../registry/subagent-registry.types.js";
|
||||
import type { createRequesterDescendantReader } from "./subagent-announce.requester-settle-descendants.js";
|
||||
import type { maybeWakeRequesterAfterAllChildrenSettled } from "./subagent-announce.requester-settle-wake.js";
|
||||
import {
|
||||
REQUESTER,
|
||||
|
|
@ -17,17 +14,20 @@ import {
|
|||
|
||||
let sessionStore: Record<string, { sessionId?: string; lastChannel?: string; lastTo?: string }>;
|
||||
|
||||
const { registryRuntimeMock, findTranscriptEventMock } = vi.hoisted(() => ({
|
||||
const { registryRuntimeMock, findTranscriptEventMock, readDescendantFacts } = vi.hoisted(() => ({
|
||||
readDescendantFacts: vi.fn<
|
||||
(
|
||||
params: Parameters<typeof createRequesterDescendantReader>[0],
|
||||
) => ReturnType<ReturnType<typeof createRequesterDescendantReader>>
|
||||
>(async () => ({ unsettled: false, active: 0 })),
|
||||
findTranscriptEventMock: vi.fn<
|
||||
typeof import("../../../config/sessions/session-accessor.js").findTranscriptEvent
|
||||
>(async () => undefined),
|
||||
registryRuntimeMock: {
|
||||
getLatestLiveSubagentRunByChildSessionKey: vi.fn(() => undefined),
|
||||
countActiveDescendantRuns: vi.fn<typeof countActiveDescendantRuns>(() => 0),
|
||||
countPendingDescendantRuns: vi.fn((_rootSessionKey: string) => 0),
|
||||
isSubagentSessionRunActive: vi.fn((_childSessionKey: string) => true),
|
||||
shouldIgnorePostCompletionAnnounceForSession: vi.fn((_childSessionKey: string) => false),
|
||||
hasDescendantRunAwaitingSettle: vi.fn<typeof hasDescendantRunAwaitingSettle>(() => false),
|
||||
listSubagentRunsForRequester: vi.fn((_requesterSessionKey: string): unknown[] => []),
|
||||
getLatestSubagentRunByChildSessionKey: vi.fn(
|
||||
(
|
||||
|
|
@ -39,6 +39,11 @@ const { registryRuntimeMock, findTranscriptEventMock } = vi.hoisted(() => ({
|
|||
}));
|
||||
|
||||
vi.mock("../registry/subagent-registry-read.js", () => registryRuntimeMock);
|
||||
vi.mock("./subagent-announce.requester-settle-descendants.js", () => ({
|
||||
createRequesterDescendantReader:
|
||||
(params: Parameters<typeof createRequesterDescendantReader>[0]) => () =>
|
||||
readDescendantFacts(params),
|
||||
}));
|
||||
|
||||
vi.mock("../../../config/sessions/session-accessor.js", () => ({
|
||||
findTranscriptEvent: findTranscriptEventMock,
|
||||
|
|
@ -104,8 +109,7 @@ beforeEach(() => {
|
|||
transitionBatchSpy.mockClear();
|
||||
completeBatchSpy.mockClear();
|
||||
sessionStore = { [REQUESTER]: { sessionId: "sess-main" } };
|
||||
registryRuntimeMock.countActiveDescendantRuns.mockReset().mockReturnValue(0);
|
||||
registryRuntimeMock.hasDescendantRunAwaitingSettle.mockReset().mockReturnValue(false);
|
||||
readDescendantFacts.mockReset().mockResolvedValue({ unsettled: false, active: 0 });
|
||||
registryRuntimeMock.listSubagentRunsForRequester.mockReset().mockReturnValue([]);
|
||||
registryRuntimeMock.getLatestSubagentRunByChildSessionKey.mockReset().mockReturnValue(undefined);
|
||||
});
|
||||
|
|
@ -116,6 +120,7 @@ function setSessionStore(store: typeof sessionStore): void {
|
|||
|
||||
export {
|
||||
sessionStore,
|
||||
readDescendantFacts,
|
||||
setSessionStore,
|
||||
registryRuntimeMock,
|
||||
findTranscriptEventMock,
|
||||
|
|
|
|||
|
|
@ -3,11 +3,24 @@ import { createDeferred } from "../../../../test/helpers/promise.js";
|
|||
import { publishSystemEventStoreResolver } from "../../../infra/system-event-ownership.js";
|
||||
import type { SubagentRunRecord } from "../registry/subagent-registry.types.js";
|
||||
import type { SubagentAnnounceDeliveryResult } from "./subagent-announce-dispatch.js";
|
||||
import type { createRequesterDescendantReader } from "./subagent-announce.requester-settle-descendants.js";
|
||||
|
||||
const readDescendantFacts = vi.hoisted(() =>
|
||||
vi.fn<
|
||||
(
|
||||
params: Parameters<typeof createRequesterDescendantReader>[0],
|
||||
) => ReturnType<ReturnType<typeof createRequesterDescendantReader>>
|
||||
>(async () => ({ unsettled: false, active: 0 })),
|
||||
);
|
||||
|
||||
vi.mock("./subagent-announce.requester-settle-descendants.js", () => ({
|
||||
createRequesterDescendantReader:
|
||||
(params: Parameters<typeof createRequesterDescendantReader>[0]) => () =>
|
||||
readDescendantFacts(params),
|
||||
}));
|
||||
|
||||
const { registryRuntimeMock, deliverSpy } = vi.hoisted(() => ({
|
||||
registryRuntimeMock: {
|
||||
countActiveDescendantRuns: vi.fn(() => 0),
|
||||
hasDescendantRunAwaitingSettle: vi.fn(() => false),
|
||||
listSubagentRunsForRequester: vi.fn<() => SubagentRunRecord[]>(() => []),
|
||||
getLatestSubagentRunByChildSessionKey: vi.fn(() => undefined),
|
||||
getLatestLiveSubagentRunByChildSessionKey: vi.fn(() => undefined),
|
||||
|
|
@ -91,6 +104,7 @@ function wakeParams() {
|
|||
}
|
||||
|
||||
beforeEach(() => {
|
||||
readDescendantFacts.mockReset().mockResolvedValue({ unsettled: false, active: 0 });
|
||||
registryRuntimeMock.listSubagentRunsForRequester.mockReset().mockReturnValue([]);
|
||||
deliverSpy.mockReset().mockResolvedValue({ delivered: true, path: "direct" });
|
||||
});
|
||||
|
|
|
|||
|
|
@ -44,3 +44,11 @@ export function readSharedBatchState(
|
|||
deferralCount: Math.max(0, ...states.map((state) => state.deferralCount ?? 0)),
|
||||
};
|
||||
}
|
||||
|
||||
export function retainedYieldIdentity(state: RequesterSettleWakeBatchState) {
|
||||
return {
|
||||
...(state.requesterYieldBatch === true ? { requesterYieldBatch: true as const } : {}),
|
||||
...(state.afterRequesterYield === true ? { afterRequesterYield: true as const } : {}),
|
||||
...(state.rearmGeneration !== undefined ? { rearmGeneration: state.rearmGeneration } : {}),
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import {
|
|||
sessionStore,
|
||||
setSessionStore,
|
||||
registryRuntimeMock,
|
||||
readDescendantFacts,
|
||||
findTranscriptEventMock,
|
||||
listedRequesterRuns,
|
||||
wakeParams,
|
||||
|
|
@ -288,14 +289,14 @@ describe("maybeWakeRequesterAfterAllChildrenSettled", () => {
|
|||
it("does not wake while other children still await settle", async () => {
|
||||
const children = [makeSettledChild({ runId: "run-a" }), makeSettledChild({ runId: "run-b" })];
|
||||
registryRuntimeMock.listSubagentRunsForRequester.mockReturnValue(children);
|
||||
registryRuntimeMock.hasDescendantRunAwaitingSettle.mockReturnValue(true);
|
||||
readDescendantFacts.mockResolvedValue({ unsettled: true, active: 0 });
|
||||
|
||||
const woke = await maybeWakeRequesterAfterAllChildrenSettled(
|
||||
wakeParams({ settledEntry: children[1] }),
|
||||
);
|
||||
|
||||
expect(woke).toBe(false);
|
||||
expect(registryRuntimeMock.hasDescendantRunAwaitingSettle).toHaveBeenCalledOnce();
|
||||
expect(readDescendantFacts).toHaveBeenCalledOnce();
|
||||
expect(transitionBatchSpy).not.toHaveBeenCalled();
|
||||
expect(deliverSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
|
@ -795,10 +796,10 @@ describe("maybeWakeRequesterAfterAllChildrenSettled", () => {
|
|||
const firstChild = makeSettledChild({ runId: "run-a" });
|
||||
const secondChild = makeSettledChild({ runId: "run-b" });
|
||||
registryRuntimeMock.listSubagentRunsForRequester.mockReturnValue([firstChild, secondChild]);
|
||||
registryRuntimeMock.hasDescendantRunAwaitingSettle
|
||||
.mockReturnValueOnce(false)
|
||||
.mockReturnValueOnce(false)
|
||||
.mockReturnValue(true);
|
||||
readDescendantFacts
|
||||
.mockResolvedValueOnce({ unsettled: false, active: 0 })
|
||||
.mockResolvedValueOnce({ unsettled: false, active: 0 })
|
||||
.mockResolvedValue({ unsettled: true, active: 0 });
|
||||
deliverSpy.mockResolvedValueOnce({ delivered: false, path: "direct" });
|
||||
|
||||
vi.useFakeTimers();
|
||||
|
|
@ -923,7 +924,7 @@ describe("maybeWakeRequesterAfterAllChildrenSettled", () => {
|
|||
completion: { required: true, resultText: "beta findings" },
|
||||
});
|
||||
registryRuntimeMock.listSubagentRunsForRequester.mockReturnValue([childA, childB]);
|
||||
registryRuntimeMock.hasDescendantRunAwaitingSettle.mockReturnValue(true);
|
||||
readDescendantFacts.mockResolvedValue({ unsettled: true, active: 0 });
|
||||
|
||||
expect(
|
||||
await maybeWakeRequesterAfterAllChildrenSettled(wakeParams({ settledEntry: childA })),
|
||||
|
|
@ -932,7 +933,7 @@ describe("maybeWakeRequesterAfterAllChildrenSettled", () => {
|
|||
|
||||
// Cold restore rehydrates both retained rows; the final settle drains
|
||||
// the same wave and carries both persisted results.
|
||||
registryRuntimeMock.hasDescendantRunAwaitingSettle.mockReturnValue(false);
|
||||
readDescendantFacts.mockResolvedValue({ unsettled: false, active: 0 });
|
||||
expect(
|
||||
await maybeWakeRequesterAfterAllChildrenSettled(wakeParams({ settledEntry: childB })),
|
||||
).toBe(true);
|
||||
|
|
@ -993,8 +994,7 @@ describe("maybeWakeRequesterAfterAllChildrenSettled", () => {
|
|||
},
|
||||
});
|
||||
registryRuntimeMock.listSubagentRunsForRequester.mockReturnValue([child]);
|
||||
registryRuntimeMock.hasDescendantRunAwaitingSettle.mockReturnValue(true);
|
||||
registryRuntimeMock.countActiveDescendantRuns.mockReturnValue(1);
|
||||
readDescendantFacts.mockResolvedValue({ unsettled: true, active: 1 });
|
||||
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(0);
|
||||
|
|
@ -1006,7 +1006,7 @@ describe("maybeWakeRequesterAfterAllChildrenSettled", () => {
|
|||
|
||||
expect(child.requesterSettleWake?.deferralCount).toBe(0);
|
||||
|
||||
registryRuntimeMock.hasDescendantRunAwaitingSettle.mockReturnValue(false);
|
||||
readDescendantFacts.mockResolvedValue({ unsettled: false, active: 1 });
|
||||
await expect(
|
||||
maybeWakeRequesterAfterAllChildrenSettled(wakeParams({ settledEntry: child })),
|
||||
).resolves.toBe(true);
|
||||
|
|
@ -1019,8 +1019,7 @@ describe("maybeWakeRequesterAfterAllChildrenSettled", () => {
|
|||
rearmGeneration: 1,
|
||||
deferralCount: 8,
|
||||
};
|
||||
registryRuntimeMock.hasDescendantRunAwaitingSettle.mockReturnValue(true);
|
||||
registryRuntimeMock.countActiveDescendantRuns.mockReturnValue(0);
|
||||
readDescendantFacts.mockResolvedValue({ unsettled: true, active: 0 });
|
||||
|
||||
await maybeWakeRequesterAfterAllChildrenSettled(wakeParams({ settledEntry: child }));
|
||||
await maybeWakeRequesterAfterAllChildrenSettled(wakeParams({ settledEntry: child }));
|
||||
|
|
|
|||
|
|
@ -28,10 +28,8 @@ import {
|
|||
selectConnectedSettledSubagentWave,
|
||||
} from "../registry/subagent-registry-queries.js";
|
||||
import {
|
||||
countActiveDescendantRuns,
|
||||
getLatestLiveSubagentRunByChildSessionKey,
|
||||
getLatestSubagentRunByChildSessionKey,
|
||||
hasDescendantRunAwaitingSettle,
|
||||
listSubagentRunsForRequester,
|
||||
} from "../registry/subagent-registry-read.js";
|
||||
import type { SubagentRunRecord } from "../registry/subagent-registry.types.js";
|
||||
|
|
@ -57,9 +55,11 @@ import {
|
|||
readChildCompletionFindings,
|
||||
} from "./subagent-announce-output.js";
|
||||
import { hasUsableSessionEntry } from "./subagent-announce.js";
|
||||
import { createRequesterDescendantReader } from "./subagent-announce.requester-settle-descendants.js";
|
||||
import { buildRequesterSettleWakeMessage } from "./subagent-announce.requester-settle-message.js";
|
||||
import {
|
||||
readSharedBatchState,
|
||||
retainedYieldIdentity,
|
||||
type RequesterSettleWakeBatchState,
|
||||
type RequesterSettleWakeBatchCallbacks,
|
||||
} from "./subagent-announce.requester-settle-state.js";
|
||||
|
|
@ -70,14 +70,6 @@ const REQUESTER_SETTLE_WAKE_MAX_DEFERRALS = 10;
|
|||
const REQUESTER_SETTLE_WAKE_RETRY_DELAYS_MS = [30_000, 120_000] as const;
|
||||
const activeRequesterSettleWakeBatches = new Map<string, () => boolean>();
|
||||
|
||||
function retainedYieldIdentity(state: RequesterSettleWakeBatchState) {
|
||||
return {
|
||||
...(state.requesterYieldBatch === true ? { requesterYieldBatch: true as const } : {}),
|
||||
...(state.afterRequesterYield === true ? { afterRequesterYield: true as const } : {}),
|
||||
...(state.rearmGeneration !== undefined ? { rearmGeneration: state.rearmGeneration } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Wakes a top-level or explicitly yielded nested requester once its batch's last
|
||||
* child and descendants settle. Await lifecycle-owned durable state transitions
|
||||
|
|
@ -217,16 +209,21 @@ export async function maybeWakeRequesterAfterAllChildrenSettled(
|
|||
const batchCreatedAt = Math.min(...settledBatch.map((entry) => entry.createdAt));
|
||||
// Keep the batch members themselves in the settle check, including paused work.
|
||||
const rootRunIds = frozenBatchRunIds?.length ? new Set(frozenBatchRunIds) : undefined;
|
||||
const requesterHasUnsettledDescendants = () =>
|
||||
hasDescendantRunAwaitingSettle(
|
||||
requesterSessionKey,
|
||||
currentSettledEntry.runId,
|
||||
requesterAgentId,
|
||||
requesterStorePath,
|
||||
batchCreatedAt,
|
||||
rootRunIds,
|
||||
);
|
||||
const hasUnsettledDescendants = requesterHasUnsettledDescendants();
|
||||
const readRequesterDescendants = createRequesterDescendantReader({
|
||||
requesterSessionKey,
|
||||
requesterAgentId,
|
||||
requesterStorePath,
|
||||
settledEntry: currentSettledEntry,
|
||||
settledBefore: batchCreatedAt,
|
||||
rootRunIds,
|
||||
signal: params.signal,
|
||||
isSourceCurrent: params.isSourceCurrent,
|
||||
});
|
||||
const initialDescendants = await readRequesterDescendants();
|
||||
if (!initialDescendants) {
|
||||
return false;
|
||||
}
|
||||
const hasUnsettledDescendants = initialDescendants.unsettled;
|
||||
if ((!frozenBatchRunIds || frozenBatchRunIds.length === 0) && hasUnsettledDescendants) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -291,13 +288,16 @@ export async function maybeWakeRequesterAfterAllChildrenSettled(
|
|||
}
|
||||
async function deferBatch(
|
||||
state: RequesterSettleWakeBatchState,
|
||||
countTowardsLimit = countActiveDescendantRuns(
|
||||
requesterSessionKey,
|
||||
requesterAgentId,
|
||||
requesterStorePath,
|
||||
rootRunIds,
|
||||
) === 0,
|
||||
countTowardsLimitOverride?: boolean,
|
||||
): Promise<void> {
|
||||
let countTowardsLimit = countTowardsLimitOverride;
|
||||
if (countTowardsLimit === undefined) {
|
||||
const descendants = await readRequesterDescendants();
|
||||
if (!descendants) {
|
||||
return;
|
||||
}
|
||||
countTowardsLimit = descendants.active === 0;
|
||||
}
|
||||
const now = Date.now();
|
||||
if ((state.nextAttemptAt ?? 0) > now) {
|
||||
return;
|
||||
|
|
@ -455,7 +455,11 @@ export async function maybeWakeRequesterAfterAllChildrenSettled(
|
|||
return false;
|
||||
}
|
||||
// Recheck owned descendants after loading findings and before dispatch.
|
||||
if (requesterHasUnsettledDescendants()) {
|
||||
const currentDescendants = await readRequesterDescendants();
|
||||
if (!currentDescendants) {
|
||||
return false;
|
||||
}
|
||||
if (currentDescendants.unsettled) {
|
||||
await deferBatch(state);
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -51,9 +51,7 @@ const { subagentRegistryRuntimeMock } = vi.hoisted(() => ({
|
|||
subagentRegistryRuntimeMock: {
|
||||
shouldIgnorePostCompletionAnnounceForSession: vi.fn(() => false),
|
||||
isSubagentSessionRunActive: vi.fn(() => true),
|
||||
countActiveDescendantRuns: vi.fn(() => 0),
|
||||
countPendingDescendantRuns: vi.fn(() => 0),
|
||||
hasDescendantRunAwaitingSettle: vi.fn(() => false),
|
||||
getLatestSubagentRunByChildSessionKey: vi.fn(() => undefined),
|
||||
listSubagentRunsForRequester: vi.fn<() => SubagentRunRecord[]>(() => []),
|
||||
replaceSubagentRunAfterSteer: vi.fn(() => true),
|
||||
|
|
@ -294,12 +292,8 @@ describe("subagent announce seam flow", () => {
|
|||
subagentRegistryRuntimeMock.shouldIgnorePostCompletionAnnounceForSession.mockReturnValue(false);
|
||||
subagentRegistryRuntimeMock.isSubagentSessionRunActive.mockReset();
|
||||
subagentRegistryRuntimeMock.isSubagentSessionRunActive.mockReturnValue(true);
|
||||
subagentRegistryRuntimeMock.countActiveDescendantRuns.mockReset();
|
||||
subagentRegistryRuntimeMock.countActiveDescendantRuns.mockReturnValue(0);
|
||||
subagentRegistryRuntimeMock.countPendingDescendantRuns.mockReset();
|
||||
subagentRegistryRuntimeMock.countPendingDescendantRuns.mockReturnValue(0);
|
||||
subagentRegistryRuntimeMock.hasDescendantRunAwaitingSettle.mockReset();
|
||||
subagentRegistryRuntimeMock.hasDescendantRunAwaitingSettle.mockReturnValue(false);
|
||||
subagentRegistryRuntimeMock.listSubagentRunsForRequester.mockReset();
|
||||
subagentRegistryRuntimeMock.listSubagentRunsForRequester.mockReturnValue([]);
|
||||
subagentRegistryRuntimeMock.replaceSubagentRunAfterSteer.mockReset();
|
||||
|
|
|
|||
|
|
@ -181,9 +181,7 @@ vi.mock("./subagent-announce.runtime.js", () => ({
|
|||
waitForEmbeddedAgentRunEndMock(sessionId, timeoutMs),
|
||||
}));
|
||||
vi.mock("../registry/subagent-registry-read.js", () => ({
|
||||
countActiveDescendantRuns: () => 0,
|
||||
countPendingDescendantRuns: () => pendingDescendantRuns,
|
||||
hasDescendantRunAwaitingSettle: () => false,
|
||||
getLatestSubagentRunByChildSessionKey: () => undefined,
|
||||
listSubagentRunsForRequester: () => [],
|
||||
isSubagentSessionRunActive: () => subagentSessionRunActive,
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import {
|
|||
stripLeadingSilentToken,
|
||||
stripSilentToken,
|
||||
} from "../../../auto-reply/tokens.js";
|
||||
import { hasSqliteWorkerOutcomeUnknown } from "../../../infra/sqlite-worker-contract.js";
|
||||
import { logWarn } from "../../../logger.js";
|
||||
import { withPluginRuntimeGatewayContextResolver } from "../../../plugins/runtime/gateway-request-scope.js";
|
||||
import { defaultRuntime } from "../../../runtime.js";
|
||||
|
|
@ -183,7 +184,7 @@ type SubagentAnnounceFlowParams = {
|
|||
signal?: AbortSignal;
|
||||
bestEffortDeliver?: boolean;
|
||||
onDeliveryResult?: (delivery: SubagentAnnounceDeliveryResult) => void | Promise<void>;
|
||||
onBeforeDeleteChildSession?: () => boolean;
|
||||
onBeforeDeleteChildSession?: () => boolean | Promise<boolean>;
|
||||
resolveGatewayContext?: import("../../../gateway/server-methods/types.js").GatewayContextResolver;
|
||||
};
|
||||
|
||||
|
|
@ -304,7 +305,14 @@ async function runSubagentAnnounceFlowBound(
|
|||
const pendingChildDescendantRuns =
|
||||
!childSessionCurrent || !childSessionEffectsAllowed()
|
||||
? 0
|
||||
: Math.max(0, countPendingDescendantRuns(params.childSessionKey));
|
||||
: Math.max(
|
||||
0,
|
||||
await countPendingDescendantRuns(params.childSessionKey, () => {
|
||||
if (!childSessionEffectsAllowed()) {
|
||||
throw new Error("Subagent child-session effects are no longer current.");
|
||||
}
|
||||
}),
|
||||
);
|
||||
if (pendingChildDescendantRuns > 0 && announceType !== "cron job") {
|
||||
shouldDeleteChildSession = false;
|
||||
return "retryable";
|
||||
|
|
@ -690,6 +698,9 @@ async function runSubagentAnnounceFlowBound(
|
|||
: (delivery.disposition ?? (delivery.delivered ? "delivered" : "retryable"));
|
||||
} catch (err) {
|
||||
shouldDeleteChildSession = false;
|
||||
if (hasSqliteWorkerOutcomeUnknown(err)) {
|
||||
throw err;
|
||||
}
|
||||
defaultRuntime.error?.(`Subagent announce failed: ${String(err)}`);
|
||||
// Best-effort follow-ups; ignore failures to avoid breaking the caller response.
|
||||
} finally {
|
||||
|
|
@ -697,7 +708,8 @@ async function runSubagentAnnounceFlowBound(
|
|||
shouldDeleteChildSession &&
|
||||
(await prepareChildSessionEffects()) &&
|
||||
childSessionEffectsAllowed() &&
|
||||
(params.onBeforeDeleteChildSession?.() ?? true)
|
||||
((await params.onBeforeDeleteChildSession?.()) ?? true) &&
|
||||
childSessionEffectsAllowed()
|
||||
) {
|
||||
await deleteSubagentSessionForCleanup({
|
||||
callGateway: callSubagentLifecycleGateway,
|
||||
|
|
|
|||
|
|
@ -453,8 +453,8 @@ export async function runWithLiveSubagentGateway(
|
|||
);
|
||||
record("parent-final-observed", { sessionKey, expectedMarker: marker, reply: firstFinal });
|
||||
expect(firstFinal, `first parent completion for ${marker}`).toBe(expected);
|
||||
await until("descendant settlement", () =>
|
||||
countPendingDescendantRuns(sessionKey) === 0 ? true : undefined,
|
||||
await until("descendant settlement", async () =>
|
||||
(await countPendingDescendantRuns(sessionKey, () => {})) === 0 ? true : undefined,
|
||||
);
|
||||
const messages = await history(sessionKey);
|
||||
expect(
|
||||
|
|
|
|||
|
|
@ -471,7 +471,7 @@ describeLive("OpenAI subagent yield and operator resume stress", () => {
|
|||
record("external-wait", {
|
||||
sessionKey: parentKey,
|
||||
gate: resumeGate.snapshot(),
|
||||
pending: countPendingDescendantRuns(parentKey),
|
||||
pending: await countPendingDescendantRuns(parentKey, () => {}),
|
||||
});
|
||||
const originalTaskRunId = paused.taskRunId ?? paused.runId;
|
||||
const originalGeneration = paused.generation ?? 0;
|
||||
|
|
@ -752,8 +752,8 @@ describeLive("OpenAI subagent yield and operator resume stress", () => {
|
|||
expect(reply, "late cancelled stdout is not a delivered result").not.toContain(
|
||||
lateResult,
|
||||
);
|
||||
await until("all child obligations settled", () =>
|
||||
countPendingDescendantRuns(parentKey) === 0 ? true : undefined,
|
||||
await until("all child obligations settled", async () =>
|
||||
(await countPendingDescendantRuns(parentKey, () => {})) === 0 ? true : undefined,
|
||||
);
|
||||
const finalMessages = await history(parentKey);
|
||||
expect(finalReplies(finalMessages, marker)).toHaveLength(1);
|
||||
|
|
|
|||
|
|
@ -109,7 +109,7 @@ export function requesterWakeDriver(inputs: ReturnType<typeof records>[]) {
|
|||
persistAsyncOrThrow: (context, callbacks, ...runIds) =>
|
||||
persistSubagentRunsToDiskAsyncOrThrow(subagentRuns, runIds, { context, ...callbacks }),
|
||||
clearPendingLifecycleError: vi.fn(),
|
||||
countPendingDescendantRuns: () => 0,
|
||||
countPendingDescendantRuns: async () => 0,
|
||||
getLatestRunForChildSession: getLatestLiveSubagentRunByChildSessionKey,
|
||||
suppressAnnounceForSteerRestart: () => false,
|
||||
shouldEmitEndedHookForRun: () => false,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,683 @@
|
|||
// Preserve fixture setup before production consumers.
|
||||
// oxfmt-ignore
|
||||
import { useSubagentControlFixture } from "./subagent-control.test-support.js";
|
||||
import path from "node:path";
|
||||
import { beforeEach, expect, it, vi } from "vitest";
|
||||
import {
|
||||
emptySqliteCounts,
|
||||
observeParentSqlite,
|
||||
} from "../../../../test/helpers/sqlite-parent-observer.js";
|
||||
import { getRuntimeConfig } from "../../../config/config.js";
|
||||
import { WRITE_SCOPE } from "../../../gateway/method-scopes.js";
|
||||
import { createGatewayMethodRegistry } from "../../../gateway/methods/registry.js";
|
||||
import { createContext } from "../../../gateway/server-plugin-in-process-dispatch.test-support.js";
|
||||
import { emitAgentEvent } from "../../../infra/agent-events.js";
|
||||
import { SqliteWorkerError } from "../../../infra/sqlite-worker-contract.js";
|
||||
import * as hookRuntime from "../../../plugins/hook-runner-global.js";
|
||||
import { createHookRunnerWithRegistry } from "../../../plugins/hooks.test-fixtures.js";
|
||||
import { createDeferredCore as createDeferred } from "../../../shared/deferred.js";
|
||||
import { closeOpenClawStateDatabaseAsync } from "../../../state/openclaw-state-db-cache.js";
|
||||
import { openOpenClawStateDatabase } from "../../../state/openclaw-state-db.js";
|
||||
import { captureOpenClawStateWorkerContext } from "../../../state/openclaw-state-worker-context.js";
|
||||
import * as stateWorker from "../../../state/openclaw-state-worker-store.js";
|
||||
import { setTestEnvValue, withEnvAsync } from "../../../test-utils/env.js";
|
||||
import { loadAgentRuntimePluginRegistryHandle } from "../../runtime-plugins.js";
|
||||
import { subagentRuns } from "./subagent-registry-memory.js";
|
||||
import * as registryPersistence from "./subagent-registry-persistence.js";
|
||||
import * as registryReads from "./subagent-registry-read-cache.js";
|
||||
import * as registryRead from "./subagent-registry-read.js";
|
||||
import * as registryState from "./subagent-registry-state.js";
|
||||
import {
|
||||
prepareSubagentSessionCleanupRevocation,
|
||||
registerSubagentRun,
|
||||
resumeSubagentRun,
|
||||
} from "./subagent-registry.js";
|
||||
import { writeSubagentSessionEntry } from "./subagent-registry.persistence.test-support.js";
|
||||
import { loadSubagentRegistryFromSqlite } from "./subagent-registry.store.sqlite.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
|
||||
vi.mock("../../../state/openclaw-state-worker-store.js", { spy: true });
|
||||
vi.mock("./subagent-registry-read-cache.js", { spy: true });
|
||||
vi.mock("./subagent-registry-read.js", { spy: true });
|
||||
vi.mock("../../../plugins/hook-runner-global.js", { spy: true });
|
||||
type AgentTurnService = ReturnType<
|
||||
(typeof import("../../../gateway/agent-turn/agent-turn-service.js"))["createAgentTurnService"]
|
||||
>;
|
||||
const turns = vi.hoisted(() => ({
|
||||
start: vi.fn<AgentTurnService["startTurn"]>(),
|
||||
wait: vi.fn<AgentTurnService["waitForTurn"]>(),
|
||||
}));
|
||||
vi.mock("../../../gateway/agent-turn/agent-turn-service.js", () => ({
|
||||
createAgentTurnService: () => ({ startTurn: turns.start, waitForTurn: turns.wait }),
|
||||
}));
|
||||
|
||||
const fixture = useSubagentControlFixture();
|
||||
const nativeState = await vi.importActual<typeof registryState>("./subagent-registry-state.js");
|
||||
const nativeAnnounce = await vi.importActual<typeof import("../announce/subagent-announce.js")>(
|
||||
"../announce/subagent-announce.js",
|
||||
);
|
||||
const nativeWorker = await vi.importActual<typeof stateWorker>(
|
||||
"../../../state/openclaw-state-worker-store.js",
|
||||
);
|
||||
const nativeReads = await vi.importActual<typeof registryReads>(
|
||||
"./subagent-registry-read-cache.js",
|
||||
);
|
||||
const nativeRegistryRead = await vi.importActual<typeof registryRead>(
|
||||
"./subagent-registry-read.js",
|
||||
);
|
||||
const nativeWake = await vi.importActual<
|
||||
typeof import("../announce/subagent-announce.requester-settle-wake.js")
|
||||
>("../announce/subagent-announce.requester-settle-wake.js");
|
||||
beforeEach(() => {
|
||||
turns.start.mockReset().mockImplementation(async ({ io, assertAdmissionCurrent }) => {
|
||||
assertAdmissionCurrent?.();
|
||||
io.emitAcceptance([
|
||||
true,
|
||||
{
|
||||
runId: "synthetic-requester-turn",
|
||||
status: "ok",
|
||||
result: { payloads: [{ text: "Reviewed synthetic child." }] },
|
||||
},
|
||||
undefined,
|
||||
]);
|
||||
});
|
||||
turns.wait.mockReset();
|
||||
});
|
||||
|
||||
async function registerCompletion(
|
||||
runId: string,
|
||||
options: {
|
||||
cleanup?: "keep" | "delete";
|
||||
expectsCompletionMessage?: boolean;
|
||||
requesterSessionKey?: string;
|
||||
} = {},
|
||||
) {
|
||||
const childSessionKey = `agent:main:subagent:${runId}`;
|
||||
await writeSubagentSessionEntry({
|
||||
stateDir: fixture.stateDir,
|
||||
agentId: "main",
|
||||
sessionKey: childSessionKey,
|
||||
defaultSessionId: `${runId}-session`,
|
||||
lifecycleRevision: "original-child",
|
||||
});
|
||||
await writeSubagentSessionEntry({
|
||||
stateDir: fixture.stateDir,
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
defaultSessionId: "synthetic-requester-session",
|
||||
lifecycleRevision: "original-requester",
|
||||
});
|
||||
fixture.capture.mockResolvedValue("Synthetic completed result.");
|
||||
const context = createContext();
|
||||
context.localEmbedded = true;
|
||||
context.getRuntimeConfig = getRuntimeConfig;
|
||||
context.chatAbortControllers = new Map();
|
||||
context.chatQueuedTurns = new Map();
|
||||
const registry = createGatewayMethodRegistry([
|
||||
{
|
||||
name: "agent",
|
||||
owner: { kind: "core", area: "test" },
|
||||
scope: WRITE_SCOPE,
|
||||
handler: () => {
|
||||
throw new Error("Expected the typed agent turn facade");
|
||||
},
|
||||
},
|
||||
]);
|
||||
context.getGatewayMethodRegistry = () => registry;
|
||||
await registerSubagentRun({
|
||||
runId,
|
||||
childSessionKey,
|
||||
requesterSessionKey: options.requesterSessionKey ?? "agent:main:main",
|
||||
requesterAgentId: "main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "retain ordinary cleanup before requester settlement",
|
||||
cleanup: options.cleanup ?? "keep",
|
||||
expectsCompletionMessage: options.expectsCompletionMessage ?? true,
|
||||
gatewayContextResolver: () => context,
|
||||
});
|
||||
vi.mocked(registryState.persistSubagentRunsToDiskAsyncOrThrow).mockImplementation(
|
||||
nativeState.persistSubagentRunsToDiskAsyncOrThrow,
|
||||
);
|
||||
return { runId, childSessionKey };
|
||||
}
|
||||
|
||||
function completeRegistered(
|
||||
run: { runId: string; childSessionKey: string },
|
||||
text: string | undefined = "Synthetic completed result.",
|
||||
) {
|
||||
emitAgentEvent({
|
||||
runId: run.runId,
|
||||
sessionKey: run.childSessionKey,
|
||||
stream: "lifecycle",
|
||||
data: {
|
||||
phase: "end",
|
||||
endedAt: Date.now(),
|
||||
...(text ? { terminalReply: { disposition: "visible", text } } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
it.each(["announce receipt delete", "announce receipt keep"] as const)(
|
||||
"persists registered %s cleanup through the worker before handing off requester settlement",
|
||||
async (mode) => {
|
||||
const deleting = mode.endsWith("delete");
|
||||
const run = await registerCompletion(`ordinary-${mode.replaceAll(" ", "-")}`, {
|
||||
cleanup: deleting ? "delete" : "keep",
|
||||
});
|
||||
const { runId } = run;
|
||||
const announceDone =
|
||||
createDeferred<Awaited<ReturnType<typeof nativeAnnounce.runSubagentAnnounceFlow>>>();
|
||||
let deliveryAllowed: (() => boolean) | undefined;
|
||||
const publicationAdmission: boolean[] = [];
|
||||
const stopObserving = nativeState.onSubagentRegistryPersisted(() => {
|
||||
const current = subagentRuns.get(runId);
|
||||
if (
|
||||
deliveryAllowed &&
|
||||
current?.delivery?.status === "delivered" &&
|
||||
current.cleanupCompletedAt === undefined
|
||||
) {
|
||||
publicationAdmission.push(deliveryAllowed());
|
||||
}
|
||||
});
|
||||
fixture.announce.mockImplementation(async (params) => {
|
||||
deliveryAllowed = params.isCompletionDeliveryAllowed;
|
||||
try {
|
||||
const outcome = await nativeAnnounce.runSubagentAnnounceFlow(params);
|
||||
announceDone.resolve(outcome);
|
||||
return outcome;
|
||||
} catch (error) {
|
||||
announceDone.reject(error);
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
|
||||
let deleted = false;
|
||||
let deleteDispatchedAt: number | undefined;
|
||||
fixture.gateway.mockImplementation(async (request) => {
|
||||
if (request.method === "agent.wait") {
|
||||
return await new Promise<never>(() => {});
|
||||
}
|
||||
if (request.method === "chat.history") {
|
||||
return {
|
||||
messages: [
|
||||
{
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: "Synthetic completed result." }],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (request.method === "agent") {
|
||||
return {
|
||||
runId: "synthetic-requester-turn",
|
||||
status: "ok",
|
||||
result: { payloads: [{ text: "Reviewed synthetic child." }] },
|
||||
};
|
||||
}
|
||||
if (request.method !== "sessions.delete") {
|
||||
throw new Error(`Unexpected completion RPC ${request.method}`);
|
||||
}
|
||||
deleteDispatchedAt = loadSubagentRegistryFromSqlite().get(runId)?.deleteCleanupDispatchedAt;
|
||||
request.assertDispatchCurrent?.();
|
||||
deleted = true;
|
||||
return {};
|
||||
});
|
||||
let settledRecord: SubagentRunRecord | undefined;
|
||||
fixture.wake.mockImplementation(async () => {
|
||||
settledRecord = loadSubagentRegistryFromSqlite().get(runId);
|
||||
return false;
|
||||
});
|
||||
try {
|
||||
completeRegistered(run);
|
||||
const outcome = await announceDone.promise;
|
||||
expect(
|
||||
outcome,
|
||||
JSON.stringify({
|
||||
methods: fixture.gateway.mock.calls.map(([request]) => request.method),
|
||||
error: subagentRuns.get(runId)?.delivery?.lastError,
|
||||
}),
|
||||
).toBe("delivered");
|
||||
await fixture.settle();
|
||||
expect(fixture.wake).toHaveBeenCalledOnce();
|
||||
expect(deleted).toBe(deleting);
|
||||
if (deleting) {
|
||||
expect(deleteDispatchedAt).toBeTypeOf("number");
|
||||
}
|
||||
expect(settledRecord).toMatchObject({
|
||||
cleanupHandled: true,
|
||||
cleanupCompletedAt: expect.any(Number),
|
||||
execution: { status: "terminal", outcome: { status: "ok" } },
|
||||
});
|
||||
expect(fixture.announce).toHaveBeenCalledOnce();
|
||||
expect(turns.start).toHaveBeenCalledOnce();
|
||||
expect(publicationAdmission.length).toBeGreaterThan(0);
|
||||
expect(publicationAdmission.every(Boolean), JSON.stringify(publicationAdmission)).toBe(true);
|
||||
expect(settledRecord?.delivery).toMatchObject({
|
||||
status: "delivered",
|
||||
deliveredAt: expect.any(Number),
|
||||
});
|
||||
} finally {
|
||||
stopObserving();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("defers registered parent delivery until its descendant settles without host descendant reads", async () => {
|
||||
await withEnvAsync({ OPENCLAW_TEST_READ_SUBAGENT_RUNS_FROM_SQLITE: "1" }, async () => {
|
||||
const parent = await registerCompletion("ordinary-deferred-parent");
|
||||
const child = await registerCompletion("ordinary-deferred-child", {
|
||||
requesterSessionKey: parent.childSessionKey,
|
||||
expectsCompletionMessage: false,
|
||||
});
|
||||
const originalParent = subagentRuns.get(parent.runId)!;
|
||||
const continuationRunId = "deferred-parent-continuation";
|
||||
turns.start.mockImplementation(async ({ io, assertAdmissionCurrent, preflight }) => {
|
||||
assertAdmissionCurrent?.();
|
||||
io.emitAcceptance([
|
||||
true,
|
||||
{
|
||||
runId:
|
||||
preflight.request.sessionKey === parent.childSessionKey
|
||||
? continuationRunId
|
||||
: "deferred-parent-outward-delivery",
|
||||
status: "ok",
|
||||
result: { payloads: [{ text: "Reviewed synthetic child." }] },
|
||||
},
|
||||
undefined,
|
||||
]);
|
||||
});
|
||||
const reads: { phase: "announce" | "cleanup"; counts: ReturnType<typeof emptySqliteCounts> }[] =
|
||||
[];
|
||||
let phase: "announce" | "cleanup" = "cleanup";
|
||||
fixture.wake.mockResolvedValue(false);
|
||||
fixture.announce.mockImplementation(async (params) => {
|
||||
phase = "announce";
|
||||
try {
|
||||
return await nativeAnnounce.runSubagentAnnounceFlow(params);
|
||||
} finally {
|
||||
phase = "cleanup";
|
||||
}
|
||||
});
|
||||
function observeRead<T>(read: () => T): T {
|
||||
const selectedPhase = phase;
|
||||
// Only the actual descendant read is measured here; transport and fixture
|
||||
// writes remain outside this window. Preserve sync/async return semantics for RED.
|
||||
const sql = observeParentSqlite();
|
||||
const record = () => {
|
||||
reads.push({ phase: selectedPhase, counts: { ...sql.counts } });
|
||||
sql.restore();
|
||||
};
|
||||
try {
|
||||
const result = read();
|
||||
if (result instanceof Promise) {
|
||||
void result.then(record, record);
|
||||
} else {
|
||||
record();
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
record();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
const count = vi
|
||||
.spyOn(registryRead, "countPendingDescendantRuns")
|
||||
.mockImplementation((...args) =>
|
||||
observeRead(() => nativeRegistryRead.countPendingDescendantRuns(...args)),
|
||||
);
|
||||
try {
|
||||
registryState.clearSubagentRunsReadCacheForTest();
|
||||
completeRegistered(parent);
|
||||
await fixture.settle();
|
||||
const deferred = loadSubagentRegistryFromSqlite().get(parent.runId);
|
||||
expect(deferred).toMatchObject({
|
||||
cleanupHandled: false,
|
||||
wakeOnDescendantSettle: true,
|
||||
execution: { status: "terminal", outcome: { status: "ok" } },
|
||||
});
|
||||
expect(deferred?.cleanupCompletedAt).toBeUndefined();
|
||||
expect(turns.start).not.toHaveBeenCalled();
|
||||
expect(fixture.wake).not.toHaveBeenCalled();
|
||||
expect(reads.map((read) => read.phase)).toEqual(
|
||||
expect.arrayContaining(["announce", "cleanup"]),
|
||||
);
|
||||
|
||||
for (const read of reads) {
|
||||
expect(read.counts, read.phase).toEqual(emptySqliteCounts());
|
||||
}
|
||||
|
||||
completeRegistered(child);
|
||||
await fixture.settle();
|
||||
expect(turns.start).toHaveBeenCalledOnce();
|
||||
expect(turns.start.mock.calls[0]?.[0].preflight.request.sessionKey).toBe(
|
||||
parent.childSessionKey,
|
||||
);
|
||||
const afterWake = loadSubagentRegistryFromSqlite();
|
||||
expect(afterWake.has(parent.runId)).toBe(false);
|
||||
expect(subagentRuns.has(parent.runId)).toBe(false);
|
||||
expect(afterWake.get(continuationRunId)).toMatchObject({
|
||||
taskRunId: parent.runId,
|
||||
childSessionKey: parent.childSessionKey,
|
||||
requesterSessionKey: "agent:main:main",
|
||||
execution: { status: "running" },
|
||||
});
|
||||
expect(afterWake.get(continuationRunId)!.generation).toBeGreaterThan(
|
||||
originalParent.generation!,
|
||||
);
|
||||
completeRegistered({ runId: continuationRunId, childSessionKey: parent.childSessionKey });
|
||||
await fixture.settle();
|
||||
expect(
|
||||
turns.start.mock.calls.map(([request]) => request.preflight.request.sessionKey),
|
||||
).toEqual([parent.childSessionKey, "agent:main:main"]);
|
||||
expect(loadSubagentRegistryFromSqlite().get(continuationRunId)).toMatchObject({
|
||||
cleanupCompletedAt: expect.any(Number),
|
||||
delivery: { status: "delivered", deliveredAt: expect.any(Number) },
|
||||
});
|
||||
expect(loadSubagentRegistryFromSqlite().get(child.runId)?.cleanupCompletedAt).toBeTypeOf(
|
||||
"number",
|
||||
);
|
||||
} finally {
|
||||
count.mockRestore();
|
||||
await fixture.settle();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps a delivered announcement fenced when its native receipt acknowledgement is lost", async () => {
|
||||
const run = await registerCompletion("uncertain-announcement-receipt", {
|
||||
cleanup: "delete",
|
||||
});
|
||||
const entry = subagentRuns.get(run.runId)!;
|
||||
const completed = createDeferred<
|
||||
{ kind: "failed"; error: unknown } | { kind: "returned"; value: string }
|
||||
>();
|
||||
let deliveryAllowed: (() => boolean) | undefined;
|
||||
fixture.announce.mockImplementation(async (params) => {
|
||||
deliveryAllowed = params.isCompletionDeliveryAllowed;
|
||||
try {
|
||||
const value = await nativeAnnounce.runSubagentAnnounceFlow(params);
|
||||
completed.resolve({ kind: "returned", value });
|
||||
return value;
|
||||
} catch (error) {
|
||||
completed.resolve({ kind: "failed", error });
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
let lost = false;
|
||||
const worker = vi
|
||||
.spyOn(stateWorker, "runOpenClawStateWorkerOperation")
|
||||
.mockImplementation((stateContext, operation, options) => {
|
||||
const receiptWrite =
|
||||
!lost && entry.delivery?.status === "delivered" && entry.cleanupCompletedAt === undefined;
|
||||
return nativeWorker.runOpenClawStateWorkerOperation(
|
||||
stateContext,
|
||||
(scope) =>
|
||||
operation({
|
||||
async execute(command, executeOptions) {
|
||||
const result = await scope.execute(command, executeOptions);
|
||||
if (receiptWrite && command.type === "subagents.persistChanges") {
|
||||
lost = true;
|
||||
throw new SqliteWorkerError(
|
||||
"Synthetic lost delivery receipt acknowledgement",
|
||||
"outcome-unknown",
|
||||
);
|
||||
}
|
||||
return result;
|
||||
},
|
||||
}),
|
||||
options,
|
||||
);
|
||||
});
|
||||
try {
|
||||
completeRegistered(run);
|
||||
const result = await completed.promise;
|
||||
expect(result.kind).toBe("failed");
|
||||
if (result.kind !== "failed") {
|
||||
throw new Error(`Expected uncertain persistence, received ${result.value}`);
|
||||
}
|
||||
expect(result.error).toMatchObject({ outcome: "unknown" });
|
||||
if (!(result.error instanceof Error)) {
|
||||
throw new Error("Expected the retained registry write failure");
|
||||
}
|
||||
await fixture.settle();
|
||||
expect(lost).toBe(true);
|
||||
expect(loadSubagentRegistryFromSqlite().get(run.runId)?.delivery?.status).toBe("delivered");
|
||||
expect(entry.delivery?.status).toBe("pending");
|
||||
expect(entry.cleanupHandled).toBe(true);
|
||||
expect(deliveryAllowed).toBeTypeOf("function");
|
||||
expect(() => deliveryAllowed?.()).toThrow(result.error);
|
||||
resumeSubagentRun(run.runId);
|
||||
await fixture.settle();
|
||||
expect(turns.start).toHaveBeenCalledOnce();
|
||||
expect(fixture.announce).toHaveBeenCalledOnce();
|
||||
expect(fixture.wake).not.toHaveBeenCalled();
|
||||
expect(
|
||||
fixture.gateway.mock.calls.some(([request]) => request.method === "sessions.delete"),
|
||||
).toBe(false);
|
||||
} finally {
|
||||
worker.mockRestore();
|
||||
await fixture.settle();
|
||||
await closeOpenClawStateDatabaseAsync();
|
||||
await nativeState.restoreSubagentRunsFromDisk({ runs: subagentRuns });
|
||||
}
|
||||
});
|
||||
|
||||
it("refuses an old registered wake after its descendant read outlives a successor", async () => {
|
||||
await withEnvAsync({ OPENCLAW_TEST_READ_SUBAGENT_RUNS_FROM_SQLITE: "1" }, async () => {
|
||||
const run = await registerCompletion("held-descendant-read", {
|
||||
expectsCompletionMessage: false,
|
||||
});
|
||||
const original = subagentRuns.get(run.runId)!;
|
||||
const ready = createDeferred();
|
||||
const release = createDeferred();
|
||||
const mutations: { entry: SubagentRunRecord; kind: "transition" | "complete" }[] = [];
|
||||
fixture.wake.mockImplementation(async (params) => {
|
||||
registryState.clearSubagentRunsReadCacheForTest();
|
||||
return nativeWake.maybeWakeRequesterAfterAllChildrenSettled({
|
||||
...params,
|
||||
transitionBatch: async (...args) => {
|
||||
mutations.push({ entry: params.settledEntry, kind: "transition" });
|
||||
return params.transitionBatch(...args);
|
||||
},
|
||||
completeBatch: async (...args) => {
|
||||
mutations.push({ entry: params.settledEntry, kind: "complete" });
|
||||
return params.completeBatch(...args);
|
||||
},
|
||||
});
|
||||
});
|
||||
let held = false;
|
||||
const reader = vi
|
||||
.spyOn(registryReads, "readCompactSubagentRuns")
|
||||
.mockImplementation(async (context) => {
|
||||
const result = await nativeReads.readCompactSubagentRuns(context);
|
||||
if (!held && original.cleanupCompletedAt !== undefined) {
|
||||
held = true;
|
||||
// The actual read has released its broker slot before successor registration.
|
||||
ready.resolve();
|
||||
await release.promise;
|
||||
}
|
||||
return result;
|
||||
});
|
||||
try {
|
||||
completeRegistered(run);
|
||||
await ready.promise;
|
||||
expect(loadSubagentRegistryFromSqlite().get(run.runId)?.requesterSettleWake).toBeDefined();
|
||||
await registerSubagentRun({
|
||||
...run,
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterAgentId: "main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "legitimate successor after the held wake",
|
||||
cleanup: "keep",
|
||||
expectsCompletionMessage: false,
|
||||
});
|
||||
const successor = subagentRuns.get(run.runId)!;
|
||||
expect(successor).not.toBe(original);
|
||||
const durableSuccessor = loadSubagentRegistryFromSqlite().get(run.runId);
|
||||
expect(durableSuccessor?.execution.status).toBe("running");
|
||||
release.resolve();
|
||||
await fixture.settle();
|
||||
expect(mutations).toEqual([]);
|
||||
expect(subagentRuns.get(run.runId)).toBe(successor);
|
||||
expect(loadSubagentRegistryFromSqlite().get(run.runId)).toEqual(durableSuccessor);
|
||||
|
||||
completeRegistered(run);
|
||||
await fixture.settle();
|
||||
expect(mutations).toEqual([{ entry: successor, kind: "complete" }]);
|
||||
const completed = loadSubagentRegistryFromSqlite().get(run.runId);
|
||||
expect(completed).toMatchObject({
|
||||
task: "legitimate successor after the held wake",
|
||||
execution: { status: "terminal", outcome: { status: "ok" } },
|
||||
cleanupCompletedAt: expect.any(Number),
|
||||
});
|
||||
expect(completed?.requesterSettleWake).toBeUndefined();
|
||||
expect(fixture.announce).not.toHaveBeenCalled();
|
||||
expect(turns.start).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
release.resolve();
|
||||
await fixture.settle();
|
||||
reader.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it.for(["current", "successor", "revoked", "source switched"] as const)(
|
||||
"retains ended-hook authority through a registered requester-wake acknowledgement (%s)",
|
||||
async (change, { signal }) => {
|
||||
const run = await registerCompletion("hook-wake-ack");
|
||||
const original = subagentRuns.get(run.runId)!;
|
||||
const revoke = await prepareSubagentSessionCleanupRevocation(run.childSessionKey);
|
||||
const source = captureOpenClawStateWorkerContext();
|
||||
const originalDatabase = openOpenClawStateDatabase({ path: source.admission.databasePath });
|
||||
const replacementDir = path.join(fixture.stateDir, "replacement-state");
|
||||
if (change === "source switched") {
|
||||
openOpenClawStateDatabase({
|
||||
path: path.join(replacementDir, path.basename(source.admission.databasePath)),
|
||||
});
|
||||
}
|
||||
fixture.announce.mockImplementation(nativeAnnounce.runSubagentAnnounceFlow);
|
||||
const pluginEntered = createDeferred();
|
||||
const mutationReady = createDeferred();
|
||||
const stampWaiting = createDeferred();
|
||||
const release = createDeferred();
|
||||
fixture.wake.mockImplementation(async (params) => {
|
||||
await pluginEntered.promise;
|
||||
return nativeWake.maybeWakeRequesterAfterAllChildrenSettled(params);
|
||||
});
|
||||
const ended = vi.fn(async () => {
|
||||
pluginEntered.resolve();
|
||||
await mutationReady.promise;
|
||||
});
|
||||
const releaseCancelledTest = () => {
|
||||
pluginEntered.resolve();
|
||||
mutationReady.resolve();
|
||||
release.resolve();
|
||||
stampWaiting.reject(signal.reason);
|
||||
};
|
||||
signal.addEventListener("abort", releaseCancelledTest, { once: true });
|
||||
const { registry, runner } = createHookRunnerWithRegistry([
|
||||
{ hookName: "subagent_ended", handler: ended },
|
||||
]);
|
||||
vi.mocked(loadAgentRuntimePluginRegistryHandle).mockReturnValue(registry);
|
||||
vi.spyOn(hookRuntime, "getGlobalHookRunner").mockReturnValue(runner);
|
||||
const waitForPending = registryPersistence.waitForPendingSubagentRegistryWrites;
|
||||
const waiting = vi
|
||||
.spyOn(registryPersistence, "waitForPendingSubagentRegistryWrites")
|
||||
.mockImplementation((...args) => {
|
||||
const pending = waitForPending(...args);
|
||||
if (ended.mock.calls.length > 0 && args[0].includes(run.runId)) {
|
||||
if (pending) {
|
||||
stampWaiting.resolve();
|
||||
} else {
|
||||
stampWaiting.reject(new Error("Stamp found no pending wake at held acknowledgement"));
|
||||
}
|
||||
}
|
||||
return pending;
|
||||
});
|
||||
let held = false;
|
||||
const worker = vi
|
||||
.spyOn(stateWorker, "runOpenClawStateWorkerOperation")
|
||||
.mockImplementation((context, operation, options) =>
|
||||
nativeWorker.runOpenClawStateWorkerOperation(
|
||||
context,
|
||||
(scope) =>
|
||||
operation({
|
||||
async execute(command, executeOptions) {
|
||||
const result = await scope.execute(command, executeOptions);
|
||||
if (
|
||||
!held &&
|
||||
command.type === "sessionDelivery.mutateSubagentCompletion" &&
|
||||
original.cleanupCompletedAt !== undefined
|
||||
) {
|
||||
held = true;
|
||||
// The actual wake has committed; its pending-write owner retains the ACK.
|
||||
mutationReady.resolve();
|
||||
await release.promise;
|
||||
}
|
||||
return result;
|
||||
},
|
||||
}),
|
||||
options,
|
||||
),
|
||||
);
|
||||
try {
|
||||
completeRegistered(run);
|
||||
await stampWaiting.promise;
|
||||
expect(ended).toHaveBeenCalledOnce();
|
||||
expect(original.endedHookEmittedAt).toBeUndefined();
|
||||
const committed = loadSubagentRegistryFromSqlite(originalDatabase).get(run.runId);
|
||||
expect(committed?.cleanupCompletedAt).toBeTypeOf("number");
|
||||
expect(committed?.requesterSettleWake).toBeUndefined();
|
||||
let successor: SubagentRunRecord | undefined;
|
||||
if (change === "successor") {
|
||||
await registerSubagentRun({
|
||||
...run,
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterAgentId: "main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "successor after observed hook",
|
||||
cleanup: "keep",
|
||||
expectsCompletionMessage: false,
|
||||
});
|
||||
successor = subagentRuns.get(run.runId)!;
|
||||
expect(successor).not.toBe(original);
|
||||
} else if (change === "revoked") {
|
||||
revoke();
|
||||
expect(original.execution.suppressSessionEffects).toBe(true);
|
||||
} else if (change === "source switched") {
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", replacementDir);
|
||||
}
|
||||
release.resolve();
|
||||
await fixture.settle();
|
||||
const stored = loadSubagentRegistryFromSqlite(originalDatabase).get(run.runId);
|
||||
expect(ended).toHaveBeenCalledOnce();
|
||||
expect(fixture.announce).toHaveBeenCalledOnce();
|
||||
expect(turns.start).toHaveBeenCalledOnce();
|
||||
if (change === "current") {
|
||||
expect(stored?.endedHookEmittedAt).toBeTypeOf("number");
|
||||
expect(stored?.requesterSettleWake).toBeUndefined();
|
||||
resumeSubagentRun(run.runId);
|
||||
await fixture.settle();
|
||||
expect(ended).toHaveBeenCalledOnce();
|
||||
} else {
|
||||
expect(original.endedHookEmittedAt).toBeUndefined();
|
||||
expect(stored?.endedHookEmittedAt).toBeUndefined();
|
||||
if (successor) {
|
||||
expect(subagentRuns.get(run.runId)).toBe(successor);
|
||||
expect(stored).toMatchObject({
|
||||
task: "successor after observed hook",
|
||||
generation: successor.generation,
|
||||
execution: { status: "running" },
|
||||
});
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
release.resolve();
|
||||
await fixture.settle();
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", fixture.stateDir);
|
||||
worker.mockRestore();
|
||||
waiting.mockRestore();
|
||||
signal.removeEventListener("abort", releaseCancelledTest);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
|
@ -0,0 +1,722 @@
|
|||
// Preserve fixture setup before production consumers.
|
||||
// oxfmt-ignore
|
||||
import { useSubagentControlFixture } from "./subagent-control.test-support.js";
|
||||
import { rename } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { observeHostDataSql } from "../../../../test/helpers/sqlite-statement-execution-counter.js";
|
||||
import { getRuntimeConfig } from "../../../config/config.js";
|
||||
import { captureGatewayOperatorRunAuthority } from "../../../gateway/operator-run-authority.js";
|
||||
import {
|
||||
createContext,
|
||||
createOperatorClient,
|
||||
} from "../../../gateway/server-plugin-in-process-dispatch.test-support.js";
|
||||
import { emitAgentEvent } from "../../../infra/agent-events.js";
|
||||
import { SqliteWorkerError } from "../../../infra/sqlite-worker-contract.js";
|
||||
import * as hookRuntime from "../../../plugins/hook-runner-global.js";
|
||||
import { createHookRunnerWithRegistry } from "../../../plugins/hooks.test-fixtures.js";
|
||||
import {
|
||||
getPluginRuntimeGatewayRequestScope,
|
||||
withPluginRuntimeGatewayRequestScope,
|
||||
} from "../../../plugins/runtime/gateway-request-scope.js";
|
||||
import { createDeferredCore } from "../../../shared/deferred.js";
|
||||
import {
|
||||
closeOpenClawStateDatabaseAsync,
|
||||
closeOpenClawStateDatabaseByPathAsync,
|
||||
} from "../../../state/openclaw-state-db-cache.js";
|
||||
import { openOpenClawStateDatabase } from "../../../state/openclaw-state-db.js";
|
||||
import { captureOpenClawStateWorkerContext } from "../../../state/openclaw-state-worker-context.js";
|
||||
import * as stateWorker from "../../../state/openclaw-state-worker-store.js";
|
||||
import { setTestEnvValue } from "../../../test-utils/env.js";
|
||||
import { loadAgentRuntimePluginRegistryHandle } from "../../runtime-plugins.js";
|
||||
import * as announceCleanup from "./subagent-registry-lifecycle-announce-cleanup.js";
|
||||
import { subagentRuns } from "./subagent-registry-memory.js";
|
||||
import { getSubagentRegistryPublicationRevision } from "./subagent-registry-publication.js";
|
||||
import * as registryState from "./subagent-registry-state.js";
|
||||
import { observeRootWork } from "./subagent-registry.browser-cleanup.test-support.js";
|
||||
import {
|
||||
registerSubagentRun,
|
||||
resumeSubagentRun,
|
||||
prepareSubagentSessionCleanupRevocation,
|
||||
} from "./subagent-registry.js";
|
||||
import { writeSubagentSessionEntry } from "./subagent-registry.persistence.test-support.js";
|
||||
import { loadSubagentRegistryFromSqlite } from "./subagent-registry.store.sqlite.js";
|
||||
import { testing } from "./subagent-registry.test-helpers.js";
|
||||
|
||||
vi.mock("../../../state/openclaw-state-worker-store.js", { spy: true });
|
||||
|
||||
vi.mock("./subagent-registry-lifecycle-announce-cleanup.js", { spy: true });
|
||||
|
||||
vi.mock("../../../plugins/hook-runner-global.js", { spy: true });
|
||||
|
||||
const fixture = useSubagentControlFixture();
|
||||
const nativeState = await vi.importActual<typeof registryState>("./subagent-registry-state.js");
|
||||
|
||||
const nativeWorker = await vi.importActual<typeof stateWorker>(
|
||||
"../../../state/openclaw-state-worker-store.js",
|
||||
);
|
||||
|
||||
const nativeCleanup = await vi.importActual<typeof announceCleanup>(
|
||||
"./subagent-registry-lifecycle-announce-cleanup.js",
|
||||
);
|
||||
|
||||
async function registerCompletion(
|
||||
runId: string,
|
||||
options: { holdForRequester?: boolean; cleanup?: "keep" | "delete" } = {},
|
||||
) {
|
||||
const childSessionKey = `agent:main:subagent:${runId}`;
|
||||
await writeSubagentSessionEntry({
|
||||
stateDir: fixture.stateDir,
|
||||
agentId: "main",
|
||||
sessionKey: childSessionKey,
|
||||
defaultSessionId: "ordinary-child-session",
|
||||
lifecycleRevision: "ordinary-child-revision",
|
||||
});
|
||||
fixture.capture.mockResolvedValue("Synthetic completed result.");
|
||||
fixture.wake.mockResolvedValue(false);
|
||||
await registerSubagentRun({
|
||||
runId,
|
||||
childSessionKey,
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterAgentId: "main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "finish ordinary cleanup",
|
||||
cleanup: options.cleanup ?? "keep",
|
||||
expectsCompletionMessage: options.holdForRequester === true,
|
||||
completionTarget: options.holdForRequester ? "parent" : undefined,
|
||||
requesterTurnRunId: options.holdForRequester ? "held-requester-turn" : undefined,
|
||||
});
|
||||
vi.mocked(registryState.persistSubagentRunsToDiskAsyncOrThrow).mockImplementation(
|
||||
nativeState.persistSubagentRunsToDiskAsyncOrThrow,
|
||||
);
|
||||
return { runId, childSessionKey };
|
||||
}
|
||||
|
||||
function completeRegistered(run: { runId: string; childSessionKey: string }) {
|
||||
emitAgentEvent({
|
||||
...run,
|
||||
sessionKey: run.childSessionKey,
|
||||
stream: "lifecycle",
|
||||
data: {
|
||||
phase: "end",
|
||||
endedAt: Date.now(),
|
||||
terminalReply: { disposition: "visible", text: "Synthetic completed result." },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
it.each(["keep", "delete"] as const)(
|
||||
"settles registered quiet %s completion without host registry writes",
|
||||
async (cleanup) => {
|
||||
const run = await registerCompletion(`ordinary-announcement-${cleanup}`, { cleanup });
|
||||
const { runId } = run;
|
||||
const deleted: unknown[] = [];
|
||||
fixture.gateway.mockImplementation(async (request) => {
|
||||
if (request.method === "agent.wait") {
|
||||
return await new Promise<never>(() => {});
|
||||
}
|
||||
if (request.method !== "sessions.delete") {
|
||||
throw new Error(`Unexpected RPC ${request.method}`);
|
||||
}
|
||||
request.assertDispatchCurrent?.();
|
||||
deleted.push({
|
||||
request: request.params,
|
||||
marker: loadSubagentRegistryFromSqlite().get(runId)?.deleteCleanupDispatchedAt,
|
||||
});
|
||||
return {};
|
||||
});
|
||||
const registryWrites: string[] = [];
|
||||
let statements: ReturnType<typeof observeHostDataSql> | undefined;
|
||||
// Terminal admission precedes browser cleanup. Measure ordinary cleanup through
|
||||
// its awaited registry bookkeeping; requester batch transport stays mocked.
|
||||
fixture.cleanup.mockImplementation(async () => {
|
||||
statements = observeHostDataSql((sql) => {
|
||||
if (/(?:insert into|update|delete from)\s+"?subagent_runs\b/i.test(sql)) {
|
||||
registryWrites.push(sql);
|
||||
}
|
||||
});
|
||||
});
|
||||
try {
|
||||
completeRegistered(run);
|
||||
await fixture.settle();
|
||||
expect(loadSubagentRegistryFromSqlite().get(runId)).toMatchObject({
|
||||
cleanupCompletedAt: expect.any(Number),
|
||||
execution: { status: "terminal", outcome: { status: "ok" } },
|
||||
delivery: { status: "not_required" },
|
||||
});
|
||||
expect(fixture.cleanup).toHaveBeenCalledOnce();
|
||||
expect(statements).toBeDefined();
|
||||
expect(fixture.announce).not.toHaveBeenCalled();
|
||||
expect(deleted).toHaveLength(cleanup === "delete" ? 1 : 0);
|
||||
if (cleanup === "delete") {
|
||||
expect(deleted[0]).toMatchObject({
|
||||
marker: expect.any(Number),
|
||||
request: {
|
||||
key: run.childSessionKey,
|
||||
expectedSessionId: "ordinary-child-session",
|
||||
expectedLifecycleRevision: "ordinary-child-revision",
|
||||
},
|
||||
});
|
||||
}
|
||||
expect(registryWrites.length).toBe(0);
|
||||
} finally {
|
||||
statements?.restore();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["not-committed", "unknown", "successor"] as const)(
|
||||
"retains registered cleanup admission while its start waits (%s)",
|
||||
async (change) => {
|
||||
const run = await registerCompletion(`cleanup-start-${change}`);
|
||||
const entry = subagentRuns.get(run.runId)!;
|
||||
const ready = createDeferredCore();
|
||||
const release = createDeferredCore();
|
||||
let intercepted = false;
|
||||
let committedBeforeLoss = false;
|
||||
const worker = vi
|
||||
.spyOn(stateWorker, "runOpenClawStateWorkerOperation")
|
||||
.mockImplementation(async (context, operation, options) => {
|
||||
const selected =
|
||||
(!intercepted || change === "not-committed") &&
|
||||
entry.cleanupHandled === true &&
|
||||
entry.cleanupCompletedAt === undefined;
|
||||
if (!selected) {
|
||||
return nativeWorker.runOpenClawStateWorkerOperation(context, operation, options);
|
||||
}
|
||||
intercepted = true;
|
||||
ready.resolve();
|
||||
await release.promise;
|
||||
if (change === "not-committed") {
|
||||
throw new Error("Synthetic refused cleanup start");
|
||||
}
|
||||
return nativeWorker.runOpenClawStateWorkerOperation(
|
||||
context,
|
||||
(scope) =>
|
||||
operation({
|
||||
async execute(command, executeOptions) {
|
||||
const result = await scope.execute(command, executeOptions);
|
||||
if (change === "unknown" && command.type === "subagents.persistChanges") {
|
||||
committedBeforeLoss = true;
|
||||
throw new SqliteWorkerError(
|
||||
"Synthetic lost cleanup acknowledgement",
|
||||
"outcome-unknown",
|
||||
);
|
||||
}
|
||||
return result;
|
||||
},
|
||||
}),
|
||||
options,
|
||||
);
|
||||
});
|
||||
try {
|
||||
completeRegistered(run);
|
||||
await ready.promise;
|
||||
const before = loadSubagentRegistryFromSqlite().get(run.runId);
|
||||
expect(entry.cleanupHandled).toBe(true);
|
||||
expect(before?.cleanupHandled).not.toBe(true);
|
||||
expect(fixture.wake).not.toHaveBeenCalled();
|
||||
resumeSubagentRun(run.runId);
|
||||
if (change === "successor") {
|
||||
await registerSubagentRun({
|
||||
runId: `${run.runId}-successor`,
|
||||
childSessionKey: run.childSessionKey,
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterAgentId: "main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "new child generation",
|
||||
cleanup: "keep",
|
||||
expectsCompletionMessage: false,
|
||||
});
|
||||
}
|
||||
release.resolve();
|
||||
await fixture.settle();
|
||||
expect(fixture.wake).not.toHaveBeenCalled();
|
||||
expect(loadSubagentRegistryFromSqlite().get(run.runId)?.cleanupCompletedAt).toBeUndefined();
|
||||
if (change === "not-committed") {
|
||||
expect(entry.cleanupHandled).toBe(false);
|
||||
expect(loadSubagentRegistryFromSqlite().get(run.runId)).toEqual(before);
|
||||
worker.mockRestore();
|
||||
resumeSubagentRun(run.runId);
|
||||
await fixture.settle();
|
||||
expect(fixture.wake).toHaveBeenCalledOnce();
|
||||
expect(loadSubagentRegistryFromSqlite().get(run.runId)?.cleanupCompletedAt).toBeTypeOf(
|
||||
"number",
|
||||
);
|
||||
} else if (change === "unknown") {
|
||||
expect(entry.cleanupHandled).toBe(true);
|
||||
expect(committedBeforeLoss).toBe(true);
|
||||
const persisted = loadSubagentRegistryFromSqlite().get(run.runId)!;
|
||||
// Serialization clears the unfinished process lock; uncertain custody remains live.
|
||||
expect(persisted.cleanupHandled).toBe(false);
|
||||
expect(persisted.execution).toEqual(before?.execution);
|
||||
expect(persisted.completion).toEqual(before?.completion);
|
||||
resumeSubagentRun(run.runId);
|
||||
await fixture.settle();
|
||||
expect(fixture.wake).not.toHaveBeenCalled();
|
||||
} else {
|
||||
expect(subagentRuns.get(`${run.runId}-successor`)?.execution.status).toBe("running");
|
||||
}
|
||||
} finally {
|
||||
release.resolve();
|
||||
worker.mockRestore();
|
||||
await fixture.settle();
|
||||
if (change === "unknown") {
|
||||
await closeOpenClawStateDatabaseAsync();
|
||||
await nativeState.restoreSubagentRunsFromDisk({ runs: subagentRuns });
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ phase: "capture", publication: "none" },
|
||||
{ phase: "initial write", publication: "none" },
|
||||
{ phase: "initial write", publication: "same-row reservation" },
|
||||
{ phase: "initial write", publication: "restored same-ID owner" },
|
||||
{ phase: "initial write", publication: "replacement source" },
|
||||
] as const)(
|
||||
"retains exact cleanup custody after source refusal ($phase, $publication)",
|
||||
async ({ phase, publication }) => {
|
||||
const joinWork = observeRootWork();
|
||||
const run = await registerCompletion("sealed-cleanup-start", {
|
||||
holdForRequester: true,
|
||||
});
|
||||
completeRegistered(run);
|
||||
await joinWork(true);
|
||||
await fixture.settle();
|
||||
const databasePath = captureOpenClawStateWorkerContext().admission.databasePath;
|
||||
const replacementPath = path.join(fixture.stateDir, "replacement.sqlite");
|
||||
if (publication === "replacement source") {
|
||||
openOpenClawStateDatabase({ path: replacementPath });
|
||||
await closeOpenClawStateDatabaseByPathAsync(replacementPath);
|
||||
}
|
||||
vi.mocked(registryState.persistSubagentRunsToDiskOrThrow).mockImplementation(
|
||||
nativeState.persistSubagentRunsToDiskOrThrow,
|
||||
);
|
||||
const entry = subagentRuns.get(run.runId)!;
|
||||
entry.suppressCompletionDelivery = true;
|
||||
nativeState.persistSubagentRunsToDiskOrThrow(subagentRuns, [run.runId]);
|
||||
const before = loadSubagentRegistryFromSqlite().get(run.runId);
|
||||
const entered = createDeferredCore();
|
||||
const release = createDeferredCore();
|
||||
let closing: Promise<void> | undefined;
|
||||
let refused: unknown;
|
||||
const start = vi
|
||||
.spyOn(announceCleanup, "startSubagentAnnounceCleanupFlow")
|
||||
.mockImplementation((...args) => {
|
||||
if (phase !== "capture") {
|
||||
return nativeCleanup.startSubagentAnnounceCleanupFlow(...args);
|
||||
}
|
||||
closing ??= closeOpenClawStateDatabaseAsync();
|
||||
try {
|
||||
return nativeCleanup.startSubagentAnnounceCleanupFlow(...args);
|
||||
} catch (error) {
|
||||
refused = error;
|
||||
throw error;
|
||||
} finally {
|
||||
entered.resolve();
|
||||
}
|
||||
});
|
||||
const worker = vi
|
||||
.spyOn(stateWorker, "runOpenClawStateWorkerOperation")
|
||||
.mockImplementation(async (...args) => {
|
||||
if (phase !== "initial write" || !entry.cleanupHandled || closing) {
|
||||
return nativeWorker.runOpenClawStateWorkerOperation(...args);
|
||||
}
|
||||
closing = closeOpenClawStateDatabaseAsync();
|
||||
try {
|
||||
return await nativeWorker.runOpenClawStateWorkerOperation(...args);
|
||||
} catch (error) {
|
||||
refused = error;
|
||||
entered.resolve();
|
||||
await release.promise;
|
||||
throw error;
|
||||
} finally {
|
||||
entered.resolve();
|
||||
}
|
||||
});
|
||||
try {
|
||||
if (phase === "capture") {
|
||||
expect(() => resumeSubagentRun(run.runId)).toThrow(
|
||||
"state database read admission is closed",
|
||||
);
|
||||
} else {
|
||||
resumeSubagentRun(run.runId);
|
||||
}
|
||||
await entered.promise;
|
||||
await closing;
|
||||
let currentOwner = entry;
|
||||
if (publication === "replacement source") {
|
||||
await rename(replacementPath, databasePath);
|
||||
await nativeState.restoreSubagentRunsFromDisk({ runs: subagentRuns });
|
||||
expect(subagentRuns.get(run.runId)).toBe(entry);
|
||||
expect(loadSubagentRegistryFromSqlite().has(run.runId)).toBe(false);
|
||||
} else if (publication !== "none") {
|
||||
(await prepareSubagentSessionCleanupRevocation(run.childSessionKey))();
|
||||
expect(subagentRuns.get(run.runId)).toBe(entry);
|
||||
expect(
|
||||
loadSubagentRegistryFromSqlite().get(run.runId)?.execution.suppressSessionEffects,
|
||||
).toBe(true);
|
||||
if (publication === "restored same-ID owner") {
|
||||
await nativeState.restoreSubagentRunsFromDisk({ runs: subagentRuns });
|
||||
currentOwner = subagentRuns.get(run.runId)!;
|
||||
expect(currentOwner).not.toBe(entry);
|
||||
expect(currentOwner.generation).toBe(entry.generation);
|
||||
}
|
||||
}
|
||||
const published = loadSubagentRegistryFromSqlite().get(run.runId);
|
||||
const ownerBeforeRelease = structuredClone(currentOwner);
|
||||
release.resolve();
|
||||
await joinWork(true);
|
||||
expect(refused).toBeInstanceOf(Error);
|
||||
expect(fixture.wake).not.toHaveBeenCalled();
|
||||
if (publication !== "none") {
|
||||
expect(subagentRuns.get(run.runId)).toBe(currentOwner);
|
||||
expect(currentOwner).toEqual(ownerBeforeRelease);
|
||||
expect(entry.cleanupHandled).toBe(true);
|
||||
expect(loadSubagentRegistryFromSqlite().get(run.runId)).toEqual(published);
|
||||
expect(fixture.announce).not.toHaveBeenCalled();
|
||||
return;
|
||||
}
|
||||
expect(entry.cleanupHandled).not.toBe(true);
|
||||
expect(loadSubagentRegistryFromSqlite().get(run.runId)).toEqual(before);
|
||||
expect(fixture.wake).not.toHaveBeenCalled();
|
||||
start.mockRestore();
|
||||
worker.mockRestore();
|
||||
resumeSubagentRun(run.runId);
|
||||
await joinWork(true);
|
||||
await fixture.settle();
|
||||
expect(loadSubagentRegistryFromSqlite().get(run.runId)?.cleanupCompletedAt).toBeTypeOf(
|
||||
"number",
|
||||
);
|
||||
expect(fixture.announce).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
release.resolve();
|
||||
start.mockRestore();
|
||||
worker.mockRestore();
|
||||
await joinWork();
|
||||
await closing;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["current", "revoked", "source switched", "yielded"] as const)(
|
||||
"uses current authority for an ended hook after suspended bookkeeping (%s)",
|
||||
async (change) => {
|
||||
const runId = "suspended-announcement-hook";
|
||||
const childSessionKey = `agent:main:subagent:${runId}`;
|
||||
await writeSubagentSessionEntry({
|
||||
stateDir: fixture.stateDir,
|
||||
agentId: "main",
|
||||
sessionKey: childSessionKey,
|
||||
defaultSessionId: "suspended-child-session",
|
||||
});
|
||||
const waitResult = createDeferredCore<Record<string, unknown>>();
|
||||
const waitStarted = createDeferredCore();
|
||||
if (change === "yielded") {
|
||||
vi.mocked(registryState.persistSubagentRunsToDisk).mockImplementation(
|
||||
nativeState.persistSubagentRunsToDisk,
|
||||
);
|
||||
fixture.gateway.mockImplementation(async (request) => {
|
||||
if (request.method !== "agent.wait") {
|
||||
throw new Error(`Unexpected RPC ${request.method}`);
|
||||
}
|
||||
|
||||
waitStarted.resolve();
|
||||
return await waitResult.promise;
|
||||
});
|
||||
}
|
||||
await registerSubagentRun({
|
||||
runId,
|
||||
childSessionKey,
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterAgentId: "main",
|
||||
requesterDisplayKey: "main",
|
||||
requesterTurnRunId: "held-parent-turn",
|
||||
completionTarget: "parent",
|
||||
task: "discard a suspended result without stale hook effects",
|
||||
cleanup: "keep",
|
||||
retainAttachmentsOnKeep: true,
|
||||
expectsCompletionMessage: true,
|
||||
});
|
||||
const registeredEntry = subagentRuns.get(runId);
|
||||
|
||||
if (change === "yielded") {
|
||||
await waitStarted.promise;
|
||||
}
|
||||
|
||||
vi.mocked(registryState.persistSubagentRunsToDiskAsyncOrThrow).mockImplementation(
|
||||
nativeState.persistSubagentRunsToDiskAsyncOrThrow,
|
||||
);
|
||||
|
||||
const settleCompletion = observeRootWork();
|
||||
try {
|
||||
emitAgentEvent({
|
||||
runId,
|
||||
sessionKey: childSessionKey,
|
||||
stream: "lifecycle",
|
||||
data: {
|
||||
phase: "end",
|
||||
endedAt: Date.now(),
|
||||
terminalReply: { disposition: "visible", text: "Suspended synthetic result." },
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
await settleCompletion();
|
||||
}
|
||||
|
||||
await fixture.settle();
|
||||
|
||||
const revokeSessionEffects = await prepareSubagentSessionCleanupRevocation(childSessionKey);
|
||||
const originalSource = captureOpenClawStateWorkerContext();
|
||||
const replacementDir = path.join(fixture.stateDir, "replacement-state");
|
||||
if (change === "source switched") {
|
||||
openOpenClawStateDatabase({
|
||||
path: path.join(replacementDir, path.basename(originalSource.admission.databasePath)),
|
||||
});
|
||||
}
|
||||
const entry = subagentRuns.get(runId)!;
|
||||
let pausedRecord: typeof entry | undefined;
|
||||
expect(entry.execution.status).toBe("terminal");
|
||||
expect(entry.endedHookEmittedAt).toBeUndefined();
|
||||
entry.delivery = {
|
||||
...entry.delivery,
|
||||
status: "suspended",
|
||||
suspendedAt: Date.now() - 8 * 24 * 60 * 60_000,
|
||||
suspendedReason: "expiry",
|
||||
};
|
||||
nativeState.persistSubagentRunsToDiskOrThrow(subagentRuns, [runId]);
|
||||
const ended = vi.fn(async () => {});
|
||||
const { registry, runner } = createHookRunnerWithRegistry([
|
||||
{ hookName: "subagent_ended", handler: ended },
|
||||
]);
|
||||
vi.mocked(loadAgentRuntimePluginRegistryHandle).mockReturnValue(registry);
|
||||
vi.spyOn(hookRuntime, "getGlobalHookRunner").mockReturnValue(runner);
|
||||
const ready = createDeferredCore();
|
||||
const release = createDeferredCore();
|
||||
let held = false;
|
||||
vi.mocked(registryState.persistSubagentRunsToDiskAsyncOrThrow).mockImplementation(
|
||||
async (...args) => {
|
||||
const bookkeeping = !held && entry.delivery?.status === "discarded";
|
||||
await nativeState.persistSubagentRunsToDiskAsyncOrThrow(...args);
|
||||
if (bookkeeping) {
|
||||
held = true;
|
||||
ready.resolve();
|
||||
await release.promise;
|
||||
}
|
||||
},
|
||||
);
|
||||
const sweeping = testing.sweepOnceForTests();
|
||||
const sweepOutcome = sweeping.then(
|
||||
() => ({ completed: true as const }),
|
||||
(error: unknown) => ({ completed: false as const, error }),
|
||||
);
|
||||
try {
|
||||
await ready.promise;
|
||||
|
||||
expect(loadSubagentRegistryFromSqlite().get(runId)?.cleanupCompletedAt).toBeTypeOf("number");
|
||||
expect(ended).not.toHaveBeenCalled();
|
||||
if (change === "revoked") {
|
||||
revokeSessionEffects();
|
||||
expect(subagentRuns.get(runId)).toBe(entry);
|
||||
expect(loadSubagentRegistryFromSqlite().get(runId)?.execution.suppressSessionEffects).toBe(
|
||||
true,
|
||||
);
|
||||
}
|
||||
if (change === "source switched") {
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", replacementDir);
|
||||
expect(() => originalSource.admission.assertCurrent()).not.toThrow();
|
||||
}
|
||||
if (change === "yielded") {
|
||||
const paused = createDeferredCore();
|
||||
const stop = nativeState.onSubagentRegistryPersisted(() => {
|
||||
if (entry.pauseReason === "sessions_yield") {
|
||||
paused.resolve();
|
||||
}
|
||||
});
|
||||
try {
|
||||
waitResult.resolve({ status: "ok", yielded: true, endedAt: Date.now() });
|
||||
await paused.promise;
|
||||
} finally {
|
||||
stop();
|
||||
}
|
||||
expect(subagentRuns.get(runId)).toBe(registeredEntry);
|
||||
pausedRecord = structuredClone(entry);
|
||||
expect(loadSubagentRegistryFromSqlite().get(runId)?.pauseReason).toBe("sessions_yield");
|
||||
}
|
||||
release.resolve();
|
||||
const outcome = await sweepOutcome;
|
||||
await fixture.settle();
|
||||
expect(ended).toHaveBeenCalledTimes(change === "current" ? 1 : 0);
|
||||
if (change === "source switched") {
|
||||
expect(outcome).toMatchObject({
|
||||
completed: false,
|
||||
error: { message: "Queued registry write lost its original database" },
|
||||
});
|
||||
expect(loadSubagentRegistryFromSqlite().has(runId)).toBe(false);
|
||||
} else if (change === "yielded") {
|
||||
expect(outcome.completed).toBe(false);
|
||||
expect(entry).toEqual(pausedRecord);
|
||||
expect(loadSubagentRegistryFromSqlite().get(runId)?.pauseReason).toBe("sessions_yield");
|
||||
} else {
|
||||
expect(outcome).toEqual({ completed: true });
|
||||
}
|
||||
if (change !== "yielded") {
|
||||
expect(entry.delivery?.status).toBe("discarded");
|
||||
}
|
||||
if (change !== "current") {
|
||||
expect(entry.endedHookEmittedAt).toBeUndefined();
|
||||
} else {
|
||||
expect(entry.endedHookEmittedAt).toBeTypeOf("number");
|
||||
}
|
||||
} finally {
|
||||
waitResult.resolve({ status: "pending" });
|
||||
release.resolve();
|
||||
try {
|
||||
await Promise.allSettled([sweeping]);
|
||||
} finally {
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", fixture.stateDir);
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each([false, true])(
|
||||
"publishes registered suspended-delivery retirement only for its original row (successor: %s)",
|
||||
async (replace) => {
|
||||
const run = await registerCompletion("suspended-retirement", {
|
||||
cleanup: "delete",
|
||||
holdForRequester: true,
|
||||
});
|
||||
completeRegistered(run);
|
||||
await fixture.settle();
|
||||
const entry = subagentRuns.get(run.runId)!;
|
||||
expect(entry.execution.status).toBe("terminal");
|
||||
entry.delivery = {
|
||||
...entry.delivery,
|
||||
status: "suspended",
|
||||
suspendedAt: Date.now() - 8 * 24 * 60 * 60_000,
|
||||
suspendedReason: "expiry",
|
||||
};
|
||||
nativeState.persistSubagentRunsToDiskOrThrow(subagentRuns, [run.runId]);
|
||||
const ended = vi.fn(async () => {});
|
||||
const { registry, runner } = createHookRunnerWithRegistry([
|
||||
{ hookName: "subagent_ended", handler: ended },
|
||||
]);
|
||||
vi.mocked(loadAgentRuntimePluginRegistryHandle).mockReturnValue(registry);
|
||||
vi.spyOn(hookRuntime, "getGlobalHookRunner").mockReturnValue(runner);
|
||||
|
||||
const context = createContext();
|
||||
context.localEmbedded = true;
|
||||
context.getRuntimeConfig = getRuntimeConfig;
|
||||
const resolveGatewayContext = () => context;
|
||||
context.resolveGatewayContext = resolveGatewayContext;
|
||||
const client = createOperatorClient({
|
||||
profileName: "retirement-successor",
|
||||
scopes: ["operator.write"],
|
||||
});
|
||||
const source = await captureGatewayOperatorRunAuthority({ client, context });
|
||||
if (!source) {
|
||||
throw new Error("Expected canonical successor authority");
|
||||
}
|
||||
client.internal = { operatorRunAuthority: source.authority };
|
||||
const ready = createDeferredCore();
|
||||
const release = createDeferredCore();
|
||||
let held = false;
|
||||
const worker = vi
|
||||
.spyOn(stateWorker, "runOpenClawStateWorkerOperation")
|
||||
.mockImplementation((stateContext, operation, options) => {
|
||||
const retiring = !held && entry.delivery?.status === "discarded";
|
||||
return nativeWorker.runOpenClawStateWorkerOperation(
|
||||
stateContext,
|
||||
(scope) =>
|
||||
operation({
|
||||
async execute(command, executeOptions) {
|
||||
const result = await scope.execute(command, executeOptions);
|
||||
if (retiring && command.type === "subagents.persistChanges") {
|
||||
held = true;
|
||||
// The real transaction has settled; only its host acknowledgement waits.
|
||||
ready.resolve();
|
||||
await release.promise;
|
||||
}
|
||||
return result;
|
||||
},
|
||||
}),
|
||||
options,
|
||||
);
|
||||
});
|
||||
const sweeping = testing.sweepOnceForTests();
|
||||
const outcome = sweeping.then(
|
||||
() => ({ completed: true as const }),
|
||||
(error: unknown) => ({ completed: false as const, error }),
|
||||
);
|
||||
let successor = entry;
|
||||
try {
|
||||
await Promise.race([
|
||||
ready.promise,
|
||||
outcome.then(() => {
|
||||
throw new Error("Registered retirement omitted its acknowledgement boundary");
|
||||
}),
|
||||
]);
|
||||
expect(subagentRuns.get(run.runId)).toBe(entry);
|
||||
expect(entry.delivery?.status).toBe("suspended");
|
||||
expect(loadSubagentRegistryFromSqlite().has(run.runId)).toBe(false);
|
||||
expect(ended).not.toHaveBeenCalled();
|
||||
if (replace) {
|
||||
// Retaining this real source before the gate avoids queueing another reader behind it.
|
||||
await withPluginRuntimeGatewayRequestScope(
|
||||
{ client, context, resolveGatewayContext, isWebchatConnect: () => false },
|
||||
() =>
|
||||
registerSubagentRun({
|
||||
runId: run.runId,
|
||||
childSessionKey: run.childSessionKey,
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterAgentId: "main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "live retirement successor",
|
||||
cleanup: "keep",
|
||||
expectsCompletionMessage: true,
|
||||
}),
|
||||
);
|
||||
successor = subagentRuns.get(run.runId)!;
|
||||
expect(successor).not.toBe(entry);
|
||||
expect(loadSubagentRegistryFromSqlite().get(run.runId)?.task).toBe(
|
||||
"live retirement successor",
|
||||
);
|
||||
}
|
||||
const publicationRevision = getSubagentRegistryPublicationRevision();
|
||||
release.resolve();
|
||||
const result = await outcome;
|
||||
await fixture.settle();
|
||||
if (replace) {
|
||||
expect(result).toMatchObject({
|
||||
completed: false,
|
||||
error: { outcome: "committed", publication: "superseded" },
|
||||
});
|
||||
expect(subagentRuns.get(run.runId)).toBe(successor);
|
||||
expect(loadSubagentRegistryFromSqlite().get(run.runId)?.task).toBe(
|
||||
"live retirement successor",
|
||||
);
|
||||
expect(getSubagentRegistryPublicationRevision()).toBe(publicationRevision);
|
||||
expect(ended).not.toHaveBeenCalled();
|
||||
subagentRuns.runWithCompletionAuthority(successor, () => {
|
||||
const retained =
|
||||
getPluginRuntimeGatewayRequestScope()?.client?.internal?.operatorRunAuthority;
|
||||
expect(retained?.source).toBe(source.authority.source);
|
||||
retained?.assertCurrent();
|
||||
});
|
||||
} else {
|
||||
expect(result).toEqual({ completed: true });
|
||||
expect(subagentRuns.has(run.runId)).toBe(false);
|
||||
expect(loadSubagentRegistryFromSqlite().has(run.runId)).toBe(false);
|
||||
expect(ended).toHaveBeenCalledOnce();
|
||||
}
|
||||
} finally {
|
||||
release.resolve();
|
||||
await outcome;
|
||||
worker.mockRestore();
|
||||
source.release();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
|
@ -4,6 +4,10 @@ import { useSubagentControlFixture } from "./subagent-control.test-support.js";
|
|||
import { existsSync } from "node:fs";
|
||||
import { expect, it, onTestFinished, vi } from "vitest";
|
||||
import { createDeferred } from "../../../../test/helpers/promise.js";
|
||||
import {
|
||||
emptySqliteCounts,
|
||||
observeParentSqlite,
|
||||
} from "../../../../test/helpers/sqlite-parent-observer.js";
|
||||
import { getRuntimeConfig } from "../../../config/config.js";
|
||||
import { resolveSessionStorePathCore } from "../../../config/sessions/paths.js";
|
||||
import {
|
||||
|
|
@ -430,9 +434,10 @@ it.each([
|
|||
}
|
||||
await fixture.settle();
|
||||
registryState.clearSubagentRunsReadCacheForTest();
|
||||
const sql = observeMainThreadSql();
|
||||
sql.calibrate();
|
||||
let counts: number[];
|
||||
// Warm initialization ends here. Count all eight host APIs through Stop and
|
||||
// its owned settlement; independent durable readbacks begin after restore.
|
||||
const sql = observeParentSqlite();
|
||||
let counts: ReturnType<typeof emptySqliteCounts>;
|
||||
let unexpectedHold: ReturnType<typeof holdQueuedSwarmRun>;
|
||||
let cancellation: ReturnType<typeof tool.execute> | undefined;
|
||||
try {
|
||||
|
|
@ -467,14 +472,9 @@ it.each([
|
|||
terminalRelease.resolve();
|
||||
admission?.release();
|
||||
await Promise.allSettled([runnerSettlement, cancellation]);
|
||||
counts = sql.calls.map((probe) => probe.mock.calls.length);
|
||||
counts = { ...sql.counts };
|
||||
sql.restore();
|
||||
if (earlierSuccess) {
|
||||
// Ordinary success cleanup shares this observed window; its composed proof owns
|
||||
// the whole-window zero-SQL assertion after the separate owner cutover.
|
||||
console.info("Earlier-success Stop SQL, including ordinary cleanup", {
|
||||
counts,
|
||||
});
|
||||
resetGlobalHookRunner();
|
||||
}
|
||||
for (const { id, sessionId, handle } of handles) {
|
||||
|
|
@ -534,9 +534,8 @@ it.each([
|
|||
loadExactSessionEntryReadOnly({ agentId: "main", sessionKey: key(runId) })?.entry
|
||||
.abortedLastRun,
|
||||
).not.toBe(true);
|
||||
} else {
|
||||
expect(counts).toEqual(counts.map(() => 0));
|
||||
}
|
||||
expect(counts).toEqual(emptySqliteCounts());
|
||||
} finally {
|
||||
await unexpectedHold?.release();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,9 +40,7 @@ import { subagentRuns } from "./subagent-registry-memory.js";
|
|||
import { buildSubagentRunReadIndexFromRuns } from "./subagent-registry-queries.js";
|
||||
import {
|
||||
buildSubagentSessionListReadIndex,
|
||||
countActiveDescendantRuns,
|
||||
countPendingDescendantRuns,
|
||||
hasDescendantRunAwaitingSettle,
|
||||
isSubagentRunLive,
|
||||
isSubagentRunQueued,
|
||||
isSubagentSessionRunActive,
|
||||
|
|
@ -139,9 +137,11 @@ it("retains quiet admitted execution in listing, admission count, and requester
|
|||
)
|
||||
.toBe(false);
|
||||
expect.soft(countActiveRunsForSession(parent)).toBe(1);
|
||||
expect.soft(countActiveDescendantRuns(parent)).toBe(1);
|
||||
expect.soft(countPendingDescendantRuns(parent)).toBe(1);
|
||||
expect.soft(hasDescendantRunAwaitingSettle(parent)).toBe(true);
|
||||
expect.soft(buildSubagentSessionListReadIndex().countActiveDescendantRuns(parent)).toBe(1);
|
||||
expect.soft(await countPendingDescendantRuns(parent, () => {})).toBe(1);
|
||||
expect
|
||||
.soft(buildSubagentSessionListReadIndex().hasDescendantRunAwaitingSettle(parent))
|
||||
.toBe(true);
|
||||
expect.soft(isSubagentSessionRunActive(entry.childSessionKey)).toBe(true);
|
||||
expect
|
||||
.soft(
|
||||
|
|
@ -252,8 +252,7 @@ it("retains an exact queued collector reservation without calling it executor-li
|
|||
.toMatchObject({ hasActiveSubagentRun: true });
|
||||
expect.soft(countActiveRunsForSession(parent, { collect: true })).toBe(1);
|
||||
expect.soft(countActiveRunsForSession(parent, { collect: false })).toBe(0);
|
||||
expect.soft(countPendingDescendantRuns(parent)).toBe(1);
|
||||
expect.soft(hasDescendantRunAwaitingSettle(parent)).toBe(true);
|
||||
expect.soft(await countPendingDescendantRuns(parent, () => {})).toBe(1);
|
||||
expect
|
||||
.soft(
|
||||
(
|
||||
|
|
@ -275,7 +274,6 @@ it("retains an exact queued collector reservation without calling it executor-li
|
|||
expect(isSubagentRunQueued(entry)).toBe(false);
|
||||
expect(prepared.getExecutionObservation(prepared.runs[0]!)).toMatchObject({ state: "unknown" });
|
||||
expect(countActiveRunsForSession(parent, { collect: true })).toBe(0);
|
||||
expect(hasDescendantRunAwaitingSettle(parent)).toBe(false);
|
||||
const released = buildSubagentSessionListReadIndex();
|
||||
expect(released.countActiveDescendantRuns(parent)).toBe(0);
|
||||
expect(released.countPendingDescendantRuns(parent)).toBe(0);
|
||||
|
|
@ -312,8 +310,10 @@ it("does not retain an old run after its last claim releases preserved routing m
|
|||
expect.soft(isSubagentRunLive(entry)).toBe(false);
|
||||
expect.soft(isSubagentSessionRunActive(entry.childSessionKey)).toBe(false);
|
||||
expect.soft(countActiveRunsForSession(parent)).toBe(0);
|
||||
expect.soft(countPendingDescendantRuns(parent)).toBe(0);
|
||||
expect.soft(hasDescendantRunAwaitingSettle(parent)).toBe(false);
|
||||
expect.soft(await countPendingDescendantRuns(parent, () => {})).toBe(0);
|
||||
expect
|
||||
.soft(buildSubagentSessionListReadIndex().hasDescendantRunAwaitingSettle(parent))
|
||||
.toBe(false);
|
||||
expect.soft(buildSubagentSessionListReadIndex().countActiveDescendantRuns(parent)).toBe(0);
|
||||
} finally {
|
||||
releaseAgentRunContext(entry.runId, claim);
|
||||
|
|
@ -367,7 +367,7 @@ it("does not borrow a same-run-ID successor's live claim through a prepared obse
|
|||
}
|
||||
expect(resolveSubagentSessionStatus(subagentRuns.get(successor.runId))).toBe("running");
|
||||
expect(countActiveRunsForSession(parent)).toBe(1);
|
||||
expect(countPendingDescendantRuns(parent)).toBe(1);
|
||||
expect(await countPendingDescendantRuns(parent, () => {})).toBe(1);
|
||||
} finally {
|
||||
releaseAgentRunContext(successor.runId, claim);
|
||||
}
|
||||
|
|
@ -515,8 +515,8 @@ it("retains durable suspended completion debt without reporting a live executor
|
|||
entry.completion = { required: true, resultText: "completed result", capturedAt: start + 1 };
|
||||
persistSubagentRunsToDiskOrThrow(subagentRuns, [entry.runId]);
|
||||
now.mockReturnValue(olderThanCutoff);
|
||||
expect(countPendingDescendantRuns(parent)).toBe(1);
|
||||
expect(hasDescendantRunAwaitingSettle(parent)).toBe(true);
|
||||
expect(await countPendingDescendantRuns(parent, () => {})).toBe(1);
|
||||
expect(buildSubagentSessionListReadIndex().hasDescendantRunAwaitingSettle(parent)).toBe(true);
|
||||
expect(
|
||||
await blockSubagentCompletionDelivery({
|
||||
subagent: entry,
|
||||
|
|
@ -526,8 +526,8 @@ it("retains durable suspended completion debt without reporting a live executor
|
|||
).toBe(true);
|
||||
const suspended = subagentRuns.get(entry.runId)!;
|
||||
expect(suspended.delivery?.status).toBe("suspended");
|
||||
expect(countPendingDescendantRuns(parent)).toBe(1);
|
||||
expect(hasDescendantRunAwaitingSettle(parent)).toBe(false);
|
||||
expect(await countPendingDescendantRuns(parent, () => {})).toBe(1);
|
||||
expect(buildSubagentSessionListReadIndex().hasDescendantRunAwaitingSettle(parent)).toBe(false);
|
||||
expect(isSubagentRunLive(suspended)).toBe(false);
|
||||
expect(isSubagentSessionRunActive(entry.childSessionKey)).toBe(false);
|
||||
expect(countActiveRunsForSession(parent)).toBe(0);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,117 @@
|
|||
import {
|
||||
isGatewayRestartDraining,
|
||||
runWithGatewayDetachedWorkAdmission,
|
||||
} from "../../../process/gateway-work-admission.js";
|
||||
import { captureOpenClawStateWorkerContext } from "../../../state/openclaw-state-worker-context.js";
|
||||
import type {
|
||||
SubagentLifecycleController,
|
||||
SubagentLifecycleOptions,
|
||||
} from "./subagent-registry-lifecycle.js";
|
||||
import { assertSubagentRegistryWriteSourceCurrent } from "./subagent-registry-persistence.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
|
||||
export function createSubagentDeliveryResumeScheduling({
|
||||
runs,
|
||||
resumedRuns,
|
||||
resumeRetryTimers,
|
||||
resumeSubagentRun,
|
||||
finalizeResumedAnnounceGiveUp,
|
||||
warn,
|
||||
admissionRetryDelayMs,
|
||||
}: {
|
||||
runs: ReadonlyMap<string, SubagentRunRecord>;
|
||||
resumedRuns: Set<string>;
|
||||
resumeRetryTimers: Set<ReturnType<typeof setTimeout>>;
|
||||
resumeSubagentRun: (runId: string) => void;
|
||||
finalizeResumedAnnounceGiveUp: SubagentLifecycleController["finalizeResumedAnnounceGiveUp"];
|
||||
warn: SubagentLifecycleOptions["warn"];
|
||||
admissionRetryDelayMs: number;
|
||||
}) {
|
||||
function scheduleSubagentDeliveryResumeRetry(
|
||||
runId: string,
|
||||
scheduledEntry: SubagentRunRecord,
|
||||
waitMs: number,
|
||||
stateContext = captureOpenClawStateWorkerContext(),
|
||||
) {
|
||||
const generation = scheduledEntry.generation;
|
||||
const timer = setTimeout(() => {
|
||||
resumeRetryTimers.delete(timer);
|
||||
void runWithGatewayDetachedWorkAdmission(async () => {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
if (
|
||||
runs.get(runId) !== scheduledEntry ||
|
||||
scheduledEntry.generation !== generation ||
|
||||
scheduledEntry.cleanupHandled
|
||||
) {
|
||||
return;
|
||||
}
|
||||
resumedRuns.delete(runId);
|
||||
resumeSubagentRun(runId);
|
||||
}, "subagents:resume-retry").catch((error: unknown) => {
|
||||
warn("failed to resume subagent delivery retry", { runId, error });
|
||||
if (
|
||||
runs.get(runId) !== scheduledEntry ||
|
||||
scheduledEntry.generation !== generation ||
|
||||
scheduledEntry.cleanupHandled
|
||||
) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
} catch {
|
||||
resumedRuns.delete(runId);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
isGatewayRestartDraining() &&
|
||||
runs.get(runId) === scheduledEntry &&
|
||||
typeof scheduledEntry.cleanupCompletedAt !== "number"
|
||||
) {
|
||||
scheduleSubagentDeliveryResumeRetry(
|
||||
runId,
|
||||
scheduledEntry,
|
||||
Math.max(waitMs, admissionRetryDelayMs),
|
||||
stateContext,
|
||||
);
|
||||
return;
|
||||
}
|
||||
resumedRuns.delete(runId);
|
||||
});
|
||||
}, waitMs);
|
||||
timer.unref?.();
|
||||
resumeRetryTimers.add(timer);
|
||||
}
|
||||
|
||||
function finalizeResumedAnnounceGiveUpInBackground(
|
||||
runId: string,
|
||||
entry: SubagentRunRecord,
|
||||
reason: "expiry" | "permanent_failure",
|
||||
) {
|
||||
const stateContext = captureOpenClawStateWorkerContext();
|
||||
const generation = entry.generation;
|
||||
void runWithGatewayDetachedWorkAdmission(async () => {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
if (runs.get(runId) !== entry || entry.generation !== generation) {
|
||||
return;
|
||||
}
|
||||
await finalizeResumedAnnounceGiveUp({ runId, entry, reason, stateContext });
|
||||
}, "subagents:delivery-finalize").catch((error: unknown) => {
|
||||
warn("failed to finalize exhausted subagent delivery", { runId, reason, error });
|
||||
try {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
isGatewayRestartDraining() &&
|
||||
runs.get(runId) === entry &&
|
||||
typeof entry.cleanupCompletedAt !== "number"
|
||||
) {
|
||||
scheduleSubagentDeliveryResumeRetry(runId, entry, admissionRetryDelayMs, stateContext);
|
||||
resumedRuns.add(runId);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return { scheduleSubagentDeliveryResumeRetry, finalizeResumedAnnounceGiveUpInBackground };
|
||||
}
|
||||
|
|
@ -301,7 +301,10 @@ export async function persistSubagentSessionTiming(
|
|||
}
|
||||
|
||||
/** Best-effort async removal for a subagent attachment directory. */
|
||||
export async function safeRemoveAttachmentsDir(entry: SubagentRunRecord): Promise<boolean> {
|
||||
export async function safeRemoveAttachmentsDir(
|
||||
entry: SubagentRunRecord,
|
||||
isCurrent?: () => boolean,
|
||||
): Promise<boolean> {
|
||||
if (!entry.attachmentId) {
|
||||
// Legacy absolute/workspace paths are untrusted and intentionally retired without traversal.
|
||||
return true;
|
||||
|
|
@ -311,6 +314,7 @@ export async function safeRemoveAttachmentsDir(entry: SubagentRunRecord): Promis
|
|||
await cleanupMaterializedSubagentAttachments({
|
||||
childSessionKey: entry.childSessionKey,
|
||||
attachmentId: entry.attachmentId,
|
||||
isCurrent,
|
||||
});
|
||||
return true;
|
||||
} catch {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { hasSqliteWorkerOutcomeUnknown } from "../../../infra/sqlite-worker-contract.js";
|
||||
import { isSystemEventStoreCurrent } from "../../../infra/system-event-ownership.js";
|
||||
import { getGatewayContextResolver } from "../../../plugins/runtime/gateway-request-scope.js";
|
||||
import { defaultRuntime } from "../../../runtime.js";
|
||||
|
|
@ -5,7 +6,6 @@ import { normalizeDeliveryContext } from "../../../utils/delivery-context.shared
|
|||
import { resolveSubagentRequesterAgentId } from "../../subagent-requester-owner.js";
|
||||
import { loadSessionEntryByKey } from "../announce/subagent-announce-delivery.runtime.js";
|
||||
import {
|
||||
ensureCompletionState,
|
||||
ensureDeliveryState,
|
||||
getDeliveryLastError,
|
||||
isDeliverySuspended,
|
||||
|
|
@ -14,38 +14,39 @@ import {
|
|||
} from "./subagent-delivery-state.js";
|
||||
import {
|
||||
resolveAnnounceDeliveryDeadline,
|
||||
resolveCleanupCompletionReason,
|
||||
resolveDeferredCleanupDecision,
|
||||
shouldSuspendPendingFinalDelivery,
|
||||
} from "./subagent-registry-cleanup.js";
|
||||
import {
|
||||
ANNOUNCE_COMPLETION_HARD_EXPIRY_MS,
|
||||
ANNOUNCE_EXPIRY_MS,
|
||||
MIN_ANNOUNCE_RETRY_DELAY_MS,
|
||||
resolveAnnounceRetryDelayMs,
|
||||
safeRemoveAttachmentsDir,
|
||||
} from "./subagent-registry-helpers.js";
|
||||
import {
|
||||
beginSubagentCleanup,
|
||||
isSubagentCompletionDeliveryAllowed,
|
||||
retireSupersededCleanupIfNeeded,
|
||||
retireSupersededCleanupInBackground,
|
||||
beginSubagentCleanup,
|
||||
runDetachedCleanupAttempt,
|
||||
scheduleResumeSubagentRun,
|
||||
} from "./subagent-registry-lifecycle-attempt.js";
|
||||
import {
|
||||
isSubagentCompletionDeliveryAllowed,
|
||||
retireSupersededCleanupInBackground,
|
||||
suspendPendingFinalDelivery,
|
||||
} from "./subagent-registry-lifecycle-cleanup.js";
|
||||
import type { SubagentLifecycleAnnounceCleanupContext } from "./subagent-registry-lifecycle-context.js";
|
||||
import {
|
||||
buildSafeLifecycleErrorMeta,
|
||||
emitCompletionEndedHookIfNeeded,
|
||||
formatAnnounceDeliveryError,
|
||||
hasPriorRequesterDeliveryMirror,
|
||||
markPendingFinalDelivery,
|
||||
maskLifecycleIdentifier,
|
||||
recordAnnounceDeliveryResult,
|
||||
} from "./subagent-registry-lifecycle-delivery.js";
|
||||
import { finalizeSubagentCleanup } from "./subagent-registry-lifecycle-finalize-cleanup.js";
|
||||
import { finalizeResumedAnnounceGiveUp } from "./subagent-registry-lifecycle-give-up.js";
|
||||
import { commitSubagentLifecycleMutation } from "./subagent-registry-lifecycle-persistence.js";
|
||||
import { subagentRuns } from "./subagent-registry-memory.js";
|
||||
import {
|
||||
assertSubagentRegistryWriteSourceCurrent,
|
||||
assertSubagentRegistryWriteOutcomeKnown,
|
||||
captureSubagentRunMutationSnapshot,
|
||||
} from "./subagent-registry-persistence.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
import { deleteSubagentSessionForCleanup } from "./subagent-session-cleanup.js";
|
||||
|
||||
|
|
@ -83,19 +84,23 @@ export const resumeAncestorCleanup = (
|
|||
}
|
||||
const endedAgo = now - (entry.execution.endedAt ?? now);
|
||||
if (entry.expectsCompletionMessage !== true && endedAgo > ANNOUNCE_EXPIRY_MS) {
|
||||
const cleanupGeneration = beginSubagentCleanup(context, runId);
|
||||
if (cleanupGeneration === undefined) {
|
||||
const attempt = beginSubagentCleanup(context, runId);
|
||||
if (!attempt) {
|
||||
continue;
|
||||
}
|
||||
const { cleanupGeneration, stateContext } = attempt;
|
||||
runDetachedCleanupAttempt(context, {
|
||||
runId,
|
||||
entry,
|
||||
cleanupGeneration,
|
||||
stateContext,
|
||||
run: () =>
|
||||
finalizeResumedAnnounceGiveUp(context, {
|
||||
runId,
|
||||
entry,
|
||||
reason: "expiry",
|
||||
cleanupGeneration,
|
||||
stateContext,
|
||||
}),
|
||||
});
|
||||
continue;
|
||||
|
|
@ -105,177 +110,6 @@ export const resumeAncestorCleanup = (
|
|||
}
|
||||
};
|
||||
|
||||
const finalizeSubagentCleanup = async (
|
||||
context: SubagentLifecycleAnnounceCleanupContext,
|
||||
runId: string,
|
||||
cleanup: "delete" | "keep",
|
||||
announceOutcome: SubagentAnnounceFlowOutcome,
|
||||
cleanupGeneration: number,
|
||||
options?: {
|
||||
skipAnnounce?: boolean;
|
||||
skipRequesterDelivery?: boolean;
|
||||
},
|
||||
) => {
|
||||
const params = context.options;
|
||||
const entry = params.runs.get(runId);
|
||||
if (!entry) {
|
||||
return;
|
||||
}
|
||||
if (!context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
|
||||
await retireSupersededCleanupIfNeeded(context, runId, entry, cleanupGeneration);
|
||||
return;
|
||||
}
|
||||
const skipRequesterDelivery =
|
||||
options?.skipRequesterDelivery === true || entry.suppressCompletionDelivery === true;
|
||||
const finishCleanup = async (
|
||||
skipRequesterSettleWake: boolean,
|
||||
completionReason?: ReturnType<typeof resolveCleanupCompletionReason>,
|
||||
) => {
|
||||
if (cleanup === "delete" || !entry.retainAttachmentsOnKeep) {
|
||||
await safeRemoveAttachmentsDir(entry);
|
||||
}
|
||||
if (!context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
|
||||
await retireSupersededCleanupIfNeeded(context, runId, entry, cleanupGeneration);
|
||||
return;
|
||||
}
|
||||
context.completeCleanupBookkeeping({
|
||||
runId,
|
||||
entry,
|
||||
cleanup,
|
||||
completedAt: Date.now(),
|
||||
skipRequesterSettleWake,
|
||||
});
|
||||
// Hook loading is best-effort; durable delivery and cleanup must already
|
||||
// be terminal before plugin code can fail or stall.
|
||||
if (!(await context.shouldSuppressSessionEffects(entry))) {
|
||||
await emitCompletionEndedHookIfNeeded(
|
||||
params,
|
||||
entry,
|
||||
completionReason ?? resolveCleanupCompletionReason(entry),
|
||||
() =>
|
||||
context.isEndedHookOwnerCurrent(runId, entry) && context.sessionEffectsHostCurrent(entry),
|
||||
async () =>
|
||||
!(await context.shouldSuppressSessionEffects(entry)) &&
|
||||
context.isEndedHookOwnerCurrent(runId, entry),
|
||||
);
|
||||
}
|
||||
};
|
||||
if (entry.expectsCompletionMessage === false || skipRequesterDelivery) {
|
||||
const intentionalNonDelivery = entry.delivery?.disposition === "intentional_non_delivery";
|
||||
clearSubagentPendingDelivery(entry);
|
||||
if (skipRequesterDelivery) {
|
||||
const delivery = ensureDeliveryState(entry);
|
||||
delivery.status = "not_required";
|
||||
// Preserve the lifecycle owner's terminal fact after cleanup clears retry state.
|
||||
delivery.disposition = intentionalNonDelivery ? "intentional_non_delivery" : undefined;
|
||||
entry.suppressCompletionDelivery = undefined;
|
||||
}
|
||||
entry.wakeOnDescendantSettle = undefined;
|
||||
await finishCleanup(skipRequesterDelivery);
|
||||
return;
|
||||
}
|
||||
if (announceOutcome === "delivered" || announceOutcome === "intentional_non_delivery") {
|
||||
const delivery = ensureDeliveryState(entry);
|
||||
const terminalNonDelivery =
|
||||
announceOutcome === "intentional_non_delivery" && delivery.status === "failed";
|
||||
const shouldCreditDelivery = announceOutcome === "delivered";
|
||||
if (shouldCreditDelivery) {
|
||||
const deliveredAt = delivery.deliveredAt ?? delivery.announcedAt ?? Date.now();
|
||||
delivery.status = "delivered";
|
||||
delivery.deliveredAt = deliveredAt;
|
||||
delivery.announcedAt = delivery.announcedAt ?? deliveredAt;
|
||||
if (!options?.skipAnnounce) {
|
||||
delivery.announcedAt = deliveredAt;
|
||||
params.persist(runId);
|
||||
}
|
||||
clearSubagentPendingDelivery(entry);
|
||||
delivery.lastDropReason = undefined;
|
||||
} else {
|
||||
// A handoff stays pending for requester-settle; explicit suppression is
|
||||
// terminal and must not start another turn that overrides the decision.
|
||||
delivery.status = terminalNonDelivery ? "failed" : "pending";
|
||||
delivery.disposition = "intentional_non_delivery";
|
||||
delivery.payload = undefined;
|
||||
delivery.createdAt = undefined;
|
||||
delivery.attemptCount = undefined;
|
||||
delivery.nextAttemptAt = undefined;
|
||||
}
|
||||
entry.wakeOnDescendantSettle = undefined;
|
||||
const completion = ensureCompletionState(entry);
|
||||
completion.fallbackResultText = undefined;
|
||||
completion.fallbackCapturedAt = undefined;
|
||||
await finishCleanup(terminalNonDelivery, resolveCleanupCompletionReason(entry));
|
||||
return;
|
||||
}
|
||||
|
||||
if (announceOutcome === "session_queued") {
|
||||
// The correlated queue owns transport now. Settlement, not admission,
|
||||
// decides delivered versus blocked and re-enters cleanup afterward.
|
||||
entry.cleanupHandled = false;
|
||||
params.resumedRuns.delete(runId);
|
||||
params.persist(runId);
|
||||
return;
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const deferredDecision = resolveDeferredCleanupDecision({
|
||||
entry,
|
||||
now,
|
||||
activeDescendantRuns: Math.max(0, params.countPendingDescendantRuns(entry.childSessionKey)),
|
||||
announceExpiryMs: ANNOUNCE_EXPIRY_MS,
|
||||
announceCompletionHardExpiryMs: ANNOUNCE_COMPLETION_HARD_EXPIRY_MS,
|
||||
deferDescendantDelayMs: MIN_ANNOUNCE_RETRY_DELAY_MS,
|
||||
resolveAnnounceRetryDelayMs,
|
||||
});
|
||||
|
||||
if (deferredDecision.kind === "defer-descendants") {
|
||||
ensureDeliveryState(entry).lastAttemptAt = now;
|
||||
entry.wakeOnDescendantSettle = true;
|
||||
entry.cleanupHandled = false;
|
||||
params.resumedRuns.delete(runId);
|
||||
params.persist(runId);
|
||||
scheduleResumeSubagentRun(context, runId, entry, deferredDecision.delayMs);
|
||||
return;
|
||||
}
|
||||
|
||||
if (deferredDecision.kind === "give-up") {
|
||||
await finalizeResumedAnnounceGiveUp(context, {
|
||||
runId,
|
||||
entry,
|
||||
reason: deferredDecision.reason,
|
||||
cleanup,
|
||||
cleanupGeneration,
|
||||
retryCount: deferredDecision.retryCount,
|
||||
completedAt: now,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const requesterTurnPending = announceOutcome === "requester_turn_pending";
|
||||
if (!requesterTurnPending) {
|
||||
markPendingFinalDelivery({
|
||||
entry,
|
||||
error: "announce deferred or direct delivery failed",
|
||||
});
|
||||
}
|
||||
const delivery = ensureDeliveryState(entry);
|
||||
delivery.status = "pending";
|
||||
delivery.payload ??= loadPendingFinalDeliveryPayload(entry);
|
||||
delivery.windowStartedAt ??= entry.execution.endedAt ?? now;
|
||||
delivery.deadlineAt ??= delivery.windowStartedAt + ANNOUNCE_COMPLETION_HARD_EXPIRY_MS;
|
||||
// An admitted requester still owns this delivery; observation is not another failed attempt.
|
||||
const resumeDelayMs = requesterTurnPending
|
||||
? Math.min(MIN_ANNOUNCE_RETRY_DELAY_MS, delivery.deadlineAt - now)
|
||||
: deferredDecision.resumeDelayMs;
|
||||
delivery.nextAttemptAt = now + (resumeDelayMs ?? 0);
|
||||
entry.cleanupHandled = false;
|
||||
params.resumedRuns.delete(runId);
|
||||
params.persist(runId);
|
||||
if (resumeDelayMs != null) {
|
||||
scheduleResumeSubagentRun(context, runId, entry, resumeDelayMs);
|
||||
}
|
||||
};
|
||||
|
||||
export const startSubagentAnnounceCleanupFlow = (
|
||||
context: SubagentLifecycleAnnounceCleanupContext,
|
||||
runId: string,
|
||||
|
|
@ -301,31 +135,53 @@ export const startSubagentAnnounceCleanupFlow = (
|
|||
}
|
||||
// A terminal delivery failure closes upward delivery, not live descendants.
|
||||
// Their completion callback re-enters this same cleanup path without a timer.
|
||||
if (
|
||||
skipRequesterDelivery &&
|
||||
entry.wakeOnDescendantSettle === true &&
|
||||
params.countPendingDescendantRuns(entry.childSessionKey) > 0
|
||||
) {
|
||||
entry.cleanupHandled = false;
|
||||
params.resumedRuns.delete(runId);
|
||||
params.persist(runId);
|
||||
context.cleanupFailureCounts.delete(entry);
|
||||
return true;
|
||||
}
|
||||
const checkDescendants = skipRequesterDelivery && entry.wakeOnDescendantSettle === true;
|
||||
let suppressSessionEffects = !context.sessionEffectsHostCurrent(entry);
|
||||
const cleanupGeneration = beginSubagentCleanup(context, runId);
|
||||
if (cleanupGeneration === undefined) {
|
||||
const attempt = beginSubagentCleanup(context, runId);
|
||||
if (!attempt) {
|
||||
return false;
|
||||
}
|
||||
if (typeof entry.delivery?.announcedAt === "number" || entry.delivery?.status === "delivered") {
|
||||
const { cleanupGeneration, stateContext } = attempt;
|
||||
const assertPersistenceCurrent = () => {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
assertSubagentRegistryWriteOutcomeKnown([runId], stateContext.admission);
|
||||
};
|
||||
const assertCurrent = () => {
|
||||
assertPersistenceCurrent();
|
||||
if (!context.isCleanupGenerationCurrent(runId, entry, cleanupGeneration)) {
|
||||
throw new Error("Subagent cleanup generation changed before persistence.");
|
||||
}
|
||||
};
|
||||
const commit = (mutate: () => void, previous?: SubagentRunRecord, onPublished?: () => void) =>
|
||||
commitSubagentLifecycleMutation(context, {
|
||||
entry,
|
||||
stateContext,
|
||||
mutate,
|
||||
previous,
|
||||
assertCurrent,
|
||||
onPublished,
|
||||
});
|
||||
if (
|
||||
!checkDescendants &&
|
||||
(typeof entry.delivery?.announcedAt === "number" || entry.delivery?.status === "delivered")
|
||||
) {
|
||||
runDetachedCleanupAttempt(context, {
|
||||
runId,
|
||||
entry,
|
||||
cleanupGeneration,
|
||||
stateContext,
|
||||
run: () =>
|
||||
finalizeSubagentCleanup(context, runId, cleanup, "delivered", cleanupGeneration, {
|
||||
skipAnnounce: true,
|
||||
}),
|
||||
finalizeSubagentCleanup(
|
||||
context,
|
||||
entry,
|
||||
cleanup,
|
||||
"delivered",
|
||||
cleanupGeneration,
|
||||
stateContext,
|
||||
{
|
||||
skipAnnounce: true,
|
||||
},
|
||||
),
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
|
@ -347,6 +203,7 @@ export const startSubagentAnnounceCleanupFlow = (
|
|||
}
|
||||
};
|
||||
const childSessionEffectsAllowed = () => {
|
||||
assertCurrent();
|
||||
if (!suppressSessionEffects && !context.sessionEffectsHostCurrent(entry)) {
|
||||
suppressChildSessionEffects();
|
||||
}
|
||||
|
|
@ -355,7 +212,9 @@ export const startSubagentAnnounceCleanupFlow = (
|
|||
);
|
||||
};
|
||||
const prepareChildSessionEffects = async () => {
|
||||
assertCurrent();
|
||||
const suppress = !suppressSessionEffects && (await context.shouldSuppressSessionEffects(entry));
|
||||
assertPersistenceCurrent();
|
||||
if (!context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -369,14 +228,52 @@ export const startSubagentAnnounceCleanupFlow = (
|
|||
runId,
|
||||
entry,
|
||||
cleanupGeneration,
|
||||
stateContext,
|
||||
run: async () => {
|
||||
// This driver is detached. Yield once so synchronous successor
|
||||
// registration can invalidate it before sessions.delete is submitted.
|
||||
await Promise.resolve();
|
||||
assertPersistenceCurrent();
|
||||
if (!context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
|
||||
await retireSupersededCleanupIfNeeded(context, runId, entry, cleanupGeneration);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
checkDescendants &&
|
||||
(await params.countPendingDescendantRuns(entry.childSessionKey, assertCurrent)) > 0
|
||||
) {
|
||||
assertCurrent();
|
||||
await commit(
|
||||
() => {
|
||||
entry.cleanupHandled = false;
|
||||
},
|
||||
undefined,
|
||||
() => params.resumedRuns.delete(runId),
|
||||
);
|
||||
if (
|
||||
(await params.countPendingDescendantRuns(entry.childSessionKey, assertCurrent)) === 0
|
||||
) {
|
||||
assertCurrent();
|
||||
params.resumeSubagentRun(runId);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (
|
||||
checkDescendants &&
|
||||
(typeof entry.delivery?.announcedAt === "number" ||
|
||||
entry.delivery?.status === "delivered")
|
||||
) {
|
||||
await finalizeSubagentCleanup(
|
||||
context,
|
||||
entry,
|
||||
cleanup,
|
||||
"delivered",
|
||||
cleanupGeneration,
|
||||
stateContext,
|
||||
{ skipAnnounce: true },
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (cleanup === "delete" && (await prepareChildSessionEffects())) {
|
||||
const cleanupSessionEntry = await loadSessionEntryByKey(entry.childSessionKey);
|
||||
const cleanupSessionIdentity =
|
||||
|
|
@ -394,8 +291,9 @@ export const startSubagentAnnounceCleanupFlow = (
|
|||
} else if (canDelete && cleanupSessionIdentity) {
|
||||
// This durable boundary prevents a late yield from reviving a run
|
||||
// after deletion may already have reached the gateway.
|
||||
entry.deleteCleanupDispatchedAt ??= Date.now();
|
||||
params.persist(runId);
|
||||
await commit(() => {
|
||||
entry.deleteCleanupDispatchedAt ??= Date.now();
|
||||
});
|
||||
const sessionCleanup = await deleteSubagentSessionForCleanup({
|
||||
callGateway: params.callGateway,
|
||||
gatewayBinding: { resolveGatewayContext: getGatewayContextResolver(entry) },
|
||||
|
|
@ -420,14 +318,23 @@ export const startSubagentAnnounceCleanupFlow = (
|
|||
}
|
||||
}
|
||||
}
|
||||
assertPersistenceCurrent();
|
||||
if (!context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
|
||||
await retireSupersededCleanupIfNeeded(context, runId, entry, cleanupGeneration);
|
||||
return;
|
||||
}
|
||||
await finalizeSubagentCleanup(context, runId, cleanup, "delivered", cleanupGeneration, {
|
||||
skipAnnounce: true,
|
||||
skipRequesterDelivery,
|
||||
});
|
||||
await finalizeSubagentCleanup(
|
||||
context,
|
||||
entry,
|
||||
cleanup,
|
||||
"delivered",
|
||||
cleanupGeneration,
|
||||
stateContext,
|
||||
{
|
||||
skipAnnounce: true,
|
||||
skipRequesterDelivery,
|
||||
},
|
||||
);
|
||||
},
|
||||
});
|
||||
return true;
|
||||
|
|
@ -443,6 +350,7 @@ export const startSubagentAnnounceCleanupFlow = (
|
|||
let latestDeliveryError = getDeliveryLastError(entry);
|
||||
let committedDelivery: SubagentRunRecord["delivery"];
|
||||
const finalizeAnnounceCleanup = async (announceOutcome: SubagentAnnounceFlowOutcome) => {
|
||||
assertPersistenceCurrent();
|
||||
if (!context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
|
||||
await retireSupersededCleanupIfNeeded(context, runId, entry, cleanupGeneration);
|
||||
return;
|
||||
|
|
@ -451,6 +359,7 @@ export const startSubagentAnnounceCleanupFlow = (
|
|||
announceOutcome !== "delivered" &&
|
||||
entry.delivery?.status !== "delivered" &&
|
||||
(await hasPriorRequesterDeliveryMirror(params, entry));
|
||||
assertPersistenceCurrent();
|
||||
if (!context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
|
||||
await retireSupersededCleanupIfNeeded(context, runId, entry, cleanupGeneration);
|
||||
return;
|
||||
|
|
@ -462,11 +371,13 @@ export const startSubagentAnnounceCleanupFlow = (
|
|||
latestDeliveryError = undefined;
|
||||
}
|
||||
if (announceOutcome !== "delivered" && latestDeliveryError) {
|
||||
ensureDeliveryState(entry).lastError = latestDeliveryError;
|
||||
await commit(() => {
|
||||
ensureDeliveryState(entry).lastError = latestDeliveryError;
|
||||
});
|
||||
}
|
||||
await finalizeSubagentCleanup(
|
||||
context,
|
||||
runId,
|
||||
entry,
|
||||
cleanup,
|
||||
shouldCreditPriorDelivery
|
||||
? "delivered"
|
||||
|
|
@ -474,6 +385,7 @@ export const startSubagentAnnounceCleanupFlow = (
|
|||
? "intentional_non_delivery"
|
||||
: announceOutcome,
|
||||
cleanupGeneration,
|
||||
stateContext,
|
||||
);
|
||||
};
|
||||
|
||||
|
|
@ -504,22 +416,25 @@ export const startSubagentAnnounceCleanupFlow = (
|
|||
suppressChildSessionEffects: suppressSessionEffects,
|
||||
isChildSessionEffectsAllowed: childSessionEffectsAllowed,
|
||||
prepareChildSessionEffects,
|
||||
isCompletionDeliveryAllowed: () =>
|
||||
isSubagentCompletionDeliveryAllowed(context, entry, cleanupGeneration, committedDelivery),
|
||||
isCompletionDeliveryAllowed: () => {
|
||||
assertPersistenceCurrent();
|
||||
return isSubagentCompletionDeliveryAllowed(
|
||||
context,
|
||||
entry,
|
||||
cleanupGeneration,
|
||||
committedDelivery,
|
||||
);
|
||||
},
|
||||
isCompletionOwnedByRequesterYield: () =>
|
||||
entry.requesterTurnYielded === true ||
|
||||
entry.requesterSettleWake?.requesterYieldBatch === true,
|
||||
onBeforeDeleteChildSession:
|
||||
cleanup === "delete"
|
||||
? () => {
|
||||
? async () => {
|
||||
if (!childSessionEffectsAllowed()) {
|
||||
return false;
|
||||
}
|
||||
const previousDelivery = entry.delivery
|
||||
? { ...entry.delivery, payload: entry.delivery.payload }
|
||||
: undefined;
|
||||
const previousDeleteCleanupDispatchedAt = entry.deleteCleanupDispatchedAt;
|
||||
try {
|
||||
await commit(() => {
|
||||
if (
|
||||
entry.completion?.required === true &&
|
||||
entry.delivery?.status !== "delivered" &&
|
||||
|
|
@ -531,22 +446,15 @@ export const startSubagentAnnounceCleanupFlow = (
|
|||
delivery.createdAt ??= Date.now();
|
||||
delivery.payload = loadPendingFinalDeliveryPayload(entry);
|
||||
}
|
||||
// Announce owns delete submission; fence late yields at the
|
||||
// exact handoff instead of when cleanup merely starts.
|
||||
entry.deleteCleanupDispatchedAt ??= Date.now();
|
||||
params.persistOrThrow(runId);
|
||||
return true;
|
||||
} catch (error) {
|
||||
entry.delivery = previousDelivery;
|
||||
entry.deleteCleanupDispatchedAt = previousDeleteCleanupDispatchedAt;
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
return childSessionEffectsAllowed();
|
||||
}
|
||||
: undefined,
|
||||
onDeliveryResult: async (delivery) => {
|
||||
const previousDropReason = entry.delivery?.lastDropReason;
|
||||
assertPersistenceCurrent();
|
||||
if (!context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
|
||||
retireSupersededCleanupInBackground(context, runId, entry, cleanupGeneration);
|
||||
retireSupersededCleanupInBackground(context, runId, entry, cleanupGeneration, stateContext);
|
||||
return;
|
||||
}
|
||||
// A stale announce cannot replace a delivery already committed by requester-settle.
|
||||
|
|
@ -577,36 +485,43 @@ export const startSubagentAnnounceCleanupFlow = (
|
|||
});
|
||||
return;
|
||||
}
|
||||
const requesterTurnPending =
|
||||
!delivery.delivered && delivery.reason === "requester_turn_pending";
|
||||
assertCurrent();
|
||||
const previous = captureSubagentRunMutationSnapshot(entry);
|
||||
recordAnnounceDeliveryResult(entry, delivery, params.runs);
|
||||
const deliveryState = ensureDeliveryState(entry);
|
||||
latestDeliveryError =
|
||||
delivery.delivered || requesterTurnPending
|
||||
? undefined
|
||||
: formatAnnounceDeliveryError(delivery);
|
||||
if (delivery.delivered) {
|
||||
const deliveryState = ensureDeliveryState(entry);
|
||||
// Later chunks retain this receipt owner; requester settlement replaces it.
|
||||
committedDelivery = deliveryState;
|
||||
deliveryState.status = "delivered";
|
||||
deliveryState.announcedAt = deliveryState.deliveredAt ?? Date.now();
|
||||
clearSubagentPendingDelivery(entry);
|
||||
// Identified platform delivery precedes best-effort transcript
|
||||
// mirroring; task ownership must become durable at that same edge.
|
||||
params.persist(runId);
|
||||
latestDeliveryError = undefined;
|
||||
return;
|
||||
}
|
||||
const deliveryState = ensureDeliveryState(entry);
|
||||
if (delivery.reason === "requester_turn_pending") {
|
||||
latestDeliveryError = undefined;
|
||||
return;
|
||||
}
|
||||
if (delivery.reason === "delivery_suppressed") {
|
||||
deliveryState.status = "failed";
|
||||
}
|
||||
latestDeliveryError = formatAnnounceDeliveryError(delivery);
|
||||
if (
|
||||
deliveryState.lastError !== latestDeliveryError ||
|
||||
deliveryState.lastDropReason !== previousDropReason
|
||||
) {
|
||||
} else if (!requesterTurnPending) {
|
||||
if (delivery.reason === "delivery_suppressed") {
|
||||
deliveryState.status = "failed";
|
||||
}
|
||||
deliveryState.lastError = latestDeliveryError;
|
||||
params.persist(runId);
|
||||
}
|
||||
if (
|
||||
requesterTurnPending ||
|
||||
(!delivery.delivered &&
|
||||
previous.delivery?.lastError === latestDeliveryError &&
|
||||
previous.delivery?.lastDropReason === deliveryState.lastDropReason)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
await commit(
|
||||
() => {},
|
||||
previous,
|
||||
delivery.delivered
|
||||
? () => {
|
||||
committedDelivery = entry.delivery;
|
||||
}
|
||||
: undefined,
|
||||
);
|
||||
},
|
||||
// Idle completion has no ambient request scope. Missing entry ownership
|
||||
// fails closed instead of widening authority to another live Gateway.
|
||||
|
|
@ -616,6 +531,7 @@ export const startSubagentAnnounceCleanupFlow = (
|
|||
runId,
|
||||
entry,
|
||||
cleanupGeneration,
|
||||
stateContext,
|
||||
run: async () => {
|
||||
let announceOutcome: SubagentAnnounceFlowOutcome = "retryable";
|
||||
const deadline = new AbortController();
|
||||
|
|
@ -643,12 +559,16 @@ export const startSubagentAnnounceCleanupFlow = (
|
|||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
if (hasSqliteWorkerOutcomeUnknown(error)) {
|
||||
throw error;
|
||||
}
|
||||
defaultRuntime.log(
|
||||
`[warn] Subagent announce flow failed during cleanup for run ${runId}: ${String(error)}`,
|
||||
);
|
||||
} finally {
|
||||
clearTimeout(deadlineTimer);
|
||||
}
|
||||
assertPersistenceCurrent();
|
||||
if (
|
||||
context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration) &&
|
||||
entry.delivery?.status !== "delivered" &&
|
||||
|
|
|
|||
|
|
@ -0,0 +1,308 @@
|
|||
import { runWithoutOwnedSessionTranscriptWrites } from "../../../config/sessions/transcript-write-context.js";
|
||||
import { hasSqliteWorkerOutcomeUnknown } from "../../../infra/sqlite-worker-contract.js";
|
||||
import {
|
||||
isGatewayRestartDraining,
|
||||
runWithGatewayDetachedWorkAdmission,
|
||||
runWithGatewayDetachedWorkContinuation,
|
||||
} from "../../../process/gateway-work-admission.js";
|
||||
import { defaultRuntime } from "../../../runtime.js";
|
||||
import { captureOpenClawStateWorkerContext } from "../../../state/openclaw-state-worker-context.js";
|
||||
import type { OpenClawStateWorkerContext } from "../../../state/openclaw-state-worker-context.types.js";
|
||||
import { withoutGatewayToolCallerIdentity } from "../../tools/gateway-caller-context.js";
|
||||
import {
|
||||
MIN_ANNOUNCE_RETRY_DELAY_MS,
|
||||
resolveAnnounceRetryDelayMs,
|
||||
} from "./subagent-registry-helpers.js";
|
||||
import type { SubagentLifecycleCleanupContext } from "./subagent-registry-lifecycle-context.js";
|
||||
import { commitSubagentLifecycleMutation } from "./subagent-registry-lifecycle-persistence.js";
|
||||
import {
|
||||
assertSubagentRegistryWriteSourceCurrent,
|
||||
SubagentRegistryWriteError,
|
||||
} from "./subagent-registry-persistence.js";
|
||||
import { subscribeSubagentRunChanges } from "./subagent-registry-publication.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
|
||||
const MAX_DETACHED_CLEANUP_RETRIES = 3;
|
||||
|
||||
export function runWithSubagentCleanupWorkAdmission<T>(run: () => Promise<T>): Promise<T> {
|
||||
// Restart remains one-way; only suspension preserves an admitted cleanup owner.
|
||||
// The registry owns cleanup after the spawning tool's caller has retired.
|
||||
return withoutGatewayToolCallerIdentity(() =>
|
||||
isGatewayRestartDraining()
|
||||
? runWithGatewayDetachedWorkAdmission(run, "subagents:lifecycle-cleanup")
|
||||
: runWithGatewayDetachedWorkContinuation(run, "subagents:lifecycle-cleanup"),
|
||||
);
|
||||
}
|
||||
|
||||
export function scheduleResumeSubagentRun(
|
||||
context: SubagentLifecycleCleanupContext,
|
||||
runId: string,
|
||||
entry: SubagentRunRecord,
|
||||
delayMs: number,
|
||||
cleanupGeneration?: number,
|
||||
stateContext = captureOpenClawStateWorkerContext(),
|
||||
): void {
|
||||
const params = context.options;
|
||||
const timer = setTimeout(() => {
|
||||
context.scheduledResumeTimers.delete(timer);
|
||||
void runWithGatewayDetachedWorkAdmission(async () => {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
if (params.runs.get(runId) !== entry) {
|
||||
return;
|
||||
}
|
||||
if (cleanupGeneration !== undefined) {
|
||||
if (!context.isCleanupGenerationCurrent(runId, entry, cleanupGeneration)) {
|
||||
return;
|
||||
}
|
||||
if (entry.cleanupHandled) {
|
||||
await commitSubagentLifecycleMutation(context, {
|
||||
entry,
|
||||
stateContext,
|
||||
assertCurrent() {
|
||||
if (!context.isCleanupGenerationCurrent(runId, entry, cleanupGeneration)) {
|
||||
throw new Error("Subagent cleanup resume generation changed.");
|
||||
}
|
||||
},
|
||||
mutate: () => {
|
||||
entry.cleanupHandled = false;
|
||||
},
|
||||
onPublished: () => params.resumedRuns.delete(runId),
|
||||
});
|
||||
}
|
||||
}
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
if (
|
||||
params.runs.get(runId) !== entry ||
|
||||
(cleanupGeneration !== undefined &&
|
||||
!context.isCleanupGenerationCurrent(runId, entry, cleanupGeneration))
|
||||
) {
|
||||
return;
|
||||
}
|
||||
params.resumedRuns.delete(runId);
|
||||
params.resumeSubagentRun(runId);
|
||||
}, "subagents:resume").catch((err: unknown) => {
|
||||
defaultRuntime.log(`[warn] subagent cleanup resume failed (${runId}): ${String(err)}`);
|
||||
const current = params.runs.get(runId);
|
||||
try {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
isGatewayRestartDraining() &&
|
||||
current === entry &&
|
||||
typeof current.cleanupCompletedAt !== "number"
|
||||
) {
|
||||
scheduleResumeSubagentRun(
|
||||
context,
|
||||
runId,
|
||||
entry,
|
||||
Math.max(delayMs, MIN_ANNOUNCE_RETRY_DELAY_MS),
|
||||
cleanupGeneration,
|
||||
stateContext,
|
||||
);
|
||||
}
|
||||
});
|
||||
}, delayMs);
|
||||
timer.unref?.();
|
||||
context.scheduledResumeTimers.add(timer);
|
||||
}
|
||||
|
||||
export function runDetachedCleanupAttempt(
|
||||
context: SubagentLifecycleCleanupContext,
|
||||
args: {
|
||||
runId: string;
|
||||
entry: SubagentRunRecord;
|
||||
cleanupGeneration: number;
|
||||
stateContext: OpenClawStateWorkerContext;
|
||||
run: () => Promise<void>;
|
||||
},
|
||||
): void {
|
||||
const params = context.options;
|
||||
const stateContext = args.stateContext;
|
||||
let startCommitted = false;
|
||||
let ownsReservation = true;
|
||||
const stopReservationObservation = subscribeSubagentRunChanges((runIds) => {
|
||||
if (runIds === undefined || runIds.includes(args.runId)) {
|
||||
ownsReservation = false;
|
||||
}
|
||||
});
|
||||
const releaseReservation = () => {
|
||||
if (
|
||||
startCommitted ||
|
||||
!ownsReservation ||
|
||||
args.entry.cleanupCompletedAt ||
|
||||
!context.isCleanupAttemptCurrent(args.runId, args.entry, args.cleanupGeneration)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
// This releases process custody only; a later effect still needs fresh admission.
|
||||
args.entry.cleanupHandled = false;
|
||||
params.resumedRuns.delete(args.runId);
|
||||
return true;
|
||||
};
|
||||
const assertCurrent = () => {
|
||||
if (!context.isCleanupGenerationCurrent(args.runId, args.entry, args.cleanupGeneration)) {
|
||||
throw new Error("Subagent cleanup generation changed before persistence.");
|
||||
}
|
||||
};
|
||||
// The registry owns the full detached attempt through its final durable write.
|
||||
// Completion outlives the spawning attempt; inherited lock owners would
|
||||
// reject requester transcript writes after that attempt is disposed.
|
||||
runWithoutOwnedSessionTranscriptWrites(() => {
|
||||
void runWithSubagentCleanupWorkAdmission(async () => {
|
||||
try {
|
||||
// cleanupHandled is a process lock; effects still wait for the existing
|
||||
// start snapshot to commit through the captured writer.
|
||||
await commitSubagentLifecycleMutation(context, {
|
||||
entry: args.entry,
|
||||
stateContext,
|
||||
assertCurrent,
|
||||
mutate() {},
|
||||
});
|
||||
startCommitted = true;
|
||||
await args.run();
|
||||
if (context.isCleanupGeneration(args.entry, args.cleanupGeneration)) {
|
||||
context.cleanupFailureCounts.delete(args.entry);
|
||||
}
|
||||
} catch (err) {
|
||||
defaultRuntime.log(
|
||||
`[warn] subagent cleanup finalize failed (${args.runId}): ${String(err)}`,
|
||||
);
|
||||
if (hasSqliteWorkerOutcomeUnknown(err)) {
|
||||
throw err;
|
||||
}
|
||||
if (err instanceof SubagentRegistryWriteError && err.outcome === "committed") {
|
||||
if (err.publication === "superseded") {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
await retireSupersededCleanupIfNeeded(
|
||||
context,
|
||||
args.runId,
|
||||
args.entry,
|
||||
args.cleanupGeneration,
|
||||
);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const current = params.runs.get(args.runId);
|
||||
if (
|
||||
!current ||
|
||||
current.cleanupCompletedAt ||
|
||||
!context.isCleanupAttemptCurrent(args.runId, args.entry, args.cleanupGeneration)
|
||||
) {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
await retireSupersededCleanupIfNeeded(
|
||||
context,
|
||||
args.runId,
|
||||
args.entry,
|
||||
args.cleanupGeneration,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (startCommitted) {
|
||||
await commitSubagentLifecycleMutation(context, {
|
||||
entry: current,
|
||||
stateContext,
|
||||
assertCurrent,
|
||||
mutate: () => {
|
||||
current.cleanupHandled = false;
|
||||
},
|
||||
onPublished: () => params.resumedRuns.delete(args.runId),
|
||||
});
|
||||
} else if (!releaseReservation()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (!context.isCleanupGenerationCurrent(args.runId, args.entry, args.cleanupGeneration)) {
|
||||
return;
|
||||
}
|
||||
const failureCount = context.incrementCleanupFailureCount(current);
|
||||
if (failureCount <= MAX_DETACHED_CLEANUP_RETRIES) {
|
||||
scheduleResumeSubagentRun(
|
||||
context,
|
||||
args.runId,
|
||||
current,
|
||||
resolveAnnounceRetryDelayMs(failureCount),
|
||||
args.cleanupGeneration,
|
||||
stateContext,
|
||||
);
|
||||
}
|
||||
}
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
defaultRuntime.log(
|
||||
`[warn] subagent cleanup admission failed (${args.runId}): ${String(err)}`,
|
||||
);
|
||||
if (
|
||||
hasSqliteWorkerOutcomeUnknown(err) ||
|
||||
(err instanceof SubagentRegistryWriteError && err.outcome === "committed") ||
|
||||
(!startCommitted && !releaseReservation())
|
||||
) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
isGatewayRestartDraining() &&
|
||||
context.isCleanupGenerationCurrent(args.runId, args.entry, args.cleanupGeneration)
|
||||
) {
|
||||
scheduleResumeSubagentRun(
|
||||
context,
|
||||
args.runId,
|
||||
args.entry,
|
||||
MIN_ANNOUNCE_RETRY_DELAY_MS,
|
||||
args.cleanupGeneration,
|
||||
stateContext,
|
||||
);
|
||||
}
|
||||
})
|
||||
.finally(stopReservationObservation);
|
||||
});
|
||||
}
|
||||
|
||||
export function beginSubagentCleanup(
|
||||
context: SubagentLifecycleCleanupContext,
|
||||
runId: string,
|
||||
): { cleanupGeneration: number; stateContext: OpenClawStateWorkerContext } | undefined {
|
||||
const params = context.options;
|
||||
const entry = params.runs.get(runId);
|
||||
if (
|
||||
!entry ||
|
||||
entry.pauseReason === "sessions_yield" ||
|
||||
entry.cleanupCompletedAt ||
|
||||
entry.cleanupHandled
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
// Failed source capture must not leave a reservation without an admitted driver.
|
||||
const stateContext = captureOpenClawStateWorkerContext();
|
||||
entry.cleanupHandled = true;
|
||||
return { cleanupGeneration: context.bumpCleanupGeneration(entry), stateContext };
|
||||
}
|
||||
|
||||
export async function retireSupersededCleanupIfNeeded(
|
||||
context: SubagentLifecycleCleanupContext,
|
||||
runId: string,
|
||||
entry: SubagentRunRecord,
|
||||
generation: number,
|
||||
): Promise<boolean> {
|
||||
const params = context.options;
|
||||
if (
|
||||
params.runs.get(runId) !== entry ||
|
||||
!context.isCleanupGeneration(entry, generation) ||
|
||||
!context.newerGenerationOwnsSession(entry)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
// Cleanup can yield to attachment, mirror, or announce work. A successor
|
||||
// registered while it was suspended owns every session-scoped side effect.
|
||||
await params.retireSupersededRun(runId, entry);
|
||||
return true;
|
||||
}
|
||||
|
|
@ -0,0 +1,207 @@
|
|||
import { clearGatewayContextResolver } from "../../../plugins/runtime/gateway-request-scope.js";
|
||||
import { runWithGatewayDetachedWorkAdmission } from "../../../process/gateway-work-admission.js";
|
||||
import { defaultRuntime } from "../../../runtime.js";
|
||||
import { captureOpenClawStateWorkerContext } from "../../../state/openclaw-state-worker-context.js";
|
||||
import { retireSessionMcpRuntimeForSessionKey } from "../../agent-bundle-mcp-tools.js";
|
||||
import { removeInternalSessionEffectsSession } from "../../internal-session-effects.js";
|
||||
import { markRequesterSettleWakePending } from "./subagent-delivery-state.js";
|
||||
import { SUBAGENT_ENDED_REASON_KILLED } from "./subagent-lifecycle-events.js";
|
||||
import type {
|
||||
CleanupBookkeepingParams,
|
||||
SubagentLifecycleWakeContext,
|
||||
} from "./subagent-registry-lifecycle-context.js";
|
||||
import {
|
||||
buildSafeLifecycleErrorMeta,
|
||||
maskLifecycleIdentifier,
|
||||
} from "./subagent-registry-lifecycle-delivery.js";
|
||||
import { commitSubagentLifecycleMutation } from "./subagent-registry-lifecycle-persistence.js";
|
||||
import { scheduleRequesterSettleWake } from "./subagent-registry-lifecycle-wake.js";
|
||||
import { subagentRuns } from "./subagent-registry-memory.js";
|
||||
import { assertSubagentRegistryWriteSourceCurrent } from "./subagent-registry-persistence.js";
|
||||
|
||||
function applyCleanupBookkeeping(
|
||||
cleanup: CleanupBookkeepingParams,
|
||||
suppressSessionEffects: boolean,
|
||||
retireAfterSettle: boolean,
|
||||
): void {
|
||||
const { entry } = cleanup;
|
||||
entry.cleanupCompletedAt = cleanup.completedAt;
|
||||
if (suppressSessionEffects) {
|
||||
entry.execution = {
|
||||
...entry.execution,
|
||||
restartRecovery: undefined,
|
||||
suppressSessionEffects: true,
|
||||
};
|
||||
entry.terminalOwner = undefined;
|
||||
}
|
||||
if (entry.collect) {
|
||||
entry.requesterSettleWake = undefined;
|
||||
} else if (!cleanup.skipRequesterSettleWake) {
|
||||
markRequesterSettleWakePending(entry, { retireAfterSettle });
|
||||
}
|
||||
}
|
||||
|
||||
export async function completeCleanupBookkeeping(
|
||||
context: SubagentLifecycleWakeContext,
|
||||
cleanupParams: CleanupBookkeepingParams,
|
||||
): Promise<void> {
|
||||
const params = context.options;
|
||||
const stateContext = cleanupParams.stateContext ?? captureOpenClawStateWorkerContext();
|
||||
// Bookkeeping can retire the row; detached child effects refresh currency below.
|
||||
const suppressSessionEffects = !context.sessionEffectsHostCurrent(cleanupParams.entry);
|
||||
const assertCurrent = () => {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
if (
|
||||
cleanupParams.isCurrent?.() === false ||
|
||||
context.sessionEffectsHostCurrent(cleanupParams.entry) === suppressSessionEffects
|
||||
) {
|
||||
throw new Error("Subagent cleanup owner changed before bookkeeping.");
|
||||
}
|
||||
};
|
||||
const scheduleCleanupTails = (options: {
|
||||
allowRetiredRow: boolean;
|
||||
isDeleteCleanup: boolean;
|
||||
}) => {
|
||||
// Retained bookkeeping requires the exact row. Immediate retirement
|
||||
// removes it first, so absence remains ownership only while no newer
|
||||
// child generation exists; any replacement blocks the stale cleanup.
|
||||
const postBookkeepingEffectsAllowed = () => {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
const current = params.runs.get(cleanupParams.runId);
|
||||
const rowOwnershipMatches =
|
||||
current === cleanupParams.entry || (options.allowRetiredRow && current === undefined);
|
||||
return (
|
||||
rowOwnershipMatches &&
|
||||
cleanupParams.isCurrent?.() !== false &&
|
||||
!context.newerGenerationOwnsSession(cleanupParams.entry) &&
|
||||
context.sessionEffectsHostCurrent(cleanupParams.entry)
|
||||
);
|
||||
};
|
||||
const runCleanupTail = (label: string, run: () => Promise<unknown>) => {
|
||||
// Admission can outlive the caller's async scope. Own the tail's lifetime
|
||||
// and recheck row ownership after waiting; surviving tails still block snapshots.
|
||||
void runWithGatewayDetachedWorkAdmission(async () => {
|
||||
if (
|
||||
!(await context.shouldSuppressSessionEffects(cleanupParams.entry)) &&
|
||||
postBookkeepingEffectsAllowed()
|
||||
) {
|
||||
await run();
|
||||
}
|
||||
}, "subagents:lifecycle-cleanup").catch((error: unknown) => {
|
||||
defaultRuntime.log(
|
||||
`[warn] subagent ${label} failed (${cleanupParams.runId}): ${String(error)}`,
|
||||
);
|
||||
});
|
||||
};
|
||||
if (postBookkeepingEffectsAllowed() && !cleanupParams.preserveTranscript) {
|
||||
runCleanupTail("session cleanup", () =>
|
||||
removeInternalSessionEffectsSession(cleanupParams.entry.execution.transcriptTarget),
|
||||
);
|
||||
}
|
||||
if (postBookkeepingEffectsAllowed() && cleanupParams.entry.spawnMode !== "session") {
|
||||
runCleanupTail("bundle MCP cleanup", () =>
|
||||
retireSessionMcpRuntimeForSessionKey({
|
||||
sessionKey: cleanupParams.entry.childSessionKey,
|
||||
reason: "subagent-run-cleanup",
|
||||
preserveActiveLeases: true,
|
||||
onError: (error, sessionId) => {
|
||||
params.warn("failed to retire subagent bundle MCP runtime", {
|
||||
error: buildSafeLifecycleErrorMeta(error),
|
||||
sessionId,
|
||||
runId: maskLifecycleIdentifier(cleanupParams.runId, "run"),
|
||||
childSessionKey: maskLifecycleIdentifier(
|
||||
cleanupParams.entry.childSessionKey,
|
||||
"session",
|
||||
),
|
||||
});
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (
|
||||
!cleanupParams.provisionalKill &&
|
||||
postBookkeepingEffectsAllowed() &&
|
||||
(options.isDeleteCleanup || !cleanupParams.entry.collect)
|
||||
) {
|
||||
runCleanupTail("context-engine cleanup", () =>
|
||||
params.notifyContextEngineSubagentEnded(
|
||||
{
|
||||
childSessionKey: cleanupParams.entry.childSessionKey,
|
||||
reason: options.isDeleteCleanup ? "deleted" : "completed",
|
||||
agentDir: cleanupParams.entry.agentDir,
|
||||
workspaceDir: cleanupParams.entry.workspaceDir,
|
||||
},
|
||||
{
|
||||
isCurrent: postBookkeepingEffectsAllowed,
|
||||
prepareCurrent: async () =>
|
||||
!(await context.shouldSuppressSessionEffects(cleanupParams.entry)) &&
|
||||
postBookkeepingEffectsAllowed(),
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
};
|
||||
if (cleanupParams.provisionalKill) {
|
||||
// The provider result or bounded kill reconciliation owns terminal settle.
|
||||
// Its kill marker was committed by the caller before reaching this tail.
|
||||
scheduleCleanupTails({ allowRetiredRow: false, isDeleteCleanup: false });
|
||||
return;
|
||||
}
|
||||
const isDeleteCleanup = cleanupParams.cleanup === "delete";
|
||||
const retireAfterSettle =
|
||||
!cleanupParams.entry.collect &&
|
||||
(isDeleteCleanup ||
|
||||
(cleanupParams.entry.endedReason === SUBAGENT_ENDED_REASON_KILLED &&
|
||||
cleanupParams.entry.suppressAnnounceReason !== "killed"));
|
||||
const retireImmediately = retireAfterSettle && cleanupParams.skipRequesterSettleWake === true;
|
||||
const assertPublishedOwner = () => {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
const current = params.runs.get(cleanupParams.runId);
|
||||
if (
|
||||
cleanupParams.isCurrent?.() === false ||
|
||||
(retireImmediately ? current !== undefined : current !== cleanupParams.entry)
|
||||
) {
|
||||
throw new Error("Subagent cleanup owner changed after publication.");
|
||||
}
|
||||
};
|
||||
if (retireImmediately) {
|
||||
await commitSubagentLifecycleMutation(context, {
|
||||
entry: cleanupParams.entry,
|
||||
stateContext,
|
||||
assertCurrent,
|
||||
mutate: () => cleanupParams.discardDelivery?.(),
|
||||
retire: true,
|
||||
});
|
||||
assertPublishedOwner();
|
||||
subagentRuns.confirmRetirement(cleanupParams.entry);
|
||||
clearGatewayContextResolver(cleanupParams.entry);
|
||||
} else {
|
||||
// Collector tombstones and announcing runs share the same durable cleanup
|
||||
// boundary; only announcing runs keep a requester-settle obligation.
|
||||
await commitSubagentLifecycleMutation(context, {
|
||||
entry: cleanupParams.entry,
|
||||
stateContext,
|
||||
assertCurrent,
|
||||
mutate: () => {
|
||||
cleanupParams.discardDelivery?.();
|
||||
applyCleanupBookkeeping(cleanupParams, suppressSessionEffects, retireAfterSettle);
|
||||
},
|
||||
});
|
||||
assertPublishedOwner();
|
||||
if (cleanupParams.entry.collect || cleanupParams.skipRequesterSettleWake) {
|
||||
clearGatewayContextResolver(cleanupParams.entry);
|
||||
}
|
||||
}
|
||||
if (isDeleteCleanup || retireAfterSettle) {
|
||||
params.clearPendingLifecycleError(cleanupParams.runId);
|
||||
}
|
||||
// A settle wake may retire its durably marked row before detached tails start.
|
||||
// A replacement row or newer child generation still fences these effects.
|
||||
scheduleCleanupTails({ allowRetiredRow: retireAfterSettle, isDeleteCleanup });
|
||||
assertPublishedOwner();
|
||||
context.resumeAncestorCleanup(cleanupParams.entry);
|
||||
if (!cleanupParams.entry.collect && !cleanupParams.skipRequesterSettleWake) {
|
||||
assertPublishedOwner();
|
||||
scheduleRequesterSettleWake(context, cleanupParams.runId, cleanupParams.entry, stateContext);
|
||||
}
|
||||
}
|
||||
|
|
@ -132,6 +132,7 @@ export function registerDirectSessionCleanupAuthorityTests({
|
|||
createRunEntry,
|
||||
createLifecycleController,
|
||||
completeRun,
|
||||
completeAndJoinCleanup,
|
||||
gatewayMocks,
|
||||
helperMocks,
|
||||
sessionEntryReadMocks,
|
||||
|
|
@ -146,6 +147,11 @@ export function registerDirectSessionCleanupAuthorityTests({
|
|||
entry: SubagentRunRecord,
|
||||
options: Pick<SubagentCompletionRequest, "triggerCleanup" | "sessionEffects">,
|
||||
) => Promise<void>;
|
||||
completeAndJoinCleanup: (
|
||||
controller: SubagentLifecycleController,
|
||||
entry: SubagentRunRecord,
|
||||
options: Pick<SubagentCompletionRequest, "triggerCleanup" | "sessionEffects">,
|
||||
) => Promise<void>;
|
||||
gatewayMocks: {
|
||||
callGateway: Mock<(options: CallGatewayOptions) => Promise<Record<string, unknown>>>;
|
||||
};
|
||||
|
|
@ -261,7 +267,6 @@ export function registerDirectSessionCleanupAuthorityTests({
|
|||
suppressCompletionDelivery: true,
|
||||
});
|
||||
const runs = new Map([[entry.runId, entry]]);
|
||||
const retired = createDeferredCore();
|
||||
let current = true;
|
||||
sessionEntryReadMocks.loadSessionEntryByKey.mockImplementationOnce(async () => {
|
||||
current = false;
|
||||
|
|
@ -275,14 +280,9 @@ export function registerDirectSessionCleanupAuthorityTests({
|
|||
const controller = createLifecycleController({
|
||||
entry,
|
||||
runs,
|
||||
persistOrThrow: () => {
|
||||
if (!runs.has(entry.runId)) {
|
||||
retired.resolve();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
await completeRun(controller, entry, {
|
||||
await completeAndJoinCleanup(controller, entry, {
|
||||
triggerCleanup: true,
|
||||
sessionEffects: {
|
||||
isCurrent: async () => current,
|
||||
|
|
@ -290,7 +290,6 @@ export function registerDirectSessionCleanupAuthorityTests({
|
|||
assertCurrentEntry: assertCurrent,
|
||||
},
|
||||
});
|
||||
await retired.promise;
|
||||
|
||||
expect(gatewayMocks.callGateway).not.toHaveBeenCalled();
|
||||
expect(entry.execution.status).toBe("terminal");
|
||||
|
|
|
|||
|
|
@ -1,22 +1,16 @@
|
|||
import { runWithoutOwnedSessionTranscriptWrites } from "../../../config/sessions/transcript-write-context.js";
|
||||
import {
|
||||
isSystemEventStoreCurrent,
|
||||
recordSystemEventStoreReplaced,
|
||||
} from "../../../infra/system-event-ownership.js";
|
||||
import {
|
||||
isGatewayRestartDraining,
|
||||
runWithGatewayIndependentRootWorkAdmission,
|
||||
runWithGatewayIndependentRootWorkContinuation,
|
||||
} from "../../../process/gateway-work-admission.js";
|
||||
import { defaultRuntime } from "../../../runtime.js";
|
||||
import { withoutGatewayToolCallerIdentity } from "../../tools/gateway-caller-context.js";
|
||||
import type { OpenClawStateWorkerContext } from "../../../state/openclaw-state-worker-context.types.js";
|
||||
import { blockSubagentCompletionDelivery } from "../completion/subagent-completion-admission.store.js";
|
||||
import { getDeliveryLastError, isDeliverySuspended } from "./subagent-delivery-state.js";
|
||||
import { logAnnounceGiveUp } from "./subagent-registry-helpers.js";
|
||||
import {
|
||||
logAnnounceGiveUp,
|
||||
MIN_ANNOUNCE_RETRY_DELAY_MS,
|
||||
resolveAnnounceRetryDelayMs,
|
||||
} from "./subagent-registry-helpers.js";
|
||||
runWithSubagentCleanupWorkAdmission,
|
||||
retireSupersededCleanupIfNeeded,
|
||||
} from "./subagent-registry-lifecycle-attempt.js";
|
||||
import type {
|
||||
SubagentLifecycleAnnounceCleanupContext,
|
||||
SubagentLifecycleCleanupContext,
|
||||
|
|
@ -25,131 +19,11 @@ import type {
|
|||
} from "./subagent-registry-lifecycle-context.js";
|
||||
import { scheduleRequesterSettleWake } from "./subagent-registry-lifecycle-wake.js";
|
||||
import { subagentRuns } from "./subagent-registry-memory.js";
|
||||
import { assertSubagentRegistryWriteSourceCurrent } from "./subagent-registry-persistence.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
|
||||
const MAX_DETACHED_CLEANUP_RETRIES = 3;
|
||||
const pendingStoreRetirements = new WeakMap<SubagentRunRecord, Promise<void>>();
|
||||
|
||||
function runWithSubagentCleanupWorkAdmission<T>(run: () => Promise<T>): Promise<T> {
|
||||
// Restart remains one-way; only suspension preserves an admitted cleanup owner.
|
||||
// The registry owns cleanup after the spawning tool's caller has retired.
|
||||
return withoutGatewayToolCallerIdentity(() =>
|
||||
isGatewayRestartDraining()
|
||||
? runWithGatewayIndependentRootWorkAdmission(run, "subagents:lifecycle-cleanup")
|
||||
: runWithGatewayIndependentRootWorkContinuation(run, "subagents:lifecycle-cleanup"),
|
||||
);
|
||||
}
|
||||
|
||||
export function scheduleResumeSubagentRun(
|
||||
context: SubagentLifecycleCleanupContext,
|
||||
runId: string,
|
||||
entry: SubagentRunRecord,
|
||||
delayMs: number,
|
||||
cleanupGeneration?: number,
|
||||
): void {
|
||||
const params = context.options;
|
||||
const timer = setTimeout(() => {
|
||||
context.scheduledResumeTimers.delete(timer);
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
if (params.runs.get(runId) !== entry) {
|
||||
return;
|
||||
}
|
||||
if (cleanupGeneration !== undefined) {
|
||||
if (!context.isCleanupGenerationCurrent(runId, entry, cleanupGeneration)) {
|
||||
return;
|
||||
}
|
||||
if (entry.cleanupHandled) {
|
||||
entry.cleanupHandled = false;
|
||||
params.persist(runId);
|
||||
}
|
||||
}
|
||||
params.resumedRuns.delete(runId);
|
||||
params.resumeSubagentRun(runId);
|
||||
}, "subagents:resume").catch((err: unknown) => {
|
||||
defaultRuntime.log(`[warn] subagent cleanup resume failed (${runId}): ${String(err)}`);
|
||||
const current = params.runs.get(runId);
|
||||
if (
|
||||
isGatewayRestartDraining() &&
|
||||
current === entry &&
|
||||
typeof current.cleanupCompletedAt !== "number"
|
||||
) {
|
||||
scheduleResumeSubagentRun(
|
||||
context,
|
||||
runId,
|
||||
entry,
|
||||
Math.max(delayMs, MIN_ANNOUNCE_RETRY_DELAY_MS),
|
||||
cleanupGeneration,
|
||||
);
|
||||
}
|
||||
});
|
||||
}, delayMs);
|
||||
timer.unref?.();
|
||||
context.scheduledResumeTimers.add(timer);
|
||||
}
|
||||
|
||||
export function runDetachedCleanupAttempt(
|
||||
context: SubagentLifecycleCleanupContext,
|
||||
args: {
|
||||
runId: string;
|
||||
entry: SubagentRunRecord;
|
||||
cleanupGeneration: number;
|
||||
run: () => Promise<void>;
|
||||
},
|
||||
): void {
|
||||
const params = context.options;
|
||||
// Completion makes the task projection non-blocking before delivery and
|
||||
// cleanup finish. This independent lease bridges that handoff and owns the
|
||||
// full detached attempt, including its final durable registry write.
|
||||
// Completion outlives the spawning attempt; inherited lock owners would
|
||||
// reject requester transcript writes after that attempt is disposed.
|
||||
runWithoutOwnedSessionTranscriptWrites(() => {
|
||||
void runWithSubagentCleanupWorkAdmission(async () => {
|
||||
try {
|
||||
await args.run();
|
||||
context.cleanupFailureCounts.delete(args.entry);
|
||||
} catch (err) {
|
||||
defaultRuntime.log(
|
||||
`[warn] subagent cleanup finalize failed (${args.runId}): ${String(err)}`,
|
||||
);
|
||||
const current = params.runs.get(args.runId);
|
||||
if (
|
||||
!current ||
|
||||
current.cleanupCompletedAt ||
|
||||
!context.isCleanupAttemptCurrent(args.runId, args.entry, args.cleanupGeneration)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
current.cleanupHandled = false;
|
||||
params.resumedRuns.delete(args.runId);
|
||||
params.persist(args.runId);
|
||||
const failureCount = context.incrementCleanupFailureCount(current);
|
||||
if (failureCount <= MAX_DETACHED_CLEANUP_RETRIES) {
|
||||
scheduleResumeSubagentRun(
|
||||
context,
|
||||
args.runId,
|
||||
current,
|
||||
resolveAnnounceRetryDelayMs(failureCount),
|
||||
args.cleanupGeneration,
|
||||
);
|
||||
}
|
||||
}
|
||||
}).catch((err: unknown) => {
|
||||
defaultRuntime.log(
|
||||
`[warn] subagent cleanup admission failed (${args.runId}): ${String(err)}`,
|
||||
);
|
||||
if (isGatewayRestartDraining()) {
|
||||
scheduleResumeSubagentRun(
|
||||
context,
|
||||
args.runId,
|
||||
args.entry,
|
||||
MIN_ANNOUNCE_RETRY_DELAY_MS,
|
||||
args.cleanupGeneration,
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function suspendPendingFinalDelivery(
|
||||
context: SubagentLifecycleCleanupContext & SubagentLifecycleWakeContext,
|
||||
args: {
|
||||
|
|
@ -291,50 +165,16 @@ export function suspendReplacedStoreNotifications(
|
|||
return Promise.all(pending).then(() => {});
|
||||
}
|
||||
|
||||
export function beginSubagentCleanup(
|
||||
context: SubagentLifecycleCleanupContext,
|
||||
runId: string,
|
||||
): number | undefined {
|
||||
const params = context.options;
|
||||
const entry = params.runs.get(runId);
|
||||
if (!entry || entry.cleanupCompletedAt || entry.cleanupHandled) {
|
||||
return undefined;
|
||||
}
|
||||
entry.cleanupHandled = true;
|
||||
const generation = context.bumpCleanupGeneration(entry);
|
||||
params.persist(runId);
|
||||
return generation;
|
||||
}
|
||||
|
||||
export async function retireSupersededCleanupIfNeeded(
|
||||
context: SubagentLifecycleCleanupContext,
|
||||
runId: string,
|
||||
entry: SubagentRunRecord,
|
||||
generation: number,
|
||||
): Promise<boolean> {
|
||||
const params = context.options;
|
||||
if (
|
||||
params.runs.get(runId) !== entry ||
|
||||
!context.isCleanupGeneration(entry, generation) ||
|
||||
!context.newerGenerationOwnsSession(entry)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
// Cleanup can yield to attachment, mirror, or announce work. A successor
|
||||
// registered while it was suspended owns every session-scoped side effect.
|
||||
await params.retireSupersededRun(runId, entry);
|
||||
return true;
|
||||
}
|
||||
|
||||
export function retireSupersededCleanupInBackground(
|
||||
context: SubagentLifecycleCleanupContext,
|
||||
runId: string,
|
||||
entry: SubagentRunRecord,
|
||||
generation: number,
|
||||
stateContext: OpenClawStateWorkerContext,
|
||||
): void {
|
||||
// Delivery callbacks are synchronous and may arrive after their announce
|
||||
// attempt returns. Give the async retirement tail its own snapshot blocker.
|
||||
// A late delivery callback still owns retirement through its original source.
|
||||
void runWithSubagentCleanupWorkAdmission(async () => {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
await retireSupersededCleanupIfNeeded(context, runId, entry, generation);
|
||||
}).catch((error: unknown) => {
|
||||
defaultRuntime.log(
|
||||
|
|
|
|||
|
|
@ -576,7 +576,7 @@ export function registerRequesterSettleRetirementTests({
|
|||
try {
|
||||
controller.startSubagentAnnounceCleanupFlow(ancestor.runId, ancestor);
|
||||
expect(ancestor.cleanupCompletedAt).toBeUndefined();
|
||||
controller.completeCleanupBookkeeping({
|
||||
await controller.completeCleanupBookkeeping({
|
||||
runId: intermediate.runId,
|
||||
entry: intermediate,
|
||||
cleanup: "delete",
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ export type SubagentLifecycleOptions = {
|
|||
...runIds: string[]
|
||||
) => Promise<void>;
|
||||
clearPendingLifecycleError(runId: string): void;
|
||||
countPendingDescendantRuns(rootSessionKey: string): number;
|
||||
countPendingDescendantRuns(rootSessionKey: string, assertCurrent: () => void): Promise<number>;
|
||||
getLatestRunForChildSession(
|
||||
childSessionKey: string,
|
||||
matches?: (entry: SubagentRunRecord) => boolean,
|
||||
|
|
@ -104,7 +104,7 @@ export interface SubagentLifecycleCleanupContext extends SubagentLifecycleCommon
|
|||
|
||||
export interface SubagentLifecycleAnnounceCleanupContext
|
||||
extends SubagentLifecycleCleanupContext, SubagentLifecycleWakeContext {
|
||||
completeCleanupBookkeeping(args: CleanupBookkeepingParams): void;
|
||||
completeCleanupBookkeeping(args: CleanupBookkeepingParams): Promise<void>;
|
||||
}
|
||||
|
||||
export type PendingRequesterSettleWakeCommit = {
|
||||
|
|
@ -169,6 +169,8 @@ export type CleanupBookkeepingParams = {
|
|||
preserveTranscript?: boolean;
|
||||
provisionalKill?: boolean;
|
||||
skipRequesterSettleWake?: boolean;
|
||||
isCurrent?: () => boolean;
|
||||
discardDelivery?: () => void;
|
||||
};
|
||||
|
||||
export type ScheduledRequesterSettleWake = {
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ export function createLifecycleControllerFixture(
|
|||
publication.onCommitted?.();
|
||||
},
|
||||
clearPendingLifecycleError: vi.fn(),
|
||||
countPendingDescendantRuns: () => 0,
|
||||
countPendingDescendantRuns: async () => 0,
|
||||
getLatestRunForChildSession: (key, matches) =>
|
||||
getLatestSubagentRunByChildSessionKeyFromRuns(runs, key, matches) ?? null,
|
||||
suppressAnnounceForSteerRestart: () => false,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,372 @@
|
|||
import { expect, it, vi } from "vitest";
|
||||
import { getActiveGatewayRootWorkCount } from "../../../process/gateway-work-admission.js";
|
||||
import { createDeferredCore } from "../../../shared/deferred.js";
|
||||
import {
|
||||
runSubagentAnnounceDispatch,
|
||||
type SubagentAnnounceDeliveryResult,
|
||||
} from "../announce/subagent-announce-dispatch.js";
|
||||
import type {
|
||||
SubagentLifecycleController,
|
||||
SubagentLifecycleOptions,
|
||||
} from "./subagent-registry-lifecycle.js";
|
||||
import { observeRootWork } from "./subagent-registry.browser-cleanup.test-support.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
|
||||
type LifecycleControllerParams = SubagentLifecycleOptions;
|
||||
type CompleteRun = (
|
||||
controller: SubagentLifecycleController,
|
||||
entry: SubagentRunRecord,
|
||||
overrides?: Omit<
|
||||
Partial<Parameters<SubagentLifecycleController["completeSubagentRun"]>[0]>,
|
||||
"runId"
|
||||
>,
|
||||
) => Promise<void>;
|
||||
|
||||
export function registerLifecycleDeliveryReceiptCases({
|
||||
createRunEntry,
|
||||
createLifecycleController,
|
||||
completeRun,
|
||||
completeAndJoinCleanup,
|
||||
waitForLifecycleState,
|
||||
}: {
|
||||
createRunEntry: (
|
||||
overrides?: Partial<SubagentRunRecord> & { endedAt?: number },
|
||||
) => SubagentRunRecord;
|
||||
createLifecycleController: (
|
||||
options: {
|
||||
entry: SubagentRunRecord;
|
||||
runs?: Map<string, SubagentRunRecord>;
|
||||
} & Partial<SubagentLifecycleOptions>,
|
||||
) => SubagentLifecycleController;
|
||||
completeRun: CompleteRun;
|
||||
completeAndJoinCleanup: CompleteRun;
|
||||
waitForLifecycleState: <T>(assertion: () => T | Promise<T>) => Promise<T>;
|
||||
}) {
|
||||
it("records completion announcement timestamps from transcript delivery", async () => {
|
||||
const persist = vi.fn();
|
||||
const entry = createRunEntry({
|
||||
expectsCompletionMessage: true,
|
||||
});
|
||||
const delivery: SubagentAnnounceDeliveryResult = {
|
||||
delivered: true,
|
||||
path: "steered",
|
||||
enqueuedAt: 4_100,
|
||||
deliveredAt: 12_300,
|
||||
};
|
||||
const runSubagentAnnounceFlow: LifecycleControllerParams["runSubagentAnnounceFlow"] = vi.fn(
|
||||
async (announceParams) => {
|
||||
await announceParams.onDeliveryResult?.(delivery);
|
||||
return "delivered" as const;
|
||||
},
|
||||
);
|
||||
|
||||
const controller = createLifecycleController({ entry, persist, runSubagentAnnounceFlow });
|
||||
|
||||
await expect(
|
||||
completeRun(controller, entry, {
|
||||
triggerCleanup: true,
|
||||
terminalReply: { disposition: "visible", text: "final completion reply" },
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
await waitForLifecycleState(() => expect(entry.delivery?.announcedAt).toBe(12_300));
|
||||
expect(entry.delivery?.enqueuedAt).toBe(4_100);
|
||||
expect(entry.delivery?.deliveredAt).toBe(12_300);
|
||||
expect(entry.delivery?.lastDropReason).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "persists steer_dropped when announce mapping preserves a live-queue refusal",
|
||||
delivery: {
|
||||
delivered: false as const,
|
||||
path: "none" as const,
|
||||
reason: "steer_dropped" as const,
|
||||
},
|
||||
lastDropReason: "steer_dropped",
|
||||
lastError: "steer_dropped",
|
||||
},
|
||||
{
|
||||
name: "persists sink_unavailable when announce mapping reports no viable requester",
|
||||
delivery: {
|
||||
delivered: false as const,
|
||||
path: "none" as const,
|
||||
},
|
||||
lastDropReason: "sink_unavailable",
|
||||
lastError: "delivery path none did not complete",
|
||||
},
|
||||
])("$name", async ({ delivery, lastDropReason, lastError }) => {
|
||||
const persist = vi.fn();
|
||||
const entry = createRunEntry({
|
||||
endedAt: 4_000,
|
||||
expectsCompletionMessage: true,
|
||||
retainAttachmentsOnKeep: true,
|
||||
});
|
||||
const runSubagentAnnounceFlow: LifecycleControllerParams["runSubagentAnnounceFlow"] = vi.fn(
|
||||
async (announceParams) => {
|
||||
await announceParams.onDeliveryResult?.(delivery);
|
||||
return "retryable" as const;
|
||||
},
|
||||
);
|
||||
|
||||
const controller = createLifecycleController({
|
||||
entry,
|
||||
persistOrThrow: persist,
|
||||
runSubagentAnnounceFlow,
|
||||
});
|
||||
|
||||
await expect(
|
||||
completeAndJoinCleanup(controller, entry, {
|
||||
triggerCleanup: true,
|
||||
terminalReply: { disposition: "visible", text: "final completion reply" },
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
await waitForLifecycleState(() => expect(entry.delivery?.lastDropReason).toBe(lastDropReason));
|
||||
expect(entry.delivery?.lastError).toBe(lastError);
|
||||
expect(entry.delivery?.status).toBe("suspended");
|
||||
expect(persist).toHaveBeenCalledWith(entry.runId);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "persists a newly failed completion",
|
||||
previousDropReason: undefined,
|
||||
reusePreviousError: false,
|
||||
persistCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "persists a changed drop reason when the direct error is unchanged",
|
||||
previousDropReason: "sink_unavailable" as const,
|
||||
reusePreviousError: true,
|
||||
persistCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "does not persist unchanged completion diagnostics",
|
||||
previousDropReason: "steer_dropped" as const,
|
||||
reusePreviousError: true,
|
||||
persistCalls: 0,
|
||||
},
|
||||
])("$name before stalled announce bookkeeping settles", async (scenario) => {
|
||||
const lastError = "failed; visible_reply_missing; direct-primary: failed";
|
||||
const persist = vi.fn();
|
||||
const entry = createRunEntry({
|
||||
endedAt: 4_000,
|
||||
expectsCompletionMessage: true,
|
||||
retainAttachmentsOnKeep: true,
|
||||
delivery: {
|
||||
status: "pending",
|
||||
...(scenario.reusePreviousError ? { lastError } : {}),
|
||||
...(scenario.previousDropReason ? { lastDropReason: scenario.previousDropReason } : {}),
|
||||
},
|
||||
});
|
||||
const receiptObserved = createDeferredCore();
|
||||
let releaseAnnounce!: () => void;
|
||||
const announcePending = new Promise<void>((resolve) => {
|
||||
releaseAnnounce = resolve;
|
||||
});
|
||||
const runSubagentAnnounceFlow: LifecycleControllerParams["runSubagentAnnounceFlow"] = vi.fn(
|
||||
async (announceParams) => {
|
||||
const delivery = await runSubagentAnnounceDispatch({
|
||||
expectsCompletionMessage: true,
|
||||
steer: async () => ({ status: "dropped" }),
|
||||
direct: async () => ({
|
||||
delivered: false,
|
||||
path: "direct",
|
||||
error: "failed",
|
||||
reason: "visible_reply_missing",
|
||||
}),
|
||||
});
|
||||
persist.mockClear();
|
||||
try {
|
||||
await announceParams.onDeliveryResult?.(delivery);
|
||||
receiptObserved.resolve();
|
||||
} catch (error) {
|
||||
receiptObserved.reject(error);
|
||||
throw error;
|
||||
}
|
||||
await announcePending;
|
||||
return "retryable" as const;
|
||||
},
|
||||
);
|
||||
const controller = createLifecycleController({
|
||||
entry,
|
||||
persistOrThrow: persist,
|
||||
runSubagentAnnounceFlow,
|
||||
});
|
||||
|
||||
const join = observeRootWork();
|
||||
try {
|
||||
await expect(
|
||||
completeRun(controller, entry, {
|
||||
triggerCleanup: true,
|
||||
terminalReply: { disposition: "visible", text: "final completion reply" },
|
||||
}),
|
||||
).resolves.toBeUndefined();
|
||||
await receiptObserved.promise;
|
||||
expect(entry.delivery?.disposition).toBe("retryable");
|
||||
expect(entry.delivery?.lastDropReason).toBe("steer_dropped");
|
||||
expect(entry.delivery?.lastError).toBe(lastError);
|
||||
expect(entry.cleanupCompletedAt).toBeUndefined();
|
||||
expect(persist).toHaveBeenCalledTimes(scenario.persistCalls);
|
||||
if (scenario.persistCalls > 0) {
|
||||
expect(persist).toHaveBeenCalledWith(entry.runId);
|
||||
}
|
||||
} finally {
|
||||
releaseAnnounce();
|
||||
await join();
|
||||
}
|
||||
expect(entry.delivery?.status).toBe("suspended");
|
||||
});
|
||||
|
||||
it("persists identified completion delivery while completing the active multipart send", async () => {
|
||||
const persist = vi.fn();
|
||||
const entry = createRunEntry({
|
||||
expectsCompletionMessage: true,
|
||||
delivery: {
|
||||
status: "pending",
|
||||
lastError: "earlier delivery failed",
|
||||
lastDropReason: "sink_unavailable",
|
||||
nextAttemptAt: 13_000,
|
||||
},
|
||||
});
|
||||
let releaseAnnounce!: () => void;
|
||||
const announcePending = new Promise<void>((resolve) => {
|
||||
releaseAnnounce = resolve;
|
||||
});
|
||||
const sentChunks: number[] = [];
|
||||
const chunksFinished = createDeferredCore();
|
||||
const runSubagentAnnounceFlow: LifecycleControllerParams["runSubagentAnnounceFlow"] = vi.fn(
|
||||
async (announceParams) => {
|
||||
try {
|
||||
for (const chunk of [1, 2, 3]) {
|
||||
if (announceParams.isCompletionDeliveryAllowed?.() === false) {
|
||||
break;
|
||||
}
|
||||
sentChunks.push(chunk);
|
||||
await announceParams.onDeliveryResult?.({
|
||||
delivered: true,
|
||||
path: "direct",
|
||||
deliveredAt: 12_300,
|
||||
});
|
||||
await Promise.resolve();
|
||||
}
|
||||
chunksFinished.resolve();
|
||||
} catch (error) {
|
||||
chunksFinished.reject(error);
|
||||
throw error;
|
||||
}
|
||||
await announcePending;
|
||||
return "delivered" as const;
|
||||
},
|
||||
);
|
||||
const controller = createLifecycleController({
|
||||
entry,
|
||||
persistOrThrow: persist,
|
||||
runSubagentAnnounceFlow,
|
||||
});
|
||||
|
||||
const join = observeRootWork();
|
||||
try {
|
||||
await completeRun(controller, entry, {
|
||||
triggerCleanup: true,
|
||||
terminalReply: { disposition: "visible", text: "final completion reply" },
|
||||
});
|
||||
await chunksFinished.promise;
|
||||
expect(entry.delivery?.status).toBe("delivered");
|
||||
|
||||
expect(entry.delivery).toMatchObject({
|
||||
status: "delivered",
|
||||
announcedAt: 12_300,
|
||||
deliveredAt: 12_300,
|
||||
});
|
||||
expect(entry.delivery?.lastError).toBeUndefined();
|
||||
expect(entry.delivery?.lastDropReason).toBeUndefined();
|
||||
expect(entry.cleanupCompletedAt).toBeUndefined();
|
||||
expect(persist).toHaveBeenCalledWith(entry.runId);
|
||||
expect.soft(sentChunks).toEqual([1, 2, 3]);
|
||||
} finally {
|
||||
releaseAnnounce();
|
||||
await join();
|
||||
}
|
||||
expect(entry.cleanupCompletedAt).toBeTypeOf("number");
|
||||
expect(entry.delivery?.nextAttemptAt).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps a late superseded-delivery retirement root-admitted", async () => {
|
||||
const entry = createRunEntry({ expectsCompletionMessage: true, generation: 1 });
|
||||
const runs = new Map([[entry.runId, entry]]);
|
||||
let onDeliveryResult: Parameters<
|
||||
LifecycleControllerParams["runSubagentAnnounceFlow"]
|
||||
>[0]["onDeliveryResult"];
|
||||
const runSubagentAnnounceFlow: LifecycleControllerParams["runSubagentAnnounceFlow"] = vi.fn(
|
||||
async (announceParams) => {
|
||||
onDeliveryResult = announceParams.onDeliveryResult;
|
||||
return "delivered" as const;
|
||||
},
|
||||
);
|
||||
let releaseRetirement = () => {};
|
||||
const retirementPending = new Promise<void>((resolve) => {
|
||||
releaseRetirement = resolve;
|
||||
});
|
||||
const retireSupersededRun = vi.fn(async () => {
|
||||
await retirementPending;
|
||||
});
|
||||
const controller = createLifecycleController({
|
||||
entry,
|
||||
runs,
|
||||
retireSupersededRun,
|
||||
runSubagentAnnounceFlow,
|
||||
});
|
||||
|
||||
await completeRun(controller, entry, { triggerCleanup: true });
|
||||
await waitForLifecycleState(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
const newer = createRunEntry({
|
||||
runId: "run-2",
|
||||
childSessionKey: entry.childSessionKey,
|
||||
generation: 2,
|
||||
});
|
||||
runs.set(newer.runId, newer);
|
||||
|
||||
await onDeliveryResult?.({ delivered: false, path: "none" });
|
||||
|
||||
await waitForLifecycleState(() =>
|
||||
expect(retireSupersededRun).toHaveBeenCalledWith(entry.runId, entry),
|
||||
);
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(1);
|
||||
releaseRetirement();
|
||||
await waitForLifecycleState(() => expect(getActiveGatewayRootWorkCount()).toBe(0));
|
||||
});
|
||||
|
||||
it("finalizes terminal visible-send failures without scheduling completion retry", async () => {
|
||||
const persist = vi.fn();
|
||||
const entry = createRunEntry({
|
||||
endedAt: 4_000,
|
||||
expectsCompletionMessage: true,
|
||||
retainAttachmentsOnKeep: true,
|
||||
});
|
||||
const runSubagentAnnounceFlow: LifecycleControllerParams["runSubagentAnnounceFlow"] = vi.fn(
|
||||
async (announceParams) => {
|
||||
await announceParams.onDeliveryResult?.({
|
||||
delivered: false,
|
||||
path: "direct",
|
||||
error: "prompt lock failed after visible send",
|
||||
terminal: true,
|
||||
});
|
||||
return "delivered" as const;
|
||||
},
|
||||
);
|
||||
|
||||
const controller = createLifecycleController({ entry, persist, runSubagentAnnounceFlow });
|
||||
|
||||
await expect(completeRun(controller, entry, { triggerCleanup: true })).resolves.toBeUndefined();
|
||||
|
||||
await waitForLifecycleState(() => expect(entry.cleanupCompletedAt).toBeTypeOf("number"));
|
||||
expect(entry.delivery?.status).toBe("delivered");
|
||||
expect(entry.delivery?.lastError).toBeUndefined();
|
||||
expect(entry.delivery?.payload).toBeUndefined();
|
||||
expect(entry.delivery?.suspendedAt).toBeUndefined();
|
||||
expect(entry.delivery?.suspendedReason).toBeUndefined();
|
||||
expect(runSubagentAnnounceFlow).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
}
|
||||
|
|
@ -12,6 +12,7 @@ import {
|
|||
} from "../../../plugins/runtime/gateway-request-scope.js";
|
||||
import { resolveAgentIdFromSessionKey } from "../../../routing/session-key.js";
|
||||
import { extractTextFromChatContent } from "../../../shared/chat-content.js";
|
||||
import { captureOpenClawStateWorkerContext } from "../../../state/openclaw-state-worker-context.js";
|
||||
import {
|
||||
buildAnnounceIdFromChildRun,
|
||||
buildAnnounceIdempotencyKey,
|
||||
|
|
@ -30,6 +31,8 @@ import type {
|
|||
SubagentLifecycleCommonContext,
|
||||
SubagentLifecycleOptions,
|
||||
} from "./subagent-registry-lifecycle-context.js";
|
||||
import { commitSubagentLifecycleMutation } from "./subagent-registry-lifecycle-persistence.js";
|
||||
import { assertSubagentRegistryWriteSourceCurrent } from "./subagent-registry-persistence.js";
|
||||
import type { PendingFinalDeliveryPayload } from "./subagent-registry-read.types.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
import { compareSubagentRunGeneration } from "./subagent-run-generation.js";
|
||||
|
|
@ -340,6 +343,15 @@ export const refreshFrozenResultFromSession = async (
|
|||
return false;
|
||||
}
|
||||
const generation = entry.generation;
|
||||
const stateContext = captureOpenClawStateWorkerContext();
|
||||
const previousResultText = entry.completion?.resultText;
|
||||
const previousCapturedAt = entry.completion?.capturedAt;
|
||||
const isCurrent = () =>
|
||||
params.runs.get(entry.runId) === entry &&
|
||||
entry.generation === generation &&
|
||||
entry.pauseReason !== "sessions_yield" &&
|
||||
entry.cleanupCompletedAt === undefined &&
|
||||
!context.newerGenerationOwnsSession(entry);
|
||||
|
||||
let captured: string | undefined;
|
||||
try {
|
||||
|
|
@ -355,22 +367,33 @@ export const refreshFrozenResultFromSession = async (
|
|||
}
|
||||
// Reply capture yields while registration can transfer session ownership.
|
||||
// Only the exact row and generation that started capture may commit its text.
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
if (
|
||||
params.runs.get(entry.runId) !== entry ||
|
||||
entry.generation !== generation ||
|
||||
context.newerGenerationOwnsSession(entry)
|
||||
!isCurrent() ||
|
||||
entry.completion?.resultText !== previousResultText ||
|
||||
entry.completion?.capturedAt !== previousCapturedAt
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const nextFrozen = capFrozenResultText(trimmed);
|
||||
const completion = ensureCompletionState(entry);
|
||||
if (completion.resultText === nextFrozen) {
|
||||
if (entry.completion?.resultText === nextFrozen) {
|
||||
return false;
|
||||
}
|
||||
completion.resultText = nextFrozen;
|
||||
completion.capturedAt = Date.now();
|
||||
params.persist(entry.runId);
|
||||
await commitSubagentLifecycleMutation(context, {
|
||||
entry,
|
||||
stateContext,
|
||||
assertCurrent() {
|
||||
if (!isCurrent()) {
|
||||
throw new Error("Subagent frozen-result owner changed before persistence.");
|
||||
}
|
||||
},
|
||||
mutate() {
|
||||
const completion = ensureCompletionState(entry);
|
||||
completion.resultText = nextFrozen;
|
||||
completion.capturedAt = Date.now();
|
||||
},
|
||||
});
|
||||
return true;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,270 @@
|
|||
import type { OpenClawStateWorkerContext } from "../../../state/openclaw-state-worker-context.types.js";
|
||||
import type { SubagentAnnounceFlowOutcome } from "../announce/subagent-announce.js";
|
||||
import {
|
||||
ensureCompletionState,
|
||||
ensureDeliveryState,
|
||||
clearSubagentPendingDelivery,
|
||||
loadPendingFinalDeliveryPayload,
|
||||
} from "./subagent-delivery-state.js";
|
||||
import {
|
||||
resolveCleanupCompletionReason,
|
||||
resolveDeferredCleanupDecision,
|
||||
} from "./subagent-registry-cleanup.js";
|
||||
import {
|
||||
ANNOUNCE_COMPLETION_HARD_EXPIRY_MS,
|
||||
ANNOUNCE_EXPIRY_MS,
|
||||
MIN_ANNOUNCE_RETRY_DELAY_MS,
|
||||
resolveAnnounceRetryDelayMs,
|
||||
safeRemoveAttachmentsDir,
|
||||
} from "./subagent-registry-helpers.js";
|
||||
import {
|
||||
retireSupersededCleanupIfNeeded,
|
||||
scheduleResumeSubagentRun,
|
||||
} from "./subagent-registry-lifecycle-attempt.js";
|
||||
import type { SubagentLifecycleAnnounceCleanupContext } from "./subagent-registry-lifecycle-context.js";
|
||||
import {
|
||||
emitCompletionEndedHookIfNeeded,
|
||||
markPendingFinalDelivery,
|
||||
} from "./subagent-registry-lifecycle-delivery.js";
|
||||
import { finalizeResumedAnnounceGiveUp } from "./subagent-registry-lifecycle-give-up.js";
|
||||
import { commitSubagentLifecycleMutation } from "./subagent-registry-lifecycle-persistence.js";
|
||||
import {
|
||||
assertSubagentRegistryWriteSourceCurrent,
|
||||
assertSubagentRegistryWriteOutcomeKnown,
|
||||
} from "./subagent-registry-persistence.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
|
||||
export const finalizeSubagentCleanup = async (
|
||||
context: SubagentLifecycleAnnounceCleanupContext,
|
||||
entry: SubagentRunRecord,
|
||||
cleanup: "delete" | "keep",
|
||||
announceOutcome: SubagentAnnounceFlowOutcome,
|
||||
cleanupGeneration: number,
|
||||
stateContext: OpenClawStateWorkerContext,
|
||||
options?: {
|
||||
skipAnnounce?: boolean;
|
||||
skipRequesterDelivery?: boolean;
|
||||
},
|
||||
) => {
|
||||
const params = context.options;
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
const { runId } = entry;
|
||||
if (params.runs.get(runId) !== entry) {
|
||||
return;
|
||||
}
|
||||
if (!context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration)) {
|
||||
await retireSupersededCleanupIfNeeded(context, runId, entry, cleanupGeneration);
|
||||
return;
|
||||
}
|
||||
const assertCurrent = () => {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
assertSubagentRegistryWriteOutcomeKnown([runId], stateContext.admission);
|
||||
if (!context.isCleanupGenerationCurrent(runId, entry, cleanupGeneration)) {
|
||||
throw new Error("Subagent cleanup generation changed before persistence.");
|
||||
}
|
||||
};
|
||||
const isCurrent = () => {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
return context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration);
|
||||
};
|
||||
const commit = (mutate: () => void, onPublished?: () => void) =>
|
||||
commitSubagentLifecycleMutation(context, {
|
||||
entry,
|
||||
stateContext,
|
||||
mutate,
|
||||
assertCurrent,
|
||||
onPublished,
|
||||
});
|
||||
assertCurrent();
|
||||
const skipRequesterDelivery =
|
||||
options?.skipRequesterDelivery === true || entry.suppressCompletionDelivery === true;
|
||||
const finishCleanup = async (
|
||||
skipRequesterSettleWake: boolean,
|
||||
completionReason?: ReturnType<typeof resolveCleanupCompletionReason>,
|
||||
) => {
|
||||
if (cleanup === "delete" || !entry.retainAttachmentsOnKeep) {
|
||||
await safeRemoveAttachmentsDir(entry, isCurrent);
|
||||
}
|
||||
if (!isCurrent()) {
|
||||
await retireSupersededCleanupIfNeeded(context, runId, entry, cleanupGeneration);
|
||||
return;
|
||||
}
|
||||
await context.completeCleanupBookkeeping({
|
||||
runId,
|
||||
entry,
|
||||
cleanup,
|
||||
completedAt: Date.now(),
|
||||
skipRequesterSettleWake,
|
||||
stateContext,
|
||||
isCurrent: () =>
|
||||
context.isCleanupGeneration(entry, cleanupGeneration) &&
|
||||
context.isEndedHookOwnerCurrent(runId, entry),
|
||||
});
|
||||
// Hook loading is best-effort; durable delivery and cleanup must already
|
||||
// be terminal before plugin code can fail or stall.
|
||||
const endedHookOwnerCurrent = () => {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
return (
|
||||
context.isCleanupGeneration(entry, cleanupGeneration) &&
|
||||
context.isEndedHookOwnerCurrent(runId, entry) &&
|
||||
context.sessionEffectsHostCurrent(entry)
|
||||
);
|
||||
};
|
||||
if (!(await context.shouldSuppressSessionEffects(entry)) && endedHookOwnerCurrent()) {
|
||||
await emitCompletionEndedHookIfNeeded(
|
||||
params,
|
||||
entry,
|
||||
completionReason ?? resolveCleanupCompletionReason(entry),
|
||||
endedHookOwnerCurrent,
|
||||
async () => !(await context.shouldSuppressSessionEffects(entry)) && endedHookOwnerCurrent(),
|
||||
);
|
||||
}
|
||||
};
|
||||
if (entry.expectsCompletionMessage === false || skipRequesterDelivery) {
|
||||
const intentionalNonDelivery = entry.delivery?.disposition === "intentional_non_delivery";
|
||||
await commit(() => {
|
||||
clearSubagentPendingDelivery(entry);
|
||||
if (skipRequesterDelivery) {
|
||||
const delivery = ensureDeliveryState(entry);
|
||||
delivery.status = "not_required";
|
||||
// Preserve the lifecycle owner's terminal fact after cleanup clears retry state.
|
||||
delivery.disposition = intentionalNonDelivery ? "intentional_non_delivery" : undefined;
|
||||
entry.suppressCompletionDelivery = undefined;
|
||||
}
|
||||
entry.wakeOnDescendantSettle = undefined;
|
||||
});
|
||||
await finishCleanup(skipRequesterDelivery);
|
||||
return;
|
||||
}
|
||||
if (announceOutcome === "delivered" || announceOutcome === "intentional_non_delivery") {
|
||||
const terminalNonDelivery =
|
||||
announceOutcome === "intentional_non_delivery" && entry.delivery?.status === "failed";
|
||||
await commit(() => {
|
||||
const delivery = ensureDeliveryState(entry);
|
||||
const shouldCreditDelivery = announceOutcome === "delivered";
|
||||
if (shouldCreditDelivery) {
|
||||
const deliveredAt = delivery.deliveredAt ?? delivery.announcedAt ?? Date.now();
|
||||
delivery.status = "delivered";
|
||||
delivery.deliveredAt = deliveredAt;
|
||||
delivery.announcedAt = delivery.announcedAt ?? deliveredAt;
|
||||
if (!options?.skipAnnounce) {
|
||||
delivery.announcedAt = deliveredAt;
|
||||
}
|
||||
clearSubagentPendingDelivery(entry);
|
||||
delivery.lastDropReason = undefined;
|
||||
} else {
|
||||
// A handoff stays pending for requester-settle; explicit suppression is
|
||||
// terminal and must not start another turn that overrides the decision.
|
||||
delivery.status = terminalNonDelivery ? "failed" : "pending";
|
||||
delivery.disposition = "intentional_non_delivery";
|
||||
delivery.payload = undefined;
|
||||
delivery.createdAt = undefined;
|
||||
delivery.attemptCount = undefined;
|
||||
delivery.nextAttemptAt = undefined;
|
||||
}
|
||||
entry.wakeOnDescendantSettle = undefined;
|
||||
const completion = ensureCompletionState(entry);
|
||||
completion.fallbackResultText = undefined;
|
||||
completion.fallbackCapturedAt = undefined;
|
||||
});
|
||||
await finishCleanup(terminalNonDelivery, resolveCleanupCompletionReason(entry));
|
||||
return;
|
||||
}
|
||||
|
||||
if (announceOutcome === "session_queued") {
|
||||
// The correlated queue owns transport now. Settlement, not admission,
|
||||
// decides delivered versus blocked and re-enters cleanup afterward.
|
||||
await commit(
|
||||
() => {
|
||||
entry.cleanupHandled = false;
|
||||
},
|
||||
() => params.resumedRuns.delete(runId),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const activeDescendantRuns = await params.countPendingDescendantRuns(
|
||||
entry.childSessionKey,
|
||||
assertCurrent,
|
||||
);
|
||||
assertCurrent();
|
||||
const now = Date.now();
|
||||
const deferredDecision = resolveDeferredCleanupDecision({
|
||||
entry,
|
||||
now,
|
||||
activeDescendantRuns: Math.max(0, activeDescendantRuns),
|
||||
announceExpiryMs: ANNOUNCE_EXPIRY_MS,
|
||||
announceCompletionHardExpiryMs: ANNOUNCE_COMPLETION_HARD_EXPIRY_MS,
|
||||
deferDescendantDelayMs: MIN_ANNOUNCE_RETRY_DELAY_MS,
|
||||
resolveAnnounceRetryDelayMs,
|
||||
});
|
||||
|
||||
if (deferredDecision.kind === "defer-descendants") {
|
||||
await commit(
|
||||
() => {
|
||||
ensureDeliveryState(entry).lastAttemptAt = now;
|
||||
entry.wakeOnDescendantSettle = true;
|
||||
entry.cleanupHandled = false;
|
||||
},
|
||||
() => params.resumedRuns.delete(runId),
|
||||
);
|
||||
scheduleResumeSubagentRun(
|
||||
context,
|
||||
runId,
|
||||
entry,
|
||||
deferredDecision.delayMs,
|
||||
cleanupGeneration,
|
||||
stateContext,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (deferredDecision.kind === "give-up") {
|
||||
await finalizeResumedAnnounceGiveUp(context, {
|
||||
runId,
|
||||
entry,
|
||||
reason: deferredDecision.reason,
|
||||
cleanup,
|
||||
cleanupGeneration,
|
||||
retryCount: deferredDecision.retryCount,
|
||||
completedAt: now,
|
||||
stateContext,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const requesterTurnPending = announceOutcome === "requester_turn_pending";
|
||||
let resumeDelayMs: number | undefined;
|
||||
await commit(
|
||||
() => {
|
||||
if (!requesterTurnPending) {
|
||||
markPendingFinalDelivery({
|
||||
entry,
|
||||
error: "announce deferred or direct delivery failed",
|
||||
});
|
||||
}
|
||||
const delivery = ensureDeliveryState(entry);
|
||||
delivery.status = "pending";
|
||||
delivery.payload ??= loadPendingFinalDeliveryPayload(entry);
|
||||
delivery.windowStartedAt ??= entry.execution.endedAt ?? now;
|
||||
delivery.deadlineAt ??= delivery.windowStartedAt + ANNOUNCE_COMPLETION_HARD_EXPIRY_MS;
|
||||
// An admitted requester still owns this delivery; observation is not another failed attempt.
|
||||
resumeDelayMs = requesterTurnPending
|
||||
? Math.min(MIN_ANNOUNCE_RETRY_DELAY_MS, delivery.deadlineAt - now)
|
||||
: deferredDecision.resumeDelayMs;
|
||||
delivery.nextAttemptAt = now + (resumeDelayMs ?? 0);
|
||||
entry.cleanupHandled = false;
|
||||
},
|
||||
() => params.resumedRuns.delete(runId),
|
||||
);
|
||||
if (resumeDelayMs != null) {
|
||||
scheduleResumeSubagentRun(
|
||||
context,
|
||||
runId,
|
||||
entry,
|
||||
resumeDelayMs,
|
||||
cleanupGeneration,
|
||||
stateContext,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
|
@ -1,20 +1,22 @@
|
|||
import { captureOpenClawStateWorkerContext } from "../../../state/openclaw-state-worker-context.js";
|
||||
import type { OpenClawStateWorkerContext } from "../../../state/openclaw-state-worker-context.types.js";
|
||||
import {
|
||||
getDeliveryLastError,
|
||||
clearSubagentPendingDelivery,
|
||||
ensureDeliveryState,
|
||||
ensureCompletionState,
|
||||
clearSubagentPendingDelivery,
|
||||
} from "./subagent-delivery-state.js";
|
||||
import {
|
||||
resolveCleanupCompletionReason,
|
||||
shouldSuspendPendingFinalDelivery,
|
||||
} from "./subagent-registry-cleanup.js";
|
||||
import { logAnnounceGiveUp, safeRemoveAttachmentsDir } from "./subagent-registry-helpers.js";
|
||||
import {
|
||||
suspendPendingFinalDelivery,
|
||||
retireSupersededCleanupIfNeeded,
|
||||
} from "./subagent-registry-lifecycle-cleanup.js";
|
||||
import { retireSupersededCleanupIfNeeded } from "./subagent-registry-lifecycle-attempt.js";
|
||||
import { suspendPendingFinalDelivery } from "./subagent-registry-lifecycle-cleanup.js";
|
||||
import type { SubagentLifecycleAnnounceCleanupContext } from "./subagent-registry-lifecycle-context.js";
|
||||
import { emitCompletionEndedHookIfNeeded } from "./subagent-registry-lifecycle-delivery.js";
|
||||
import { commitSubagentLifecycleMutation } from "./subagent-registry-lifecycle-persistence.js";
|
||||
import { assertSubagentRegistryWriteSourceCurrent } from "./subagent-registry-persistence.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
|
||||
export const finalizeResumedAnnounceGiveUp = async (
|
||||
|
|
@ -27,11 +29,26 @@ export const finalizeResumedAnnounceGiveUp = async (
|
|||
cleanupGeneration?: number;
|
||||
retryCount?: number;
|
||||
completedAt?: number;
|
||||
stateContext?: OpenClawStateWorkerContext;
|
||||
},
|
||||
) => {
|
||||
const params = context.options;
|
||||
const { runId, entry, reason, cleanup, cleanupGeneration, retryCount, completedAt } =
|
||||
giveUpParams;
|
||||
const stateContext = giveUpParams.stateContext ?? captureOpenClawStateWorkerContext();
|
||||
const generation = entry.generation;
|
||||
const isCurrent = () => {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
return (
|
||||
params.runs.get(runId) === entry &&
|
||||
entry.generation === generation &&
|
||||
(cleanupGeneration === undefined ||
|
||||
context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration))
|
||||
);
|
||||
};
|
||||
if (!isCurrent()) {
|
||||
return;
|
||||
}
|
||||
if (shouldSuspendPendingFinalDelivery(entry)) {
|
||||
await suspendPendingFinalDelivery(context, {
|
||||
runId,
|
||||
|
|
@ -42,48 +59,71 @@ export const finalizeResumedAnnounceGiveUp = async (
|
|||
return;
|
||||
}
|
||||
const deliveryError = getDeliveryLastError(entry) ?? reason;
|
||||
clearSubagentPendingDelivery(entry);
|
||||
const failedDelivery = ensureDeliveryState(entry);
|
||||
failedDelivery.status = "failed";
|
||||
failedDelivery.lastError = deliveryError;
|
||||
if (retryCount != null) {
|
||||
failedDelivery.attemptCount = retryCount;
|
||||
failedDelivery.lastAttemptAt = completedAt ?? Date.now();
|
||||
}
|
||||
entry.wakeOnDescendantSettle = undefined;
|
||||
const completion = ensureCompletionState(entry);
|
||||
completion.fallbackResultText = undefined;
|
||||
completion.fallbackCapturedAt = undefined;
|
||||
await commitSubagentLifecycleMutation(context, {
|
||||
entry,
|
||||
stateContext,
|
||||
assertCurrent() {
|
||||
if (
|
||||
cleanupGeneration !== undefined &&
|
||||
!context.isCleanupGenerationCurrent(runId, entry, cleanupGeneration)
|
||||
) {
|
||||
throw new Error("Subagent give-up owner changed before persistence.");
|
||||
}
|
||||
},
|
||||
mutate() {
|
||||
clearSubagentPendingDelivery(entry);
|
||||
const failedDelivery = ensureDeliveryState(entry);
|
||||
failedDelivery.status = "failed";
|
||||
failedDelivery.lastError = deliveryError;
|
||||
if (retryCount != null) {
|
||||
failedDelivery.attemptCount = retryCount;
|
||||
failedDelivery.lastAttemptAt = completedAt ?? Date.now();
|
||||
}
|
||||
entry.wakeOnDescendantSettle = undefined;
|
||||
const completion = ensureCompletionState(entry);
|
||||
completion.fallbackResultText = undefined;
|
||||
completion.fallbackCapturedAt = undefined;
|
||||
},
|
||||
});
|
||||
if ((cleanup ?? entry.cleanup) === "delete" || !entry.retainAttachmentsOnKeep) {
|
||||
await safeRemoveAttachmentsDir(entry);
|
||||
await safeRemoveAttachmentsDir(entry, isCurrent);
|
||||
}
|
||||
if (
|
||||
cleanupGeneration !== undefined &&
|
||||
!context.isCleanupAttemptCurrent(runId, entry, cleanupGeneration)
|
||||
) {
|
||||
await retireSupersededCleanupIfNeeded(context, runId, entry, cleanupGeneration);
|
||||
if (!isCurrent()) {
|
||||
if (cleanupGeneration !== undefined) {
|
||||
await retireSupersededCleanupIfNeeded(context, runId, entry, cleanupGeneration);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const completionReason = resolveCleanupCompletionReason(entry);
|
||||
logAnnounceGiveUp(entry, reason);
|
||||
// Retry-limit / expiry give-up should not leave cleanup stuck behind the
|
||||
// best-effort ended hook. Mark the run cleaned first, then fire the hook.
|
||||
context.completeCleanupBookkeeping({
|
||||
await context.completeCleanupBookkeeping({
|
||||
runId,
|
||||
entry,
|
||||
cleanup: cleanup ?? entry.cleanup,
|
||||
completedAt: completedAt ?? Date.now(),
|
||||
stateContext,
|
||||
isCurrent: () =>
|
||||
(cleanupGeneration === undefined || context.isCleanupGeneration(entry, cleanupGeneration)) &&
|
||||
context.isEndedHookOwnerCurrent(runId, entry),
|
||||
});
|
||||
if (!(await context.shouldSuppressSessionEffects(entry))) {
|
||||
const endedHookOwnerCurrent = () => {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
return (
|
||||
entry.generation === generation &&
|
||||
(cleanupGeneration === undefined || context.isCleanupGeneration(entry, cleanupGeneration)) &&
|
||||
context.isEndedHookOwnerCurrent(runId, entry) &&
|
||||
context.sessionEffectsHostCurrent(entry)
|
||||
);
|
||||
};
|
||||
if (!(await context.shouldSuppressSessionEffects(entry)) && endedHookOwnerCurrent()) {
|
||||
await emitCompletionEndedHookIfNeeded(
|
||||
params,
|
||||
entry,
|
||||
completionReason,
|
||||
() =>
|
||||
context.isEndedHookOwnerCurrent(runId, entry) && context.sessionEffectsHostCurrent(entry),
|
||||
async () =>
|
||||
!(await context.shouldSuppressSessionEffects(entry)) &&
|
||||
context.isEndedHookOwnerCurrent(runId, entry),
|
||||
endedHookOwnerCurrent,
|
||||
async () => !(await context.shouldSuppressSessionEffects(entry)) && endedHookOwnerCurrent(),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
import type { OpenClawStateWorkerContext } from "../../../state/openclaw-state-worker-context.types.js";
|
||||
import type { SubagentLifecycleCommonContext } from "./subagent-registry-lifecycle-context.js";
|
||||
import {
|
||||
assertSubagentRegistryWriteSourceCurrent,
|
||||
captureSubagentRunMutationSnapshot,
|
||||
publishSubagentRunPostimages,
|
||||
SubagentRegistryWriteError,
|
||||
} from "./subagent-registry-persistence.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
|
||||
/** Keep cleanup's speculative fields private until its original writer acknowledges them. */
|
||||
export async function commitSubagentLifecycleMutation(
|
||||
context: SubagentLifecycleCommonContext,
|
||||
args: {
|
||||
entry: SubagentRunRecord;
|
||||
stateContext: OpenClawStateWorkerContext;
|
||||
assertCurrent: () => void;
|
||||
mutate: () => void;
|
||||
retire?: boolean;
|
||||
previous?: SubagentRunRecord;
|
||||
onPublished?: () => void;
|
||||
},
|
||||
): Promise<void> {
|
||||
const assertCurrent = () => {
|
||||
assertSubagentRegistryWriteSourceCurrent(args.stateContext);
|
||||
args.assertCurrent();
|
||||
if (context.options.runs.get(args.entry.runId) !== args.entry) {
|
||||
throw new Error("Subagent cleanup lost its original registry row.");
|
||||
}
|
||||
};
|
||||
if (!args.previous) {
|
||||
assertCurrent();
|
||||
}
|
||||
const previous = args.previous ?? captureSubagentRunMutationSnapshot(args.entry);
|
||||
args.mutate();
|
||||
const published = await publishSubagentRunPostimages({
|
||||
runs: context.options.runs,
|
||||
previous: new Map([[args.entry, previous]]),
|
||||
retire: args.retire ? new Set([args.entry]) : undefined,
|
||||
persist: context.options.persistAsyncOrThrow,
|
||||
context: args.stateContext,
|
||||
assertCurrent,
|
||||
onPublished: args.onPublished,
|
||||
});
|
||||
if (published.publication === "superseded") {
|
||||
throw new SubagentRegistryWriteError(
|
||||
"committed",
|
||||
new Error("Subagent cleanup changed before publication."),
|
||||
"superseded",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -136,7 +136,7 @@ describe("requester settle retry lifetime", () => {
|
|||
persistedWakes.push(structuredClone(entry.requesterSettleWake));
|
||||
},
|
||||
clearPendingLifecycleError: vi.fn(),
|
||||
countPendingDescendantRuns: () => 0,
|
||||
countPendingDescendantRuns: async () => 0,
|
||||
getLatestRunForChildSession: () => null,
|
||||
suppressAnnounceForSteerRestart: () => false,
|
||||
shouldEmitEndedHookForRun: () => false,
|
||||
|
|
|
|||
|
|
@ -2,27 +2,16 @@ import { isDeepStrictEqual } from "node:util";
|
|||
import { runWithoutOwnedSessionTranscriptWrites } from "../../../config/sessions/transcript-write-context.js";
|
||||
import { hasSqliteWorkerOutcomeUnknown } from "../../../infra/sqlite-worker-contract.js";
|
||||
import { clearGatewayContextResolver } from "../../../plugins/runtime/gateway-request-scope.js";
|
||||
import {
|
||||
isGatewayRestartDrainError,
|
||||
runWithGatewayDetachedWorkAdmission,
|
||||
} from "../../../process/gateway-work-admission.js";
|
||||
import { defaultRuntime } from "../../../runtime.js";
|
||||
import { isGatewayRestartDrainError } from "../../../process/gateway-work-admission.js";
|
||||
import { captureOpenClawStateWorkerContext } from "../../../state/openclaw-state-worker-context.js";
|
||||
import type { OpenClawStateWorkerContext } from "../../../state/openclaw-state-worker-context.types.js";
|
||||
import { retireSessionMcpRuntimeForSessionKey } from "../../agent-bundle-mcp-tools.js";
|
||||
import { removeInternalSessionEffectsSession } from "../../internal-session-effects.js";
|
||||
import { resolveSubagentRequesterAgentId } from "../../subagent-requester-owner.js";
|
||||
import type { SubagentAnnounceDeliveryResult } from "../announce/subagent-announce-dispatch.js";
|
||||
import { settleRequesterCompletionBatch } from "../completion/subagent-completion-admission.store.js";
|
||||
import { revokeRequesterCronAuthorityBatch } from "../requester-cron-authority.js";
|
||||
import { revokeRequesterFinalAttachment } from "../requester-final-attachment.js";
|
||||
import {
|
||||
isCompletedRequesterDeliveryBlocked,
|
||||
markRequesterSettleWakePending,
|
||||
} from "./subagent-delivery-state.js";
|
||||
import { SUBAGENT_ENDED_REASON_KILLED } from "./subagent-lifecycle-events.js";
|
||||
import { isCompletedRequesterDeliveryBlocked } from "./subagent-delivery-state.js";
|
||||
import type {
|
||||
CleanupBookkeepingParams,
|
||||
PendingRequesterSettleWakeCommit,
|
||||
SubagentLifecycleWakeContext,
|
||||
} from "./subagent-registry-lifecycle-context.js";
|
||||
|
|
@ -250,43 +239,6 @@ export async function cancelRequesterSettleWake(
|
|||
}
|
||||
}
|
||||
|
||||
function persistCleanupBookkeeping(
|
||||
context: SubagentLifecycleWakeContext,
|
||||
cleanup: CleanupBookkeepingParams,
|
||||
suppressSessionEffects: boolean,
|
||||
retireAfterSettle: boolean,
|
||||
): void {
|
||||
const { entry } = cleanup;
|
||||
const previous = {
|
||||
cleanupCompletedAt: entry.cleanupCompletedAt,
|
||||
execution: entry.execution,
|
||||
terminalOwner: entry.terminalOwner,
|
||||
...(entry.collect || !cleanup.skipRequesterSettleWake
|
||||
? { requesterSettleWake: entry.requesterSettleWake }
|
||||
: {}),
|
||||
};
|
||||
entry.cleanupCompletedAt = cleanup.completedAt;
|
||||
if (suppressSessionEffects) {
|
||||
entry.execution = {
|
||||
...entry.execution,
|
||||
restartRecovery: undefined,
|
||||
suppressSessionEffects: true,
|
||||
};
|
||||
entry.terminalOwner = undefined;
|
||||
}
|
||||
if (entry.collect) {
|
||||
entry.requesterSettleWake = undefined;
|
||||
} else if (!cleanup.skipRequesterSettleWake) {
|
||||
markRequesterSettleWakePending(entry, { retireAfterSettle });
|
||||
}
|
||||
try {
|
||||
context.options.persistOrThrow(cleanup.runId);
|
||||
} catch (error) {
|
||||
Object.assign(entry, previous);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Once a child reaches a terminal settle, let the announce layer decide
|
||||
// whether its requester's batch has fully drained and, if so, wake the
|
||||
// registry-less top-level requester to synthesize. Settle bookkeeping never
|
||||
|
|
@ -596,138 +548,3 @@ export function scheduleRequesterSettleWake(
|
|||
});
|
||||
});
|
||||
}
|
||||
|
||||
export function completeCleanupBookkeeping(
|
||||
context: SubagentLifecycleWakeContext,
|
||||
cleanupParams: CleanupBookkeepingParams,
|
||||
): void {
|
||||
const stateContext = cleanupParams.stateContext ?? captureOpenClawStateWorkerContext();
|
||||
const params = context.options;
|
||||
// Bookkeeping can retire the row; detached child effects refresh currency below.
|
||||
const suppressSessionEffects = !context.sessionEffectsHostCurrent(cleanupParams.entry);
|
||||
const scheduleCleanupTails = (options: {
|
||||
allowRetiredRow: boolean;
|
||||
isDeleteCleanup: boolean;
|
||||
}) => {
|
||||
// Retained bookkeeping requires the exact row. Immediate retirement
|
||||
// removes it first, so absence remains ownership only while no newer
|
||||
// child generation exists; any replacement blocks the stale cleanup.
|
||||
const postBookkeepingEffectsAllowed = () => {
|
||||
const current = params.runs.get(cleanupParams.runId);
|
||||
const rowOwnershipMatches =
|
||||
current === cleanupParams.entry || (options.allowRetiredRow && current === undefined);
|
||||
return (
|
||||
rowOwnershipMatches &&
|
||||
!context.newerGenerationOwnsSession(cleanupParams.entry) &&
|
||||
context.sessionEffectsHostCurrent(cleanupParams.entry)
|
||||
);
|
||||
};
|
||||
const runCleanupTail = (label: string, run: () => Promise<unknown>) => {
|
||||
// Admission can outlive the caller's async scope. Own the tail's lifetime
|
||||
// and recheck row ownership after waiting; surviving tails still block snapshots.
|
||||
void runWithGatewayDetachedWorkAdmission(async () => {
|
||||
if (
|
||||
!(await context.shouldSuppressSessionEffects(cleanupParams.entry)) &&
|
||||
postBookkeepingEffectsAllowed()
|
||||
) {
|
||||
await run();
|
||||
}
|
||||
}, "subagents:lifecycle-cleanup").catch((error: unknown) => {
|
||||
defaultRuntime.log(
|
||||
`[warn] subagent ${label} failed (${cleanupParams.runId}): ${String(error)}`,
|
||||
);
|
||||
});
|
||||
};
|
||||
if (postBookkeepingEffectsAllowed() && !cleanupParams.preserveTranscript) {
|
||||
runCleanupTail("session cleanup", () =>
|
||||
removeInternalSessionEffectsSession(cleanupParams.entry.execution.transcriptTarget),
|
||||
);
|
||||
}
|
||||
if (postBookkeepingEffectsAllowed() && cleanupParams.entry.spawnMode !== "session") {
|
||||
runCleanupTail("bundle MCP cleanup", () =>
|
||||
retireSessionMcpRuntimeForSessionKey({
|
||||
sessionKey: cleanupParams.entry.childSessionKey,
|
||||
reason: "subagent-run-cleanup",
|
||||
preserveActiveLeases: true,
|
||||
onError: (error, sessionId) => {
|
||||
params.warn("failed to retire subagent bundle MCP runtime", {
|
||||
error: buildSafeLifecycleErrorMeta(error),
|
||||
sessionId,
|
||||
runId: maskLifecycleIdentifier(cleanupParams.runId, "run"),
|
||||
childSessionKey: maskLifecycleIdentifier(
|
||||
cleanupParams.entry.childSessionKey,
|
||||
"session",
|
||||
),
|
||||
});
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (
|
||||
!cleanupParams.provisionalKill &&
|
||||
postBookkeepingEffectsAllowed() &&
|
||||
(options.isDeleteCleanup || !cleanupParams.entry.collect)
|
||||
) {
|
||||
runCleanupTail("context-engine cleanup", () =>
|
||||
params.notifyContextEngineSubagentEnded(
|
||||
{
|
||||
childSessionKey: cleanupParams.entry.childSessionKey,
|
||||
reason: options.isDeleteCleanup ? "deleted" : "completed",
|
||||
agentDir: cleanupParams.entry.agentDir,
|
||||
workspaceDir: cleanupParams.entry.workspaceDir,
|
||||
},
|
||||
{
|
||||
isCurrent: postBookkeepingEffectsAllowed,
|
||||
prepareCurrent: async () =>
|
||||
!(await context.shouldSuppressSessionEffects(cleanupParams.entry)) &&
|
||||
postBookkeepingEffectsAllowed(),
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
};
|
||||
if (cleanupParams.provisionalKill) {
|
||||
// The provider result or bounded kill reconciliation owns terminal settle.
|
||||
// Its kill marker was committed by the caller before reaching this tail.
|
||||
scheduleCleanupTails({ allowRetiredRow: false, isDeleteCleanup: false });
|
||||
return;
|
||||
}
|
||||
const isDeleteCleanup = cleanupParams.cleanup === "delete";
|
||||
if (isDeleteCleanup) {
|
||||
params.clearPendingLifecycleError(cleanupParams.runId);
|
||||
}
|
||||
const retireAfterSettle =
|
||||
!cleanupParams.entry.collect &&
|
||||
(isDeleteCleanup ||
|
||||
(cleanupParams.entry.endedReason === SUBAGENT_ENDED_REASON_KILLED &&
|
||||
cleanupParams.entry.suppressAnnounceReason !== "killed"));
|
||||
// Reconciled keep-mode kills retire the registry row, not the child session.
|
||||
if (retireAfterSettle && !isDeleteCleanup) {
|
||||
params.clearPendingLifecycleError(cleanupParams.runId);
|
||||
}
|
||||
if (retireAfterSettle && cleanupParams.skipRequesterSettleWake) {
|
||||
params.runs.delete(cleanupParams.runId);
|
||||
try {
|
||||
params.persistOrThrow(cleanupParams.runId);
|
||||
} catch (error) {
|
||||
params.runs.set(cleanupParams.runId, cleanupParams.entry);
|
||||
throw error;
|
||||
}
|
||||
subagentRuns.confirmRetirement(cleanupParams.entry);
|
||||
clearGatewayContextResolver(cleanupParams.entry);
|
||||
} else {
|
||||
// Collector tombstones and announcing runs share the same durable cleanup
|
||||
// boundary; only announcing runs keep a requester-settle obligation.
|
||||
persistCleanupBookkeeping(context, cleanupParams, suppressSessionEffects, retireAfterSettle);
|
||||
if (cleanupParams.entry.collect || cleanupParams.skipRequesterSettleWake) {
|
||||
clearGatewayContextResolver(cleanupParams.entry);
|
||||
}
|
||||
}
|
||||
// A settle wake may retire its durably marked row before detached tails start.
|
||||
// A replacement row or newer child generation still fences these effects.
|
||||
scheduleCleanupTails({ allowRetiredRow: retireAfterSettle, isDeleteCleanup });
|
||||
context.resumeAncestorCleanup(cleanupParams.entry);
|
||||
if (!cleanupParams.entry.collect && !cleanupParams.skipRequesterSettleWake) {
|
||||
scheduleRequesterSettleWake(context, cleanupParams.runId, cleanupParams.entry, stateContext);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -26,6 +26,7 @@ import {
|
|||
resumeAncestorCleanup,
|
||||
startSubagentAnnounceCleanupFlow,
|
||||
} from "./subagent-registry-lifecycle-announce-cleanup.js";
|
||||
import { completeCleanupBookkeeping } from "./subagent-registry-lifecycle-bookkeeping.js";
|
||||
import { completeSubagentRunAttempt } from "./subagent-registry-lifecycle-completion.js";
|
||||
import type {
|
||||
CleanupBookkeepingParams,
|
||||
|
|
@ -37,7 +38,6 @@ import { refreshFrozenResultFromSession } from "./subagent-registry-lifecycle-de
|
|||
import { finalizeResumedAnnounceGiveUp } from "./subagent-registry-lifecycle-give-up.js";
|
||||
import {
|
||||
cancelRequesterSettleWake,
|
||||
completeCleanupBookkeeping,
|
||||
scheduleRequesterSettleWake,
|
||||
} from "./subagent-registry-lifecycle-wake.js";
|
||||
import { assertSubagentRegistryWriteSourceCurrent } from "./subagent-registry-persistence.js";
|
||||
|
|
@ -220,7 +220,11 @@ export class SubagentLifecycleController {
|
|||
entry.cleanupHandled === true && this.isCleanupGenerationCurrent(runId, entry, generation);
|
||||
isEndedHookOwnerCurrent = (runId: string, entry: SubagentRunRecord): boolean => {
|
||||
const current = this.options.runs.get(runId);
|
||||
return (current === undefined || current === entry) && !this.newerGenerationOwnsSession(entry);
|
||||
return (
|
||||
(current === undefined || current === entry) &&
|
||||
entry.pauseReason !== "sessions_yield" &&
|
||||
!this.newerGenerationOwnsSession(entry)
|
||||
);
|
||||
};
|
||||
|
||||
bumpTerminalGeneration(entry: SubagentRunRecord): number {
|
||||
|
|
@ -293,7 +297,7 @@ export class SubagentLifecycleController {
|
|||
};
|
||||
|
||||
completeCleanupBookkeeping = (params: CleanupBookkeepingParams) => {
|
||||
completeCleanupBookkeeping(this, params);
|
||||
return completeCleanupBookkeeping(this, params);
|
||||
};
|
||||
|
||||
resumeAncestorCleanup = (settledEntry: SubagentRunRecord): void =>
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ function matchesSubagentCacheAdmission(
|
|||
}
|
||||
}
|
||||
|
||||
export function applySubagentRunChanges<T extends SubagentRunReadRecord>(
|
||||
function applySubagentRunChanges<T extends SubagentRunReadRecord>(
|
||||
runs: Map<string, T>,
|
||||
changes: Map<string, SubagentRunChange<T>> | undefined,
|
||||
): Map<string, T> {
|
||||
|
|
@ -133,7 +133,7 @@ export function applySubagentRunChanges<T extends SubagentRunReadRecord>(
|
|||
return runs;
|
||||
}
|
||||
|
||||
export function retainUnpublishedSubagentChanges<T>(
|
||||
function retainUnpublishedSubagentChanges<T>(
|
||||
changes: Map<string, SubagentRunChange<T>> | undefined,
|
||||
) {
|
||||
for (const [runId, change] of changes ?? []) {
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ import { getLatestSubagentRunByChildSessionKeyFromRuns } from "./subagent-regist
|
|||
import {
|
||||
clearSubagentRunsReadCacheForTest,
|
||||
getSubagentRunsSnapshotForRead,
|
||||
getSubagentRunsSnapshotForSessions,
|
||||
persistSubagentRunsToDiskOrThrow,
|
||||
persistSubagentRunsToDisk,
|
||||
withSubagentRunReadSnapshot,
|
||||
|
|
@ -66,12 +65,12 @@ afterEach(async () => {
|
|||
|
||||
describe("prepared subagent publication ownership", () => {
|
||||
it.each([
|
||||
{ warm: false, tree: false },
|
||||
{ warm: true, tree: false },
|
||||
{ warm: false, tree: true },
|
||||
{ warm: false, preparedFirst: false },
|
||||
{ warm: true, preparedFirst: false },
|
||||
{ warm: false, preparedFirst: true },
|
||||
])(
|
||||
"retains failed named deletion through hydration (warm=$warm, tree=$tree) and clears only exact successful rows",
|
||||
async ({ warm, tree }) => {
|
||||
"retains failed named deletion through hydration (warm=$warm, prepared first=$preparedFirst) and clears only exact successful rows",
|
||||
async ({ warm, preparedFirst }) => {
|
||||
const first = run("first", 100);
|
||||
const second = run("second", 200);
|
||||
saveSubagentRegistryToSqlite(
|
||||
|
|
@ -90,8 +89,8 @@ describe("prepared subagent publication ownership", () => {
|
|||
});
|
||||
persistSubagentRunsToDisk(new Map(), [first.runId, second.runId]);
|
||||
fail.mockRestore();
|
||||
if (tree) {
|
||||
expect(getSubagentRunsSnapshotForSessions(new Map(), [childSessionKey]).size).toBe(0);
|
||||
if (preparedFirst) {
|
||||
expect(await readLatest()).toBeNull();
|
||||
}
|
||||
expect(getSubagentRunsSnapshotForRead(new Map()).size).toBe(0);
|
||||
expect(store.loadSubagentRegistryFromSqlite().size).toBe(2);
|
||||
|
|
|
|||
|
|
@ -1,10 +1,16 @@
|
|||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
countActiveDescendantRunsFromRuns,
|
||||
hasDescendantRunAwaitingSettleFromRuns,
|
||||
} from "./subagent-registry-queries.js";
|
||||
import type { withSubagentRunReadSnapshot } from "./subagent-registry-state.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
|
||||
const mocks = vi.hoisted(() => {
|
||||
const liveRuns = new Map<string, SubagentRunRecord>();
|
||||
return {
|
||||
liveRuns,
|
||||
readSnapshot: new Map<string, SubagentRunRecord>(),
|
||||
getSubagentRunsForChildSession: vi.fn<(childSessionKey: string) => Iterable<SubagentRunRecord>>(
|
||||
() => [],
|
||||
),
|
||||
|
|
@ -28,12 +34,6 @@ const mocks = vi.hoisted(() => {
|
|||
>(() => {
|
||||
throw new Error("unexpected full registry hydration");
|
||||
}),
|
||||
getSubagentRunsSnapshotForSessions: vi.fn<
|
||||
(
|
||||
runs: Map<string, SubagentRunRecord>,
|
||||
keys: readonly string[],
|
||||
) => Map<string, SubagentRunRecord>
|
||||
>(() => new Map()),
|
||||
};
|
||||
});
|
||||
|
||||
|
|
@ -47,7 +47,13 @@ vi.mock("./subagent-registry-state.js", () => ({
|
|||
getSubagentRunsSnapshotForChildSession: mocks.getSubagentRunsSnapshotForChildSession,
|
||||
getSubagentRunsSnapshotForController: mocks.getSubagentRunsSnapshotForController,
|
||||
getSubagentRunsSnapshotForRead: mocks.getSubagentRunsSnapshotForRead,
|
||||
getSubagentRunsSnapshotForSessions: mocks.getSubagentRunsSnapshotForSessions,
|
||||
withSubagentRunReadSnapshot: (async (_runs, select, consume) => {
|
||||
const selected = select(mocks.readSnapshot);
|
||||
return consume(
|
||||
selected,
|
||||
new Map([...mocks.readSnapshot].filter(([runId]) => selected.runIds.includes(runId))),
|
||||
);
|
||||
}) satisfies typeof withSubagentRunReadSnapshot,
|
||||
}));
|
||||
|
||||
function createRun(overrides: Partial<SubagentRunRecord>): SubagentRunRecord {
|
||||
|
|
@ -71,11 +77,11 @@ describe("subagent registry scoped reads", () => {
|
|||
|
||||
beforeEach(async () => {
|
||||
mocks.liveRuns.clear();
|
||||
mocks.readSnapshot.clear();
|
||||
mocks.getSubagentRunsForChildSession.mockReset().mockReturnValue([]);
|
||||
mocks.getSubagentSessionListRunsSnapshotForRead.mockReset().mockReturnValue(new Map());
|
||||
mocks.getSubagentRunsSnapshotForChildSession.mockReset().mockReturnValue(new Map());
|
||||
mocks.getSubagentRunsSnapshotForController.mockReset().mockReturnValue(new Map());
|
||||
mocks.getSubagentRunsSnapshotForSessions.mockReset().mockReturnValue(new Map());
|
||||
mocks.getSubagentRunsSnapshotForRead.mockReset().mockImplementation(() => {
|
||||
throw new Error("unexpected full registry hydration");
|
||||
});
|
||||
|
|
@ -156,7 +162,6 @@ describe("subagent registry scoped reads", () => {
|
|||
for (const run of runs) {
|
||||
mocks.liveRuns.set(run.runId, run);
|
||||
}
|
||||
mocks.getSubagentRunsSnapshotForSessions.mockReturnValue(new Map(mocks.liveRuns));
|
||||
expect(
|
||||
mod
|
||||
.listSubagentRunsForRequester(root, {
|
||||
|
|
@ -165,8 +170,12 @@ describe("subagent registry scoped reads", () => {
|
|||
})
|
||||
.map((run) => run.runId),
|
||||
).toEqual(ids);
|
||||
expect(mod.countActiveDescendantRuns(root, "main", storePath)).toBe(active);
|
||||
expect(mod.hasDescendantRunAwaitingSettle(root, excluded, "main", storePath)).toBe(waiting);
|
||||
expect(countActiveDescendantRunsFromRuns(mocks.liveRuns, root, "main", storePath)).toBe(
|
||||
active,
|
||||
);
|
||||
expect(
|
||||
hasDescendantRunAwaitingSettleFromRuns(mocks.liveRuns, root, excluded, "main", storePath),
|
||||
).toBe(waiting);
|
||||
},
|
||||
);
|
||||
|
||||
|
|
@ -250,7 +259,7 @@ describe("subagent registry scoped reads", () => {
|
|||
|
||||
it.each(["delivered", "intentional_non_delivery", "permanent_failure", "ambiguous"] as const)(
|
||||
"reads %s settlement and descendant counts without hydrating unrelated payloads",
|
||||
(disposition) => {
|
||||
async (disposition) => {
|
||||
const root = "agent:main:root";
|
||||
const run = createRun({
|
||||
requesterSessionKey: root,
|
||||
|
|
@ -258,18 +267,20 @@ describe("subagent registry scoped reads", () => {
|
|||
execution: { status: "terminal", endedAt: 100 },
|
||||
delivery: { status: "pending", disposition },
|
||||
});
|
||||
mocks.getSubagentRunsSnapshotForSessions.mockReturnValue(new Map([[run.runId, run]]));
|
||||
expect(mod.countActiveDescendantRuns(root, "main")).toBe(0);
|
||||
expect(mod.countPendingDescendantRuns(root)).toBe(1);
|
||||
expect(mod.hasDescendantRunAwaitingSettle(root, undefined, "main")).toBe(
|
||||
disposition === "ambiguous",
|
||||
);
|
||||
expect(mod.hasDescendantRunAwaitingSettle(root, run.runId, "main")).toBe(false);
|
||||
mocks.readSnapshot.set(run.runId, run);
|
||||
expect(countActiveDescendantRunsFromRuns(mocks.readSnapshot, root, "main")).toBe(0);
|
||||
expect(await mod.countPendingDescendantRuns(root, () => {})).toBe(1);
|
||||
expect(
|
||||
hasDescendantRunAwaitingSettleFromRuns(mocks.readSnapshot, root, undefined, "main"),
|
||||
).toBe(disposition === "ambiguous");
|
||||
expect(
|
||||
hasDescendantRunAwaitingSettleFromRuns(mocks.readSnapshot, root, run.runId, "main"),
|
||||
).toBe(false);
|
||||
expect(mocks.getSubagentRunsSnapshotForRead).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps every bound read equivalent to its documented snapshot scope", () => {
|
||||
it("keeps every bound read equivalent to its documented snapshot scope", async () => {
|
||||
const now = Date.now();
|
||||
const root = "agent:main:root";
|
||||
const controller = "agent:main:controller";
|
||||
|
|
@ -362,7 +373,6 @@ describe("subagent registry scoped reads", () => {
|
|||
[...mocks.liveRuns.values()].filter((run) => run.childSessionKey === childSessionKey),
|
||||
);
|
||||
mocks.getSubagentRunsSnapshotForRead.mockReturnValue(snapshot);
|
||||
mocks.getSubagentRunsSnapshotForSessions.mockReturnValue(snapshot);
|
||||
mocks.getSubagentRunsSnapshotForChildSession.mockReturnValue(childSnapshot);
|
||||
mocks.getSubagentRunsSnapshotForController.mockReturnValue(controllerSnapshot);
|
||||
mocks.getSubagentSessionListRunsSnapshotForRead.mockReturnValue(snapshot);
|
||||
|
|
@ -372,10 +382,11 @@ describe("subagent registry scoped reads", () => {
|
|||
freshTerminal,
|
||||
parentRun,
|
||||
]);
|
||||
expect(mod.countActiveDescendantRuns(root)).toBe(2);
|
||||
expect(mod.countActiveDescendantRuns(root, "main")).toBe(1);
|
||||
expect(mod.countPendingDescendantRuns(root)).toBe(4);
|
||||
expect(mod.hasDescendantRunAwaitingSettle(root, pendingRun.runId)).toBe(true);
|
||||
expect(countActiveDescendantRunsFromRuns(snapshot, root)).toBe(2);
|
||||
expect(countActiveDescendantRunsFromRuns(snapshot, root, "main")).toBe(1);
|
||||
mocks.readSnapshot = snapshot;
|
||||
expect(await mod.countPendingDescendantRuns(root, () => {})).toBe(4);
|
||||
expect(hasDescendantRunAwaitingSettleFromRuns(snapshot, root, pendingRun.runId)).toBe(true);
|
||||
expect(mod.getSubagentRunByChildSessionKey(reusedChild)).toBe(oldActive);
|
||||
expect(mod.getLatestSubagentRunByChildSessionKey(reusedChild)).toBe(freshTerminal);
|
||||
expect(mod.buildSubagentSessionListReadIndex(now).getDisplaySubagentRun(reusedChild)).toBe(
|
||||
|
|
|
|||
|
|
@ -4,11 +4,9 @@ import { getSubagentRunsForChildSession, subagentRuns } from "./subagent-registr
|
|||
import {
|
||||
buildLatestSubagentRunReadIndexFromRuns,
|
||||
buildSubagentRunReadIndexFromRuns,
|
||||
countActiveDescendantRunsFromRuns,
|
||||
countPendingDescendantRunsFromRuns,
|
||||
getLatestSubagentRunByChildSessionKeyFromRuns,
|
||||
getSubagentRunByChildSessionKeyFromRuns,
|
||||
hasDescendantRunAwaitingSettleFromRuns,
|
||||
listRunsForControllerFromRuns,
|
||||
listRunsForRequesterFromRuns,
|
||||
resolveRequesterForChildSessionFromRuns,
|
||||
|
|
@ -21,12 +19,13 @@ import {
|
|||
getSubagentSessionListRunsSnapshotForChildSessions,
|
||||
getSubagentRunsSnapshotForChildSession,
|
||||
getSubagentRunsSnapshotForController,
|
||||
getSubagentRunsSnapshotForSessions,
|
||||
getSubagentSessionListRunsSnapshotForRead,
|
||||
getSubagentSessionListRunsSnapshotForSessions,
|
||||
withSubagentRunReadSnapshot,
|
||||
} from "./subagent-registry-state.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
import { isSubagentRunLive } from "./subagent-run-liveness.js";
|
||||
import { collectSubagentSessionReadKeys } from "./subagent-session-read-scope.js";
|
||||
export { isSubagentRunLive, isSubagentRunQueued } from "./subagent-run-liveness.js";
|
||||
|
||||
export type { SubagentRunReadIndex } from "./subagent-registry-queries.js";
|
||||
|
|
@ -88,46 +87,30 @@ export function listSubagentRunsForController(
|
|||
);
|
||||
}
|
||||
|
||||
export function countActiveDescendantRuns(
|
||||
export async function countPendingDescendantRuns(
|
||||
rootSessionKey: string,
|
||||
requesterAgentId?: string,
|
||||
requesterStorePath?: string | null,
|
||||
rootRunIds?: ReadonlySet<string>,
|
||||
): number {
|
||||
return countActiveDescendantRunsFromRuns(
|
||||
getSubagentRunsSnapshotForSessions(subagentRuns, [rootSessionKey]),
|
||||
rootSessionKey,
|
||||
requesterAgentId,
|
||||
requesterStorePath,
|
||||
rootRunIds,
|
||||
);
|
||||
}
|
||||
|
||||
export function countPendingDescendantRuns(rootSessionKey: string): number {
|
||||
return countPendingDescendantRunsFromRuns(
|
||||
getSubagentRunsSnapshotForSessions(subagentRuns, [rootSessionKey]),
|
||||
rootSessionKey,
|
||||
);
|
||||
}
|
||||
|
||||
/** True when any descendant run still awaits terminal settle (suspended delivery counts as settled). */
|
||||
export function hasDescendantRunAwaitingSettle(
|
||||
rootSessionKey: string,
|
||||
excludeRunId?: string,
|
||||
requesterAgentId?: string,
|
||||
requesterStorePath?: string | null,
|
||||
settledBefore?: number,
|
||||
rootRunIds?: ReadonlySet<string>,
|
||||
): boolean {
|
||||
return hasDescendantRunAwaitingSettleFromRuns(
|
||||
getSubagentRunsSnapshotForSessions(subagentRuns, [rootSessionKey]),
|
||||
rootSessionKey,
|
||||
excludeRunId,
|
||||
requesterAgentId,
|
||||
requesterStorePath,
|
||||
settledBefore,
|
||||
rootRunIds,
|
||||
assertCurrent: () => void,
|
||||
): Promise<number> {
|
||||
assertCurrent();
|
||||
const count = await withSubagentRunReadSnapshot(
|
||||
subagentRuns,
|
||||
(snapshot) => {
|
||||
assertCurrent();
|
||||
const sessionKeys = collectSubagentSessionReadKeys([rootSessionKey], snapshot.values());
|
||||
return {
|
||||
runIds: [...snapshot.values()]
|
||||
.filter((entry) => sessionKeys.has(entry.childSessionKey.trim()))
|
||||
.map((entry) => entry.runId),
|
||||
sessionKeys: [],
|
||||
};
|
||||
},
|
||||
(_selection, runs) => {
|
||||
assertCurrent();
|
||||
return countPendingDescendantRunsFromRuns(new Map(runs), rootSessionKey);
|
||||
},
|
||||
);
|
||||
assertCurrent();
|
||||
return count;
|
||||
}
|
||||
|
||||
/** Resolves the requester session and normalized origin for a child subagent session. */
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ it("keeps a known requester wake commit while native staging waits for its ackno
|
|||
...callbacks,
|
||||
}),
|
||||
clearPendingLifecycleError: unexpected,
|
||||
countPendingDescendantRuns: () => 0,
|
||||
countPendingDescendantRuns: async () => 0,
|
||||
getLatestRunForChildSession: () => null,
|
||||
suppressAnnounceForSteerRestart: () => false,
|
||||
shouldEmitEndedHookForRun: () => false,
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ export function registerSubagentResultRefreshCases(params: {
|
|||
getLifecycleHandler: () => (event: AgentEventPayload) => void;
|
||||
mocks: Pick<
|
||||
ReturnType<typeof createSubagentRegistryMockState>,
|
||||
"callGateway" | "captureSubagentCompletionReply" | "persistSubagentRunsToDisk"
|
||||
"callGateway" | "captureSubagentCompletionReply" | "persistSubagentRunsToDiskOrThrow"
|
||||
>;
|
||||
}) {
|
||||
const { getRegistry, getLifecycleHandler, mocks } = params;
|
||||
|
|
@ -62,8 +62,8 @@ export function registerSubagentResultRefreshCases(params: {
|
|||
captureStarted.resolve();
|
||||
return capture.promise;
|
||||
});
|
||||
mocks.persistSubagentRunsToDisk.mockClear();
|
||||
mocks.persistSubagentRunsToDisk.mockImplementationOnce(() => persisted.resolve());
|
||||
mocks.persistSubagentRunsToDiskOrThrow.mockClear();
|
||||
mocks.persistSubagentRunsToDiskOrThrow.mockImplementationOnce(() => persisted.resolve());
|
||||
const lifecycleHandler = getLifecycleHandler();
|
||||
|
||||
const emitEnd = () => {
|
||||
|
|
@ -104,7 +104,7 @@ export function registerSubagentResultRefreshCases(params: {
|
|||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(getActiveGatewayRootWorkCount()).toBe(0);
|
||||
expect(entry?.completion?.resultText).toBe("replacement final reply");
|
||||
expect(mocks.persistSubagentRunsToDisk).toHaveBeenCalledOnce();
|
||||
expect(mocks.persistSubagentRunsToDiskOrThrow).toHaveBeenCalledOnce();
|
||||
} finally {
|
||||
capture.resolve("replacement final reply");
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ import {
|
|||
createSubagentSessionListReadView,
|
||||
getSubagentRunsSnapshotForChildSession,
|
||||
getSubagentRunsSnapshotForRead,
|
||||
getSubagentRunsSnapshotForSessions,
|
||||
getSubagentMaintenanceRunsSnapshotForRead,
|
||||
getSubagentSessionListRunsSnapshotForRead,
|
||||
getSubagentSessionListReadSnapshotIdentity,
|
||||
|
|
@ -802,12 +801,6 @@ it.each(["best effort", "strict refusal", "strict commit", "atomic commit"])(
|
|||
model: refused ? "before" : "after",
|
||||
execution: { status: refused ? "running" : "terminal" },
|
||||
});
|
||||
expect(
|
||||
getSubagentRunsSnapshotForSessions(new Map(), [entry.childSessionKey]).get("one"),
|
||||
).toMatchObject({
|
||||
model: refused ? "before" : "after",
|
||||
execution: { status: refused ? "running" : "terminal" },
|
||||
});
|
||||
const maintenance = getSubagentMaintenanceRunsSnapshotForRead(new Map()).get("one");
|
||||
expect(maintenance?.execution.status).toBe(refused ? "running" : "terminal");
|
||||
expect(maintenance?.cleanupCompletedAt).toBe(refused ? undefined : 2);
|
||||
|
|
@ -843,9 +836,6 @@ it("keeps retired publications with their draining source across source switches
|
|||
persistSubagentRunsToDiskOrThrow(runs("after"), ["one"]);
|
||||
expect(selectedChild().get("one")?.model).toBe("after");
|
||||
expect(getSubagentRunsSnapshotForRead(new Map()).get("one")?.model).toBe("after");
|
||||
expect(
|
||||
getSubagentRunsSnapshotForSessions(new Map(), ["agent:main:subagent:one"]).get("one")?.model,
|
||||
).toBe("after");
|
||||
await withEnvAsync({ OPENCLAW_STATE_DIR: other.stateDir }, async () => {
|
||||
expect(selectedChild().has("one")).toBe(false);
|
||||
expect(getSubagentRunsSnapshotForRead(new Map()).has("one")).toBe(false);
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ import {
|
|||
getSubagentSessionListRunsSnapshotForChildSessions,
|
||||
getSubagentRunsSnapshotForController,
|
||||
getSubagentRunsSnapshotForRead,
|
||||
getSubagentRunsSnapshotForSessions,
|
||||
prepareSubagentRunsSnapshotForRunIds,
|
||||
onSubagentRegistryPersisted,
|
||||
persistSubagentRunsToDisk,
|
||||
|
|
@ -40,13 +39,6 @@ const mocks = vi.hoisted(() => ({
|
|||
loadSubagentMaintenanceRunsFromSqlite: vi.fn<() => Map<string, SubagentRunMaintenanceRecord>>(),
|
||||
readCompactRuns: vi.fn<() => Map<string, SubagentRunReadRecord>>(),
|
||||
nativeCompactRead: vi.fn<() => Map<string, SubagentRunReadRecord>>(),
|
||||
loadSubagentRunsForSessionsFromSqlite: vi.fn<
|
||||
() => {
|
||||
sessionKeys: Set<string>;
|
||||
runs: Map<string, SubagentRunReadRecord>;
|
||||
complete: boolean;
|
||||
}
|
||||
>(),
|
||||
saveSubagentRegistryChangesToSqlite:
|
||||
vi.fn<(runs: Map<string, SubagentRunRecord>, changedRunIds: readonly string[]) => void>(),
|
||||
saveSubagentRegistryToSqlite: vi.fn<(runs: Map<string, SubagentRunRecord>) => void>(),
|
||||
|
|
@ -58,7 +50,6 @@ vi.mock("./subagent-registry.store.sqlite.js", () => ({
|
|||
loadSubagentRegistryFromSqlite: mocks.loadSubagentRegistryFromSqlite,
|
||||
loadSubagentMaintenanceRunsFromSqlite: mocks.loadSubagentMaintenanceRunsFromSqlite,
|
||||
loadSubagentSessionListRunsFromSqlite: mocks.nativeCompactRead,
|
||||
loadSubagentRunsForSessionsFromSqlite: mocks.loadSubagentRunsForSessionsFromSqlite,
|
||||
saveSubagentRegistryChangesToSqlite: mocks.saveSubagentRegistryChangesToSqlite,
|
||||
saveSubagentRegistryToSqlite: mocks.saveSubagentRegistryToSqlite,
|
||||
}));
|
||||
|
|
@ -93,7 +84,6 @@ describe("subagent registry state read cache", () => {
|
|||
mocks.nativeCompactRead.mockReset().mockImplementation(() => {
|
||||
throw new Error("Compact registry reads must use the read worker");
|
||||
});
|
||||
mocks.loadSubagentRunsForSessionsFromSqlite.mockReset();
|
||||
mocks.saveSubagentRegistryChangesToSqlite.mockReset();
|
||||
mocks.saveSubagentRegistryToSqlite.mockReset();
|
||||
vi.spyOn(stateReads, "executeExistingOpenClawStateRead").mockImplementation(
|
||||
|
|
@ -332,10 +322,8 @@ describe("subagent registry state read cache", () => {
|
|||
expect(getSubagentSessionListRunsSnapshotForRead(new Map(), [" "])).toEqual(new Map());
|
||||
});
|
||||
|
||||
it.each([
|
||||
["session-list", getSubagentSessionListRunsSnapshotForSessions],
|
||||
["full", getSubagentRunsSnapshotForSessions],
|
||||
] as const)("keeps %s tree reads on the shared write-through cache", (_kind, read) => {
|
||||
it("keeps session-list tree reads on the shared write-through cache", () => {
|
||||
const read = getSubagentSessionListRunsSnapshotForSessions;
|
||||
const root = "agent:main:tree";
|
||||
const child = { ...createRun("child"), requesterSessionKey: root };
|
||||
const grandchild = { ...createRun("grandchild"), requesterSessionKey: child.childSessionKey };
|
||||
|
|
@ -365,7 +353,6 @@ describe("subagent registry state read cache", () => {
|
|||
childSessionKey: movedOut.childSessionKey,
|
||||
requesterSessionKey: movedOut.requesterSessionKey,
|
||||
});
|
||||
expect(mocks.loadSubagentRunsForSessionsFromSqlite).not.toHaveBeenCalled();
|
||||
|
||||
vi.advanceTimersByTime(60_000);
|
||||
const updated = { ...child, model: "updated-model" };
|
||||
|
|
@ -377,7 +364,6 @@ describe("subagent registry state read cache", () => {
|
|||
persistSubagentRunsToDisk(runs, [renamed.runId]);
|
||||
expect([...read(new Map(), [root]).keys()]).toEqual([child.runId, cycle.runId]);
|
||||
expect(read(new Map(), [root]).get(child.runId)?.childSessionKey).toBe(renamed.childSessionKey);
|
||||
expect(mocks.loadSubagentRunsForSessionsFromSqlite).not.toHaveBeenCalled();
|
||||
expect(mocks.readCompactRuns).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
|
@ -404,14 +390,11 @@ describe("subagent registry state read cache", () => {
|
|||
expect(
|
||||
index.listDescendantRunsForRequester(requesterSessionKey).map((run) => run.runId),
|
||||
).toEqual([newer.runId, descendant.runId]);
|
||||
expect(mocks.loadSubagentRunsForSessionsFromSqlite).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
["session-list", getSubagentSessionListRunsSnapshotForSessions],
|
||||
["full", getSubagentRunsSnapshotForSessions],
|
||||
] as const)("keeps a shared %s requester edge until its last member moves", (_kind, read) => {
|
||||
it("keeps a shared session-list requester edge until its last member moves", () => {
|
||||
const read = getSubagentSessionListRunsSnapshotForSessions;
|
||||
const root = "agent:main:shared-edge";
|
||||
const first = { ...createRun("first"), requesterSessionKey: root };
|
||||
const second = {
|
||||
|
|
@ -450,130 +433,65 @@ describe("subagent registry state read cache", () => {
|
|||
descendant.runId,
|
||||
first.runId,
|
||||
]);
|
||||
expect(mocks.loadSubagentRunsForSessionsFromSqlite).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
"session-list",
|
||||
getSubagentSessionListRunsSnapshotForSessions,
|
||||
getSubagentSessionListRunsSnapshotForRead,
|
||||
mocks.readCompactRuns,
|
||||
],
|
||||
[
|
||||
"full",
|
||||
getSubagentRunsSnapshotForSessions,
|
||||
getSubagentRunsSnapshotForRead,
|
||||
mocks.loadSubagentRegistryFromSqlite,
|
||||
],
|
||||
] as const)(
|
||||
"preserves %s snapshot order through named updates and delete/reinsert",
|
||||
async (kind, read, readAll, load) => {
|
||||
const root = "agent:main:first-parent";
|
||||
const other = "agent:main:second-parent";
|
||||
const first = { ...createRun("first"), requesterSessionKey: root };
|
||||
const middle = { ...createRun("middle"), requesterSessionKey: other };
|
||||
const last = { ...createRun("last"), requesterSessionKey: root };
|
||||
load.mockReturnValueOnce(
|
||||
new Map([
|
||||
["cached-alias-for-first", first],
|
||||
[middle.runId, middle],
|
||||
[last.runId, last],
|
||||
]),
|
||||
);
|
||||
if (kind === "session-list") {
|
||||
await prepareSubagentSessionListReadCache();
|
||||
}
|
||||
expect([...readAll(new Map()).keys()]).toEqual([first.runId, middle.runId, last.runId]);
|
||||
const keys = [other, root];
|
||||
expect([...read(new Map(), keys).keys()]).toEqual([first.runId, middle.runId, last.runId]);
|
||||
it("preserves session-list snapshot order through named updates and delete/reinsert", async () => {
|
||||
const read = getSubagentSessionListRunsSnapshotForSessions;
|
||||
const readAll = getSubagentSessionListRunsSnapshotForRead;
|
||||
const load = mocks.readCompactRuns;
|
||||
const root = "agent:main:first-parent";
|
||||
const other = "agent:main:second-parent";
|
||||
const first = { ...createRun("first"), requesterSessionKey: root };
|
||||
const middle = { ...createRun("middle"), requesterSessionKey: other };
|
||||
const last = { ...createRun("last"), requesterSessionKey: root };
|
||||
load.mockReturnValueOnce(
|
||||
new Map([
|
||||
["cached-alias-for-first", first],
|
||||
[middle.runId, middle],
|
||||
[last.runId, last],
|
||||
]),
|
||||
);
|
||||
await prepareSubagentSessionListReadCache();
|
||||
expect([...readAll(new Map()).keys()]).toEqual([first.runId, middle.runId, last.runId]);
|
||||
const keys = [other, root];
|
||||
expect([...read(new Map(), keys).keys()]).toEqual([first.runId, middle.runId, last.runId]);
|
||||
|
||||
const changed = { ...middle, model: "updated-model" };
|
||||
const writes = new Map([[changed.runId, changed]]);
|
||||
persistSubagentRunsToDisk(writes, [changed.runId]);
|
||||
const updated = read(new Map(), keys);
|
||||
expect([...updated.keys()]).toEqual([first.runId, middle.runId, last.runId]);
|
||||
expect(updated.get(middle.runId)?.model).toBe("updated-model");
|
||||
const changed = { ...middle, model: "updated-model" };
|
||||
const writes = new Map([[changed.runId, changed]]);
|
||||
persistSubagentRunsToDisk(writes, [changed.runId]);
|
||||
const updated = read(new Map(), keys);
|
||||
expect([...updated.keys()]).toEqual([first.runId, middle.runId, last.runId]);
|
||||
expect(updated.get(middle.runId)?.model).toBe("updated-model");
|
||||
|
||||
writes.delete(middle.runId);
|
||||
persistSubagentRunsToDisk(writes, [middle.runId]);
|
||||
expect([...read(new Map(), keys).keys()]).toEqual([first.runId, last.runId]);
|
||||
writes.set(middle.runId, changed);
|
||||
persistSubagentRunsToDisk(writes, [middle.runId]);
|
||||
expect([...read(new Map(), keys).keys()]).toEqual([first.runId, last.runId, middle.runId]);
|
||||
expect(load).toHaveBeenCalledOnce();
|
||||
expect(mocks.loadSubagentRunsForSessionsFromSqlite).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it("reuses a complete tree snapshot without making a partial tree globally authoritative", () => {
|
||||
const root = "agent:main:tree";
|
||||
const child = { ...createRun("child"), requesterSessionKey: root };
|
||||
const unrelated = createRun("unrelated");
|
||||
mocks.loadSubagentRunsForSessionsFromSqlite.mockReturnValue({
|
||||
sessionKeys: new Set([root, child.childSessionKey]),
|
||||
runs: new Map([[child.runId, child]]),
|
||||
complete: false,
|
||||
});
|
||||
expect(getSubagentRunsSnapshotForSessions(new Map(), [root]).size).toBe(1);
|
||||
mocks.loadSubagentRegistryFromSqlite.mockReturnValue(new Map([[unrelated.runId, unrelated]]));
|
||||
expect(getSubagentRunsSnapshotForRead(new Map()).has(unrelated.runId)).toBe(true);
|
||||
|
||||
clearSubagentRunsReadCacheForTest();
|
||||
mocks.loadSubagentRunsForSessionsFromSqlite.mockReturnValue({
|
||||
sessionKeys: new Set([root, child.childSessionKey]),
|
||||
runs: new Map([[child.runId, child]]),
|
||||
complete: true,
|
||||
});
|
||||
expect(getSubagentRunsSnapshotForSessions(new Map(), [root]).size).toBe(1);
|
||||
expect(getSubagentRunsSnapshotForRead(new Map()).has(child.runId)).toBe(true);
|
||||
expect(getSubagentRunsSnapshotForRead(new Map()).has(unrelated.runId)).toBe(false);
|
||||
expect(mocks.loadSubagentRegistryFromSqlite).toHaveBeenCalledOnce();
|
||||
writes.delete(middle.runId);
|
||||
persistSubagentRunsToDisk(writes, [middle.runId]);
|
||||
expect([...read(new Map(), keys).keys()]).toEqual([first.runId, last.runId]);
|
||||
writes.set(middle.runId, changed);
|
||||
persistSubagentRunsToDisk(writes, [middle.runId]);
|
||||
expect([...read(new Map(), keys).keys()]).toEqual([first.runId, last.runId, middle.runId]);
|
||||
expect(load).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["full", getSubagentRunsSnapshotForSessions, getSubagentRunsSnapshotForRead],
|
||||
[
|
||||
"session-list",
|
||||
getSubagentSessionListRunsSnapshotForSessions,
|
||||
getSubagentSessionListRunsSnapshotForRead,
|
||||
],
|
||||
] as const)(
|
||||
"keeps cold failed deletions when complete %s facts become ready",
|
||||
async (kind, readTree, readAll) => {
|
||||
const removed = createRun("removed");
|
||||
const retained = createRun("retained");
|
||||
const nested = { ...createRun("nested"), requesterSessionKey: removed.childSessionKey };
|
||||
const stored = new Map([removed, retained, nested].map((run) => [run.runId, run]));
|
||||
mocks.readCompactRuns.mockReturnValue(stored);
|
||||
mocks.loadSubagentRunsForSessionsFromSqlite.mockReturnValue({
|
||||
sessionKeys: new Set([
|
||||
removed.requesterSessionKey,
|
||||
removed.childSessionKey,
|
||||
retained.childSessionKey,
|
||||
nested.childSessionKey,
|
||||
]),
|
||||
runs: stored,
|
||||
complete: true,
|
||||
});
|
||||
mocks.saveSubagentRegistryChangesToSqlite.mockImplementationOnce(() => {
|
||||
throw new Error("disk unavailable");
|
||||
});
|
||||
persistSubagentRunsToDisk(new Map(), [removed.runId]);
|
||||
if (kind === "session-list") {
|
||||
await prepareSubagentSessionListReadCache();
|
||||
}
|
||||
it("keeps cold failed deletions when complete session-list facts become ready", async () => {
|
||||
const readTree = getSubagentSessionListRunsSnapshotForSessions;
|
||||
const readAll = getSubagentSessionListRunsSnapshotForRead;
|
||||
const removed = createRun("removed");
|
||||
const retained = createRun("retained");
|
||||
const nested = { ...createRun("nested"), requesterSessionKey: removed.childSessionKey };
|
||||
const stored = new Map([removed, retained, nested].map((run) => [run.runId, run]));
|
||||
mocks.readCompactRuns.mockReturnValue(stored);
|
||||
mocks.saveSubagentRegistryChangesToSqlite.mockImplementationOnce(() => {
|
||||
throw new Error("disk unavailable");
|
||||
});
|
||||
persistSubagentRunsToDisk(new Map(), [removed.runId]);
|
||||
await prepareSubagentSessionListReadCache();
|
||||
|
||||
expect([...readTree(new Map(), [removed.requesterSessionKey]).keys()]).toEqual([
|
||||
retained.runId,
|
||||
]);
|
||||
expect([...readAll(new Map()).keys()]).toEqual([retained.runId, nested.runId]);
|
||||
expect(mocks.loadSubagentRunsForSessionsFromSqlite).toHaveBeenCalledTimes(
|
||||
kind === "full" ? 1 : 0,
|
||||
);
|
||||
expect(mocks.readCompactRuns).toHaveBeenCalledTimes(kind === "session-list" ? 1 : 0);
|
||||
},
|
||||
);
|
||||
expect([...readTree(new Map(), [removed.requesterSessionKey]).keys()]).toEqual([
|
||||
retained.runId,
|
||||
]);
|
||||
expect([...readAll(new Map()).keys()]).toEqual([retained.runId, nested.runId]);
|
||||
expect(mocks.readCompactRuns).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("preserves unrelated projected rows across incremental writes", async () => {
|
||||
const retained = createRun("retained");
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ import {
|
|||
} from "./subagent-registry-persistence.js";
|
||||
import { publishSubagentRunChanges } from "./subagent-registry-publication.js";
|
||||
import {
|
||||
applySubagentRunChanges,
|
||||
assertSubagentReadContext,
|
||||
captureSubagentFactsAdmission,
|
||||
consumeFreshSubagentRuns,
|
||||
|
|
@ -34,7 +33,6 @@ import {
|
|||
prepareSubagentRunsCache,
|
||||
readCompactSubagentRuns,
|
||||
rememberSubagentRunsSnapshot,
|
||||
retainUnpublishedSubagentChanges,
|
||||
shouldReadPersistedSubagentRuns,
|
||||
SubagentSessionListUnavailableError,
|
||||
type SubagentRunsCache,
|
||||
|
|
@ -54,7 +52,6 @@ import {
|
|||
loadSubagentRunsForControllerFromSqlite,
|
||||
loadSubagentRegistryFromSqlite,
|
||||
loadSubagentMaintenanceRunsFromSqlite,
|
||||
loadSubagentRunsForSessionsFromSqlite,
|
||||
saveSubagentRegistryChangesToSqlite,
|
||||
saveSubagentRegistryToSqlite,
|
||||
} from "./subagent-registry.store.sqlite.js";
|
||||
|
|
@ -571,7 +568,6 @@ function getSubagentSessionTreeSnapshot<T extends SubagentRunReadRecord>(
|
|||
inMemoryRuns: Map<string, SubagentRunRecord>,
|
||||
sessionKeys: readonly string[],
|
||||
cache: SubagentRunsCache<T>,
|
||||
load?: () => { sessionKeys: Set<string>; runs: Map<string, T>; complete: boolean },
|
||||
): Map<string, T> {
|
||||
if (!sessionKeys.some((key) => key.trim())) {
|
||||
return new Map();
|
||||
|
|
@ -579,7 +575,7 @@ function getSubagentSessionTreeSnapshot<T extends SubagentRunReadRecord>(
|
|||
const cached = shouldReadPersistedSubagentRuns() ? getPersistedSubagentRunsSnapshot(cache) : null;
|
||||
const lookup = cached ? getSessionListLookup(cache, cached) : undefined;
|
||||
const indexed = lookup?.selectSessions(sessionKeys, inMemoryRuns.values());
|
||||
let selected =
|
||||
const selected =
|
||||
indexed?.sessionKeys ??
|
||||
collectSubagentSessionReadKeys(sessionKeys, cached?.values() ?? [], inMemoryRuns.values());
|
||||
return getSubagentRunsSnapshot(inMemoryRuns, cache, {
|
||||
|
|
@ -591,28 +587,7 @@ function getSubagentSessionTreeSnapshot<T extends SubagentRunReadRecord>(
|
|||
if (cached) {
|
||||
return indexed ? indexedSnapshotRows(cached, indexed.cacheKeys) : cached.values();
|
||||
}
|
||||
if (!load) {
|
||||
throw new Error("Subagent session-list facts must be prepared before synchronous reads");
|
||||
}
|
||||
const snapshot = load();
|
||||
// A tree covering every physical row may populate the existing full cache.
|
||||
if (snapshot.complete) {
|
||||
applySubagentRunChanges(snapshot.runs, cache.state.changes);
|
||||
snapshot.sessionKeys = collectSubagentSessionReadKeys(
|
||||
sessionKeys,
|
||||
snapshot.runs.values(),
|
||||
inMemoryRuns.values(),
|
||||
);
|
||||
const admission = cache.captureAdmission?.();
|
||||
cache.state = {
|
||||
snapshot: snapshot.runs,
|
||||
changes: retainUnpublishedSubagentChanges(cache.state.changes),
|
||||
admission,
|
||||
sourceIdentity: admission?.identity.key,
|
||||
};
|
||||
}
|
||||
selected = snapshot.sessionKeys;
|
||||
return snapshot.runs.values();
|
||||
throw new Error("Subagent session-list facts must be prepared before synchronous reads");
|
||||
},
|
||||
matches: (entry) => selected.has(entry.childSessionKey.trim()),
|
||||
});
|
||||
|
|
@ -630,19 +605,6 @@ export function getSubagentSessionListRunsSnapshotForSessions(
|
|||
);
|
||||
}
|
||||
|
||||
/** Settlement reads retain the canonical codec and raw local reservation ownership. */
|
||||
export function getSubagentRunsSnapshotForSessions(
|
||||
inMemoryRuns: Map<string, SubagentRunRecord>,
|
||||
sessionKeys: readonly string[],
|
||||
): Map<string, SubagentRunRecord> {
|
||||
return getSubagentSessionTreeSnapshot(
|
||||
inMemoryRuns,
|
||||
sessionKeys,
|
||||
persistedSubagentRunsReadCache,
|
||||
() => loadSubagentRunsForSessionsFromSqlite(sessionKeys, inMemoryRuns.values()),
|
||||
);
|
||||
}
|
||||
|
||||
export function getSubagentRunsSnapshotForController(
|
||||
inMemoryRuns: Map<string, SubagentRunRecord>,
|
||||
controllerSessionKey: string,
|
||||
|
|
|
|||
|
|
@ -1,11 +1,15 @@
|
|||
import { captureOpenClawStateWorkerContext } from "../../../state/openclaw-state-worker-context.js";
|
||||
import { isDeliverySuspended } from "./subagent-delivery-state.js";
|
||||
import {
|
||||
SUBAGENT_ENDED_REASON_COMPLETE,
|
||||
type SubagentLifecycleEndedReason,
|
||||
} from "./subagent-lifecycle-events.js";
|
||||
import { shouldSuppressSubagentRecoverySessionEffects } from "./subagent-recovery-state.js";
|
||||
import { safeRemoveAttachmentsDir } from "./subagent-registry-helpers.js";
|
||||
import type { SubagentLifecycleController } from "./subagent-registry-lifecycle.js";
|
||||
import type {
|
||||
SubagentLifecycleController,
|
||||
SubagentLifecycleOptions,
|
||||
} from "./subagent-registry-lifecycle.js";
|
||||
import { assertSubagentRegistryWriteSourceCurrent } from "./subagent-registry-persistence.js";
|
||||
import type { SubagentRunRecord } from "./subagent-registry.types.js";
|
||||
|
||||
const SUBAGENT_SUSPENDED_DELIVERY_RETENTION_MS = 7 * 24 * 60 * 60_000;
|
||||
|
|
@ -52,48 +56,45 @@ export async function discardSuspendedPendingFinalDelivery(params: {
|
|||
clearPendingLifecycleError: (runId: string) => void;
|
||||
clearPendingLifecycleTimeout: (runId: string) => void;
|
||||
discardTerminalDelivery: typeof SubagentLifecycleController.discardTerminalDelivery;
|
||||
completeCleanupBookkeeping: (params: {
|
||||
runId: string;
|
||||
entry: SubagentRunRecord;
|
||||
cleanup: "delete" | "keep";
|
||||
completedAt: number;
|
||||
skipRequesterSettleWake: true;
|
||||
}) => void;
|
||||
completeCleanupBookkeeping: SubagentLifecycleController["completeCleanupBookkeeping"];
|
||||
isCurrent: () => boolean;
|
||||
sessionEffectsHostCurrent: SubagentLifecycleController["sessionEffectsHostCurrent"];
|
||||
shouldSuppressSessionEffects: SubagentLifecycleController["shouldSuppressSessionEffects"];
|
||||
shouldEmitEndedHookForRun: (params: {
|
||||
entry: SubagentRunRecord;
|
||||
reason: SubagentLifecycleEndedReason;
|
||||
}) => boolean;
|
||||
emitSubagentEndedHookForRun: (params: {
|
||||
entry: SubagentRunRecord;
|
||||
reason: SubagentLifecycleEndedReason;
|
||||
sendFarewell: true;
|
||||
}) => Promise<void>;
|
||||
emitSubagentEndedHookForRun: SubagentLifecycleOptions["emitSubagentEndedHookForRun"];
|
||||
warn: (message: string, meta?: Record<string, unknown>) => void;
|
||||
}): Promise<void> {
|
||||
const { runId, entry, now, reason, resumedRuns } = params;
|
||||
const snapshot = structuredClone(entry);
|
||||
const wasResumed = resumedRuns.has(runId);
|
||||
params.discardTerminalDelivery(entry, now, reason);
|
||||
const suppressSessionEffects = shouldSuppressSubagentRecoverySessionEffects(entry);
|
||||
const stateContext = captureOpenClawStateWorkerContext();
|
||||
const generation = entry.generation;
|
||||
const isCurrent = () => {
|
||||
assertSubagentRegistryWriteSourceCurrent(stateContext);
|
||||
return entry.generation === generation && params.isCurrent();
|
||||
};
|
||||
const assertCurrent = () => {
|
||||
if (!isCurrent()) {
|
||||
throw new Error("Subagent suspended delivery cleanup owner changed.");
|
||||
}
|
||||
};
|
||||
assertCurrent();
|
||||
const isHookCurrent = () => isCurrent() && params.sessionEffectsHostCurrent(entry);
|
||||
const prepareHookCurrent = async () =>
|
||||
isHookCurrent() && !(await params.shouldSuppressSessionEffects(entry)) && isHookCurrent();
|
||||
const completionReason = entry.endedReason ?? SUBAGENT_ENDED_REASON_COMPLETE;
|
||||
try {
|
||||
params.completeCleanupBookkeeping({
|
||||
runId,
|
||||
entry,
|
||||
cleanup: entry.cleanup,
|
||||
completedAt: now,
|
||||
skipRequesterSettleWake: true,
|
||||
});
|
||||
} catch (error) {
|
||||
for (const key of Object.keys(entry)) {
|
||||
Reflect.deleteProperty(entry, key);
|
||||
}
|
||||
Object.assign(entry, snapshot);
|
||||
if (wasResumed) {
|
||||
resumedRuns.add(runId);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
await params.completeCleanupBookkeeping({
|
||||
runId,
|
||||
entry,
|
||||
cleanup: entry.cleanup,
|
||||
completedAt: now,
|
||||
skipRequesterSettleWake: true,
|
||||
stateContext,
|
||||
isCurrent,
|
||||
discardDelivery: () => params.discardTerminalDelivery(entry, now, reason),
|
||||
});
|
||||
assertCurrent();
|
||||
resumedRuns.delete(runId);
|
||||
params.clearPendingLifecycleError(runId);
|
||||
params.clearPendingLifecycleTimeout(runId);
|
||||
|
|
@ -104,10 +105,11 @@ export async function discardSuspendedPendingFinalDelivery(params: {
|
|||
requesterSessionKey: entry.requesterSessionKey,
|
||||
});
|
||||
if (entry.cleanup === "delete" || !entry.retainAttachmentsOnKeep) {
|
||||
await safeRemoveAttachmentsDir(entry);
|
||||
await safeRemoveAttachmentsDir(entry, isCurrent);
|
||||
}
|
||||
assertCurrent();
|
||||
if (
|
||||
!suppressSessionEffects &&
|
||||
(await prepareHookCurrent()) &&
|
||||
entry.expectsCompletionMessage === true &&
|
||||
params.shouldEmitEndedHookForRun({ entry, reason: completionReason })
|
||||
) {
|
||||
|
|
@ -115,6 +117,8 @@ export async function discardSuspendedPendingFinalDelivery(params: {
|
|||
entry,
|
||||
reason: completionReason,
|
||||
sendFarewell: true,
|
||||
isCurrent: isHookCurrent,
|
||||
prepareCurrent: prepareHookCurrent,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -47,7 +47,14 @@ export function createSubagentSweeperHarness(
|
|||
}) => 0,
|
||||
);
|
||||
const completeSubagentRunWithRecovery = vi.fn();
|
||||
const completeCleanupBookkeeping = vi.fn();
|
||||
const completeCleanupBookkeeping = vi.fn<
|
||||
Parameters<typeof createSubagentRegistrySweeper>[0]["completeCleanupBookkeeping"]
|
||||
>(async (params) => {
|
||||
if (params.isCurrent && !params.isCurrent()) {
|
||||
return;
|
||||
}
|
||||
params.discardDelivery?.();
|
||||
});
|
||||
const discardTerminalDelivery =
|
||||
vi.fn<Parameters<typeof createSubagentRegistrySweeper>[0]["discardTerminalDelivery"]>();
|
||||
const emitSubagentEndedHookForRun = vi.fn();
|
||||
|
|
@ -68,6 +75,10 @@ export function createSubagentSweeperHarness(
|
|||
resumeRequesterSettleWake,
|
||||
startSubagentAnnounceCleanupFlow: vi.fn(() => true),
|
||||
completeCleanupBookkeeping,
|
||||
isEndedHookOwnerCurrent: (runId, selected) => runs.get(runId) === selected || !runs.has(runId),
|
||||
sessionEffectsHostCurrent: (selected) => selected.execution.suppressSessionEffects !== true,
|
||||
shouldSuppressSessionEffects: async (selected) =>
|
||||
selected.execution.suppressSessionEffects === true,
|
||||
discardTerminalDelivery,
|
||||
shouldEmitEndedHookForRun: vi.fn(() => false),
|
||||
emitSubagentEndedHookForRun,
|
||||
|
|
|
|||
|
|
@ -68,6 +68,9 @@ export function createSubagentRegistrySweeper(params: {
|
|||
resumeRequesterSettleWake: SubagentLifecycleController["resumeRequesterSettleWake"];
|
||||
startSubagentAnnounceCleanupFlow: SubagentLifecycleController["startSubagentAnnounceCleanupFlow"];
|
||||
completeCleanupBookkeeping: SubagentLifecycleController["completeCleanupBookkeeping"];
|
||||
isEndedHookOwnerCurrent: SubagentLifecycleController["isEndedHookOwnerCurrent"];
|
||||
sessionEffectsHostCurrent: SubagentLifecycleController["sessionEffectsHostCurrent"];
|
||||
shouldSuppressSessionEffects: SubagentLifecycleController["shouldSuppressSessionEffects"];
|
||||
discardTerminalDelivery: typeof SubagentLifecycleController.discardTerminalDelivery;
|
||||
shouldEmitEndedHookForRun: SubagentLifecycleOptions["shouldEmitEndedHookForRun"];
|
||||
emitSubagentEndedHookForRun: SubagentLifecycleOptions["emitSubagentEndedHookForRun"];
|
||||
|
|
@ -338,11 +341,13 @@ export function createSubagentRegistrySweeper(params: {
|
|||
clearPendingLifecycleTimeout: params.clearPendingLifecycleTimeout,
|
||||
discardTerminalDelivery: params.discardTerminalDelivery,
|
||||
completeCleanupBookkeeping: params.completeCleanupBookkeeping,
|
||||
isCurrent: () => params.isEndedHookOwnerCurrent(runId, entry),
|
||||
sessionEffectsHostCurrent: params.sessionEffectsHostCurrent,
|
||||
shouldSuppressSessionEffects: params.shouldSuppressSessionEffects,
|
||||
shouldEmitEndedHookForRun: params.shouldEmitEndedHookForRun,
|
||||
emitSubagentEndedHookForRun: params.emitSubagentEndedHookForRun,
|
||||
warn: params.warn,
|
||||
});
|
||||
mutatedRunIds.add(runId);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,91 @@
|
|||
import { expect, it } from "vitest";
|
||||
import { createDeferred } from "../../../../test/helpers/promise.js";
|
||||
import {
|
||||
expectRecordFields,
|
||||
mockCallArg as getMockCallArg,
|
||||
mockGatewayMethods,
|
||||
type SubagentRegistryHarness,
|
||||
} from "../../subagent-test-fixtures.test-helpers.js";
|
||||
import { observeRootWork } from "./subagent-registry.browser-cleanup.test-support.js";
|
||||
import type { createSubagentRegistryMockState } from "./subagent-registry.mock-state.test-support.js";
|
||||
|
||||
export function registerYieldedParentCleanupCase({
|
||||
getRegistry,
|
||||
mocks,
|
||||
}: {
|
||||
getRegistry: () => SubagentRegistryHarness;
|
||||
mocks: Pick<
|
||||
ReturnType<typeof createSubagentRegistryMockState>,
|
||||
| "entries"
|
||||
| "callGateway"
|
||||
| "persistSubagentRunsToDiskOrThrow"
|
||||
| "runSubagentAnnounceFlow"
|
||||
| "runSubagentEnded"
|
||||
>;
|
||||
}) {
|
||||
it("keeps a paused parent out of ordinary terminal cleanup when descendants settle", async () => {
|
||||
const mod = getRegistry();
|
||||
mocks.entries = {
|
||||
"agent:main:subagent:parent": {
|
||||
sessionId: "sess-parent",
|
||||
updatedAt: 1,
|
||||
},
|
||||
"agent:main:subagent:child": {
|
||||
sessionId: "sess-child",
|
||||
updatedAt: 1,
|
||||
},
|
||||
};
|
||||
mockGatewayMethods(mocks.callGateway, {
|
||||
"agent.wait": { status: "ok", startedAt: Date.now() - 1, endedAt: Date.now() },
|
||||
});
|
||||
|
||||
mod.addSubagentRunForTests({
|
||||
runId: "run-yielded-parent",
|
||||
childSessionKey: "agent:main:subagent:parent",
|
||||
task: "yielded parent waiting on descendants",
|
||||
createdAt: Date.parse("2026-06-26T02:17:00Z"),
|
||||
startedAt: Date.parse("2026-06-26T02:18:00Z"),
|
||||
endedAt: Date.parse("2026-06-26T02:19:00Z"),
|
||||
pauseReason: "sessions_yield",
|
||||
wakeOnDescendantSettle: true,
|
||||
cleanupHandled: false,
|
||||
cleanupCompletedAt: undefined,
|
||||
});
|
||||
|
||||
const parent = mod.getSubagentRunByRunId("run-yielded-parent");
|
||||
const terminalCommitted = createDeferred();
|
||||
const persist = mocks.persistSubagentRunsToDiskOrThrow.getMockImplementation();
|
||||
mocks.persistSubagentRunsToDiskOrThrow.mockImplementation((...args) => {
|
||||
const result = persist?.(...args);
|
||||
if (args[0].get("run-yielded-child-finished")?.execution.status === "terminal") {
|
||||
terminalCommitted.resolve();
|
||||
}
|
||||
return result;
|
||||
});
|
||||
const join = observeRootWork();
|
||||
try {
|
||||
await mod.registerSubagentRun({
|
||||
runId: "run-yielded-child-finished",
|
||||
requesterSessionKey: "agent:main:subagent:parent",
|
||||
requesterDisplayKey: "parent",
|
||||
task: "descendant settles after yield",
|
||||
});
|
||||
await terminalCommitted.promise;
|
||||
} finally {
|
||||
await join();
|
||||
}
|
||||
expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledOnce();
|
||||
expectRecordFields(
|
||||
getMockCallArg(mocks.runSubagentAnnounceFlow, 0, 0, "child finished announce"),
|
||||
{ childRunId: "run-yielded-child-finished" },
|
||||
"child finished announce params",
|
||||
);
|
||||
expect(mod.getSubagentRunByRunId("run-yielded-parent")).toBe(parent);
|
||||
expect(parent).toMatchObject({ pauseReason: "sessions_yield", cleanupHandled: false });
|
||||
expect(parent?.cleanupCompletedAt).toBeUndefined();
|
||||
expect(mocks.runSubagentEnded).not.toHaveBeenCalledWith(
|
||||
expect.objectContaining({ runId: "run-yielded-parent" }),
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
|
@ -4,7 +4,10 @@
|
|||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import "./subagent-registry.mocks.shared.js";
|
||||
import { countPendingDescendantRuns, countActiveDescendantRuns } from "./subagent-registry-read.js";
|
||||
import {
|
||||
countPendingDescendantRuns,
|
||||
buildSubagentSessionListReadIndex,
|
||||
} from "./subagent-registry-read.js";
|
||||
|
||||
vi.mock("../../../config/config.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("../../../config/config.js")>(
|
||||
|
|
@ -140,7 +143,7 @@ describe("subagent registry nested agent tracking", () => {
|
|||
expect(countActiveRunsForSession("agent:main:subagent:orch1")).toBe(2);
|
||||
});
|
||||
|
||||
it("countActiveDescendantRuns traverses through ended parents", () => {
|
||||
it("counts active descendants through ended parents", () => {
|
||||
const { addSubagentRunForTests } = subagentRegistry;
|
||||
|
||||
addSubagentRunForTests({
|
||||
|
|
@ -167,11 +170,17 @@ describe("subagent registry nested agent tracking", () => {
|
|||
cleanupHandled: false,
|
||||
});
|
||||
|
||||
expect(countActiveDescendantRuns("agent:main:main")).toBe(1);
|
||||
expect(countActiveDescendantRuns("agent:main:subagent:orch-ended")).toBe(1);
|
||||
expect(buildSubagentSessionListReadIndex().countActiveDescendantRuns("agent:main:main")).toBe(
|
||||
1,
|
||||
);
|
||||
expect(
|
||||
buildSubagentSessionListReadIndex().countActiveDescendantRuns(
|
||||
"agent:main:subagent:orch-ended",
|
||||
),
|
||||
).toBe(1);
|
||||
});
|
||||
|
||||
it("countPendingDescendantRuns includes ended descendants until cleanup completes", () => {
|
||||
it("countPendingDescendantRuns includes ended descendants until cleanup completes", async () => {
|
||||
const { addSubagentRunForTests } = subagentRegistry;
|
||||
|
||||
addSubagentRunForTests({
|
||||
|
|
@ -201,8 +210,8 @@ describe("subagent registry nested agent tracking", () => {
|
|||
cleanupCompletedAt: undefined,
|
||||
});
|
||||
|
||||
expect(countPendingDescendantRuns("agent:main:main")).toBe(2);
|
||||
expect(countPendingDescendantRuns("agent:main:subagent:orch-pending")).toBe(1);
|
||||
expect(await countPendingDescendantRuns("agent:main:main", () => {})).toBe(2);
|
||||
expect(await countPendingDescendantRuns("agent:main:subagent:orch-pending", () => {})).toBe(1);
|
||||
|
||||
addSubagentRunForTests({
|
||||
runId: "run-leaf-completed",
|
||||
|
|
@ -217,10 +226,10 @@ describe("subagent registry nested agent tracking", () => {
|
|||
cleanupHandled: true,
|
||||
cleanupCompletedAt: 3,
|
||||
});
|
||||
expect(countPendingDescendantRuns("agent:main:subagent:orch-pending")).toBe(1);
|
||||
expect(await countPendingDescendantRuns("agent:main:subagent:orch-pending", () => {})).toBe(1);
|
||||
});
|
||||
|
||||
it("keeps parent pending for parallel children until both descendants complete cleanup", () => {
|
||||
it("keeps parent pending for parallel children until both descendants complete cleanup", async () => {
|
||||
const { addSubagentRunForTests } = subagentRegistry;
|
||||
const parentSessionKey = "agent:main:subagent:orch-parallel";
|
||||
|
||||
|
|
@ -263,7 +272,7 @@ describe("subagent registry nested agent tracking", () => {
|
|||
cleanupCompletedAt: undefined,
|
||||
});
|
||||
|
||||
expect(countPendingDescendantRuns(parentSessionKey)).toBe(2);
|
||||
expect(await countPendingDescendantRuns(parentSessionKey, () => {})).toBe(2);
|
||||
|
||||
addSubagentRunForTests({
|
||||
runId: "run-leaf-a",
|
||||
|
|
@ -278,7 +287,7 @@ describe("subagent registry nested agent tracking", () => {
|
|||
cleanupHandled: true,
|
||||
cleanupCompletedAt: 3,
|
||||
});
|
||||
expect(countPendingDescendantRuns(parentSessionKey)).toBe(1);
|
||||
expect(await countPendingDescendantRuns(parentSessionKey, () => {})).toBe(1);
|
||||
|
||||
addSubagentRunForTests({
|
||||
runId: "run-leaf-b",
|
||||
|
|
@ -293,6 +302,6 @@ describe("subagent registry nested agent tracking", () => {
|
|||
cleanupHandled: true,
|
||||
cleanupCompletedAt: 5,
|
||||
});
|
||||
expect(countPendingDescendantRuns(parentSessionKey)).toBe(0);
|
||||
expect(await countPendingDescendantRuns(parentSessionKey, () => {})).toBe(0);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ import { expect, it, vi } from "vitest";
|
|||
import { createSubagentRunParams } from "../../subagent-test-fixtures.test-helpers.js";
|
||||
import { createSessionsYieldTool } from "../../tools/sessions-yield-tool.js";
|
||||
import { maybeWakeRequesterAfterAllChildrenSettled } from "../announce/subagent-announce.requester-settle-wake.js";
|
||||
import { subagentRuns } from "./subagent-registry-memory.js";
|
||||
import { countActiveDescendantRunsFromRuns } from "./subagent-registry-queries.js";
|
||||
import type { GatewayRequest } from "./subagent-registry.lifecycle-fixture.test-support.js";
|
||||
import * as registry from "./subagent-registry.test-helpers.js";
|
||||
|
||||
|
|
@ -199,7 +201,7 @@ export function registerRequesterWakeSettlementBoundaryTests({
|
|||
},
|
||||
]);
|
||||
expect(batch.requesterSettleWake).toBeUndefined();
|
||||
expect(registry.countActiveDescendantRuns(requesterSessionKey)).toBe(1);
|
||||
expect(registry.countActiveDescendantRuns(requesterSessionKey, "main")).toBe(0);
|
||||
expect(countActiveDescendantRunsFromRuns(subagentRuns, requesterSessionKey)).toBe(1);
|
||||
expect(countActiveDescendantRunsFromRuns(subagentRuns, requesterSessionKey, "main")).toBe(0);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ import { announceTesting as subagentAnnounceTesting } from "../announce/subagent
|
|||
import { maybeWakeRequesterAfterAllChildrenSettled } from "../announce/subagent-announce.requester-settle-wake.js";
|
||||
import * as completionStore from "../completion/subagent-completion-admission.store.js";
|
||||
import { registerRequesterFinalAttachment } from "../requester-final-attachment.js";
|
||||
import { subagentRuns } from "./subagent-registry-memory.js";
|
||||
import { countActiveDescendantRunsFromRuns } from "./subagent-registry-queries.js";
|
||||
import { observeRootWork } from "./subagent-registry.browser-cleanup.test-support.js";
|
||||
import type {
|
||||
GatewayRequest,
|
||||
|
|
@ -815,9 +817,9 @@ describe("requester settle wake product flow", () => {
|
|||
expect(visibleFinals).toBe(0);
|
||||
expect(append).not.toHaveBeenCalled();
|
||||
if (acceptNextChild) {
|
||||
expect(registry.countActiveDescendantRuns(MAIN_REQUESTER_SESSION_KEY, "main")).toBe(
|
||||
1,
|
||||
);
|
||||
expect(
|
||||
countActiveDescendantRunsFromRuns(subagentRuns, MAIN_REQUESTER_SESSION_KEY, "main"),
|
||||
).toBe(1);
|
||||
expect(registry.getSubagentRunByRunId(beta.runId)).toMatchObject({
|
||||
requesterTurnRunId: undefined,
|
||||
requesterSettleWake: {
|
||||
|
|
@ -868,7 +870,9 @@ describe("requester settle wake product flow", () => {
|
|||
]);
|
||||
expect(visibleFinals).toBe(1);
|
||||
expect(sendMessageMock).not.toHaveBeenCalled();
|
||||
expect(registry.countActiveDescendantRuns(MAIN_REQUESTER_SESSION_KEY, "main")).toBe(0);
|
||||
expect(
|
||||
countActiveDescendantRunsFromRuns(subagentRuns, MAIN_REQUESTER_SESSION_KEY, "main"),
|
||||
).toBe(0);
|
||||
if (attachRequesterFinal) {
|
||||
expect(append).toHaveBeenCalledExactlyOnceWith("completion delivered");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ import {
|
|||
loadSubagentRunsForControllerFromSqlite,
|
||||
loadSubagentRegistryFromSqlite,
|
||||
loadSubagentSessionListRunsFromSqlite,
|
||||
loadSubagentRunsForSessionsFromSqlite,
|
||||
loadSubagentRunsForSessionsInDatabase,
|
||||
saveSubagentRegistryChangesToSqlite,
|
||||
saveSubagentRegistryToSqlite,
|
||||
} from "./subagent-registry.store.sqlite.js";
|
||||
|
|
@ -332,10 +332,12 @@ describe("subagent registry sqlite store", () => {
|
|||
});
|
||||
|
||||
it("keeps full identity selection and records in one snapshot across an external move", async () => {
|
||||
const read = (keys: readonly string[]) => loadSubagentRunsForSessionsFromSqlite(keys, []);
|
||||
const run = createRun({ model: "original-model" });
|
||||
saveSubagentRegistryToSqlite(new Map([[run.runId, run]]));
|
||||
const { db, path: databasePath } = openOpenClawStateDatabase();
|
||||
const database = openOpenClawStateDatabase();
|
||||
const { db, path: databasePath } = database;
|
||||
const read = (keys: readonly string[]) =>
|
||||
loadSubagentRunsForSessionsInDatabase(database, keys, []);
|
||||
const writer = new DatabaseSync(databasePath);
|
||||
let moved = false;
|
||||
db.setAuthorizer((action, table, column) => {
|
||||
|
|
|
|||
|
|
@ -592,13 +592,6 @@ export function subagentRunsDurableBasisMatches(
|
|||
);
|
||||
}
|
||||
|
||||
export function loadSubagentRunsForSessionsFromSqlite(
|
||||
sessionKeys: readonly string[],
|
||||
inMemoryRuns: Iterable<SubagentRunReadRecord>,
|
||||
) {
|
||||
return loadSubagentRunsForSessions(openOpenClawStateDatabase(), sessionKeys, inMemoryRuns);
|
||||
}
|
||||
|
||||
/** Saves the complete subagent run snapshot to sqlite and prunes rows not in the snapshot. */
|
||||
export function saveSubagentRegistryToSqlite(runs: Map<string, SubagentRunRecord>): void {
|
||||
const values = [...runs.values()].map(bindSubagentRunRecord);
|
||||
|
|
|
|||
|
|
@ -1,14 +1,12 @@
|
|||
export * from "./subagent-registry.js";
|
||||
export {
|
||||
buildSubagentSessionListReadIndex,
|
||||
countActiveDescendantRuns,
|
||||
countPendingDescendantRuns,
|
||||
getLatestLiveSubagentRunByChildSessionKey,
|
||||
getLatestSubagentRunByChildSessionKey,
|
||||
getSubagentRunByChildSessionKey,
|
||||
getSubagentSessionRuntimeMs,
|
||||
getSubagentSessionStartedAt,
|
||||
hasDescendantRunAwaitingSettle,
|
||||
isSubagentRunLive,
|
||||
isSubagentSessionRunActive,
|
||||
listSubagentRunsForController,
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ import { mockRegistryRequesterWakeMutation } from "./subagent-registry-lifecycle
|
|||
import { subagentRuns } from "./subagent-registry-memory.js";
|
||||
import { registerSubagentResultRefreshCases } from "./subagent-registry-result-refresh.test-support.js";
|
||||
import { persistSubagentRunsToDiskOrThrow } from "./subagent-registry-state.js";
|
||||
import { registerYieldedParentCleanupCase } from "./subagent-registry-yielded-cleanup.test-support.js";
|
||||
import {
|
||||
observeRootWork,
|
||||
registerBrowserCleanupBoundaryTests,
|
||||
|
|
@ -145,7 +146,6 @@ vi.mock("./subagent-registry-state.js", async () => ({
|
|||
getSubagentRunsSnapshotForChildSession: mocks.getSubagentRunsSnapshotForChildSession,
|
||||
getSubagentRunsSnapshotForController: mocks.getSubagentRunsSnapshotForController,
|
||||
getSubagentRunsSnapshotForRead: mocks.getSubagentRunsSnapshotForRead,
|
||||
getSubagentRunsSnapshotForSessions: mocks.getSubagentRunsSnapshotForRead,
|
||||
getSubagentMaintenanceRunsSnapshotForRead: mocks.getSubagentRunsSnapshotForRead,
|
||||
...(await import("../../subagent-test-fixtures.test-helpers.js")).createSubagentPersistenceMock(
|
||||
mocks,
|
||||
|
|
@ -2519,64 +2519,71 @@ describe("subagent registry seam flow", () => {
|
|||
});
|
||||
|
||||
it("retires stable operator cancellation despite a late persisted completion", async () => {
|
||||
{
|
||||
const now = Date.parse("2026-03-24T12:00:00Z");
|
||||
const startedAt = now - 10_000;
|
||||
const killedAt = now - 1_000;
|
||||
const completedAt = now;
|
||||
const runId = "run-killed-stable-cancellation";
|
||||
const childSessionKey = "agent:main:subagent:stable-cancellation";
|
||||
mocks.entries = {
|
||||
[childSessionKey]: {
|
||||
lifecycleRevision: "revision-stable-cancellation",
|
||||
sessionId: "sess-stable-cancellation",
|
||||
updatedAt: completedAt,
|
||||
status: "done",
|
||||
startedAt,
|
||||
endedAt: completedAt,
|
||||
},
|
||||
};
|
||||
mod.addSubagentRunForTests(
|
||||
makeKilledRun(killedAt, {
|
||||
runId,
|
||||
childSessionKey,
|
||||
task: "preserve operator cancellation",
|
||||
killReconciliation: { killedAt, taskCancellationAccepted: true },
|
||||
cleanup: "delete",
|
||||
expectsCompletionMessage: true,
|
||||
createdAt: startedAt,
|
||||
startedAt,
|
||||
archiveAtMs: Date.now(),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(killedAt + 5 * 60_000).toBeGreaterThan(Date.now());
|
||||
vi.setSystemTime(killedAt + 5 * 60_000);
|
||||
|
||||
await mod.testing.sweepOnceForTests();
|
||||
|
||||
await waitForFast(() => {
|
||||
expect(
|
||||
mod
|
||||
.listSubagentRunsForRequester("agent:main:main")
|
||||
.some((entry) => entry.runId === runId),
|
||||
).toBe(false);
|
||||
expect(mocks.callGateway).toHaveBeenCalledWith({
|
||||
method: "sessions.delete",
|
||||
params: {
|
||||
key: childSessionKey,
|
||||
deleteTranscript: true,
|
||||
emitLifecycleHooks: false,
|
||||
expectedLifecycleRevision: "revision-stable-cancellation",
|
||||
expectedSessionId: "sess-stable-cancellation",
|
||||
const config = mocks.getRuntimeConfig();
|
||||
await mocks.getRuntimeConfig.withImplementation(
|
||||
() => ({
|
||||
...config,
|
||||
session: { ...config.session, store: mocks.resolveStorePath() },
|
||||
}),
|
||||
async () => {
|
||||
const now = Date.parse("2026-03-24T12:00:00Z");
|
||||
const startedAt = now - 10_000;
|
||||
const killedAt = now - 1_000;
|
||||
const completedAt = now;
|
||||
const runId = "run-killed-stable-cancellation";
|
||||
const childSessionKey = "agent:main:subagent:stable-cancellation";
|
||||
mocks.entries = {
|
||||
[childSessionKey]: {
|
||||
lifecycleRevision: "revision-stable-cancellation",
|
||||
sessionId: "sess-stable-cancellation",
|
||||
updatedAt: completedAt,
|
||||
status: "done",
|
||||
startedAt,
|
||||
endedAt: completedAt,
|
||||
},
|
||||
timeoutMs: 10_000,
|
||||
assertDispatchCurrent: expect.any(Function),
|
||||
prepareDispatchCurrent: expect.any(Function),
|
||||
};
|
||||
mod.addSubagentRunForTests(
|
||||
makeKilledRun(killedAt, {
|
||||
runId,
|
||||
childSessionKey,
|
||||
task: "preserve operator cancellation",
|
||||
killReconciliation: { killedAt, taskCancellationAccepted: true },
|
||||
cleanup: "delete",
|
||||
expectsCompletionMessage: true,
|
||||
createdAt: startedAt,
|
||||
startedAt,
|
||||
archiveAtMs: Date.now(),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(killedAt + 5 * 60_000).toBeGreaterThan(Date.now());
|
||||
vi.setSystemTime(killedAt + 5 * 60_000);
|
||||
|
||||
await mod.testing.sweepOnceForTests();
|
||||
|
||||
await waitForFast(() => {
|
||||
expect(
|
||||
mod
|
||||
.listSubagentRunsForRequester("agent:main:main")
|
||||
.some((entry) => entry.runId === runId),
|
||||
).toBe(false);
|
||||
expect(mocks.callGateway).toHaveBeenCalledWith({
|
||||
method: "sessions.delete",
|
||||
params: {
|
||||
key: childSessionKey,
|
||||
deleteTranscript: true,
|
||||
emitLifecycleHooks: false,
|
||||
expectedLifecycleRevision: "revision-stable-cancellation",
|
||||
expectedSessionId: "sess-stable-cancellation",
|
||||
},
|
||||
timeoutMs: 10_000,
|
||||
assertDispatchCurrent: expect.any(Function),
|
||||
prepareDispatchCurrent: expect.any(Function),
|
||||
});
|
||||
});
|
||||
});
|
||||
expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled();
|
||||
}
|
||||
expect(mocks.runSubagentAnnounceFlow).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("restores an explicit timeout that predates stable operator cancellation", async () => {
|
||||
|
|
@ -3648,55 +3655,7 @@ describe("subagent registry seam flow", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("wakes a sessions_yield-paused parent when pending descendants settle", async () => {
|
||||
mocks.entries = {
|
||||
"agent:main:subagent:parent": {
|
||||
sessionId: "sess-parent",
|
||||
updatedAt: 1,
|
||||
},
|
||||
"agent:main:subagent:child": {
|
||||
sessionId: "sess-child",
|
||||
updatedAt: 1,
|
||||
},
|
||||
};
|
||||
|
||||
mod.addSubagentRunForTests({
|
||||
runId: "run-yielded-parent",
|
||||
childSessionKey: "agent:main:subagent:parent",
|
||||
task: "yielded parent waiting on descendants",
|
||||
createdAt: Date.parse("2026-06-26T02:17:00Z"),
|
||||
startedAt: Date.parse("2026-06-26T02:18:00Z"),
|
||||
endedAt: Date.parse("2026-06-26T02:19:00Z"),
|
||||
pauseReason: "sessions_yield",
|
||||
wakeOnDescendantSettle: true,
|
||||
cleanupHandled: false,
|
||||
cleanupCompletedAt: undefined,
|
||||
});
|
||||
|
||||
await mod.registerSubagentRun({
|
||||
runId: "run-yielded-child-finished",
|
||||
requesterSessionKey: "agent:main:subagent:parent",
|
||||
requesterDisplayKey: "parent",
|
||||
task: "descendant settles after yield",
|
||||
});
|
||||
|
||||
await waitForFast(() => {
|
||||
expect(mocks.runSubagentAnnounceFlow).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
expectRecordFields(
|
||||
getMockCallArg(mocks.runSubagentAnnounceFlow, 0, 0, "child finished announce"),
|
||||
{ childRunId: "run-yielded-child-finished" },
|
||||
"child finished announce params",
|
||||
);
|
||||
expectRecordFields(
|
||||
getMockCallArg(mocks.runSubagentAnnounceFlow, 1, 0, "yielded parent wake announce"),
|
||||
{
|
||||
childRunId: "run-yielded-parent",
|
||||
wakeOnDescendantSettle: true,
|
||||
},
|
||||
"yielded parent wake announce params",
|
||||
);
|
||||
});
|
||||
registerYieldedParentCleanupCase({ getRegistry: () => mod, mocks });
|
||||
|
||||
it("defers the killed hook until the provisional result reconciles", async () => {
|
||||
mockPendingAgentWait();
|
||||
|
|
@ -4081,7 +4040,16 @@ describe("subagent registry seam flow", () => {
|
|||
workspaceDir: undefined,
|
||||
});
|
||||
});
|
||||
expect(mocks.persistSubagentRunsToDisk).toHaveBeenCalled();
|
||||
const stored = mocks.persistSubagentRunsToDiskOrThrow.mock.calls.at(-1)?.[0].get(runId);
|
||||
expect(stored).toMatchObject({
|
||||
cleanupCompletedAt: now,
|
||||
delivery: {
|
||||
status: "discarded",
|
||||
payload: undefined,
|
||||
discardedAt: now,
|
||||
discardReason: "expired",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("does not emit ended hooks before suspended delete retirement is durable", async () => {
|
||||
|
|
|
|||
|
|
@ -11,10 +11,7 @@ import {
|
|||
bindGatewayContextResolver,
|
||||
getGatewayContextResolver,
|
||||
} from "../../../plugins/runtime/gateway-request-scope.js";
|
||||
import {
|
||||
isGatewayRestartDraining,
|
||||
runWithGatewayIndependentRootWorkAdmission,
|
||||
} from "../../../process/gateway-work-admission.js";
|
||||
import { runWithGatewayIndependentRootWorkAdmission } from "../../../process/gateway-work-admission.js";
|
||||
import type { OpenClawStateWorkerContext } from "../../../state/openclaw-state-worker-context.types.js";
|
||||
import { prependAgentSteeringPrompt } from "../../agent-steering-queue.js";
|
||||
import { resolveAgentTimeoutMs } from "../../timeout.js";
|
||||
|
|
@ -25,6 +22,7 @@ import { SUBAGENT_ENDED_REASON_ERROR } from "./subagent-lifecycle-events.js";
|
|||
import { createSubagentRegistryCompletionRuntime } from "./subagent-registry-completion-runtime.js";
|
||||
import { emitSubagentProgressEndedHook } from "./subagent-registry-completion.js";
|
||||
import { createSubagentRegistryContextCleanup } from "./subagent-registry-context-cleanup.js";
|
||||
import { createSubagentDeliveryResumeScheduling } from "./subagent-registry-delivery-resume.js";
|
||||
import {
|
||||
callSubagentRegistryGateway,
|
||||
loadSubagentAnnounceModule,
|
||||
|
|
@ -153,7 +151,7 @@ const subagentLifecycleController = new SubagentLifecycleController({
|
|||
clearPendingLifecycleError,
|
||||
// Lifecycle wiring precedes publicApi construction; inject this read query
|
||||
// as a late-bound callback instead of threading a partially built API object.
|
||||
countPendingDescendantRuns: (rootSessionKey) => countPendingDescendantRuns(rootSessionKey),
|
||||
countPendingDescendantRuns,
|
||||
getLatestRunForChildSession: getLatestLiveSubagentRunByChildSessionKey,
|
||||
suppressAnnounceForSteerRestart: contextCleanup.suppressAnnounceForSteerRestart,
|
||||
shouldEmitEndedHookForRun: contextCleanup.shouldEmitEndedHookForRun,
|
||||
|
|
@ -200,60 +198,16 @@ registerSystemEventStoreOwner(
|
|||
suspendReplacedNotificationsInBackground,
|
||||
);
|
||||
|
||||
function scheduleSubagentDeliveryResumeRetry(
|
||||
runId: string,
|
||||
scheduledEntry: SubagentRunRecord,
|
||||
waitMs: number,
|
||||
) {
|
||||
const timer = setTimeout(() => {
|
||||
resumeRetryTimers.delete(timer);
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
if (subagentRuns.get(runId) !== scheduledEntry) {
|
||||
resumedRuns.delete(runId);
|
||||
return;
|
||||
}
|
||||
resumedRuns.delete(runId);
|
||||
resumeSubagentRun(runId);
|
||||
}, "subagents:resume-retry").catch((error: unknown) => {
|
||||
log.warn("failed to resume subagent delivery retry", { runId, error });
|
||||
if (
|
||||
isGatewayRestartDraining() &&
|
||||
subagentRuns.get(runId) === scheduledEntry &&
|
||||
typeof scheduledEntry.cleanupCompletedAt !== "number"
|
||||
) {
|
||||
scheduleSubagentDeliveryResumeRetry(
|
||||
runId,
|
||||
scheduledEntry,
|
||||
Math.max(waitMs, GATEWAY_ADMISSION_RETRY_DELAY_MS),
|
||||
);
|
||||
return;
|
||||
}
|
||||
resumedRuns.delete(runId);
|
||||
});
|
||||
}, waitMs);
|
||||
timer.unref?.();
|
||||
resumeRetryTimers.add(timer);
|
||||
}
|
||||
|
||||
function finalizeResumedAnnounceGiveUpInBackground(
|
||||
runId: string,
|
||||
entry: SubagentRunRecord,
|
||||
reason: "expiry" | "permanent_failure",
|
||||
) {
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await finalizeResumedAnnounceGiveUp({ runId, entry, reason });
|
||||
}, "subagents:delivery-finalize").catch((error: unknown) => {
|
||||
log.warn("failed to finalize exhausted subagent delivery", { runId, reason, error });
|
||||
if (
|
||||
isGatewayRestartDraining() &&
|
||||
subagentRuns.get(runId) === entry &&
|
||||
typeof entry.cleanupCompletedAt !== "number"
|
||||
) {
|
||||
scheduleSubagentDeliveryResumeRetry(runId, entry, GATEWAY_ADMISSION_RETRY_DELAY_MS);
|
||||
resumedRuns.add(runId);
|
||||
}
|
||||
const { scheduleSubagentDeliveryResumeRetry, finalizeResumedAnnounceGiveUpInBackground } =
|
||||
createSubagentDeliveryResumeScheduling({
|
||||
runs: subagentRuns,
|
||||
resumedRuns,
|
||||
resumeRetryTimers,
|
||||
resumeSubagentRun,
|
||||
finalizeResumedAnnounceGiveUp,
|
||||
warn: (message, meta) => log.warn(message, meta),
|
||||
admissionRetryDelayMs: GATEWAY_ADMISSION_RETRY_DELAY_MS,
|
||||
});
|
||||
}
|
||||
|
||||
export function resumeSubagentRun(runId: string, source: "live" | "restore" = "live") {
|
||||
if (!runId || resumedRuns.has(runId)) {
|
||||
|
|
@ -489,6 +443,11 @@ const subagentSweeper = createSubagentRegistrySweeper({
|
|||
resumeRequesterSettleWake,
|
||||
startSubagentAnnounceCleanupFlow,
|
||||
completeCleanupBookkeeping,
|
||||
isEndedHookOwnerCurrent: subagentLifecycleController.isEndedHookOwnerCurrent,
|
||||
sessionEffectsHostCurrent: (entry) =>
|
||||
subagentLifecycleController.sessionEffectsHostCurrent(entry),
|
||||
shouldSuppressSessionEffects: (entry, effects) =>
|
||||
subagentLifecycleController.shouldSuppressSessionEffects(entry, effects),
|
||||
discardTerminalDelivery: SubagentLifecycleController.discardTerminalDelivery,
|
||||
shouldEmitEndedHookForRun: contextCleanup.shouldEmitEndedHookForRun,
|
||||
emitSubagentEndedHookForRun: contextCleanup.emitSubagentEndedHookForRun,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
/** Admitted parent, worker, and registered spawn fixtures shared by recursive boundary proofs. */
|
||||
import { vi } from "vitest";
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import { expect, it, vi, type Mock } from "vitest";
|
||||
import { createDeferred } from "../../../../test/helpers/promise.js";
|
||||
import { getRuntimeConfig } from "../../../config/config.js";
|
||||
import { registerChatAbortController } from "../../../gateway/chat-abort.js";
|
||||
import type { createGatewayInstanceRuntime } from "../../../gateway/server-instance-runtime.js";
|
||||
import { createChatAbortContext } from "../../../gateway/server-methods/chat.abort.test-helpers.js";
|
||||
import type { GatewayRequestContext } from "../../../gateway/server-methods/types.js";
|
||||
import { placementTurnOwner } from "../../../gateway/worker-environments/placement-record.js";
|
||||
|
|
@ -17,7 +20,9 @@ import {
|
|||
} from "../../../plugins/runtime/gateway-request-scope.js";
|
||||
import { AsyncWorkScope } from "../../../shared/async-work-scope.js";
|
||||
import { openOpenClawStateDatabase } from "../../../state/openclaw-state-db.js";
|
||||
import { normalizeAcceptedSessionSpawnResult } from "../../accepted-session-spawn.js";
|
||||
import {
|
||||
resolvePreparedRunAdmission,
|
||||
createAdmittedRunOperatorAuthority,
|
||||
createOperationalRunInstanceRef,
|
||||
getAdmittedRunDelegatedAuthority,
|
||||
|
|
@ -25,6 +30,7 @@ import {
|
|||
type AdmittedRunOperatorAuthority,
|
||||
} from "../../admitted-run-context.js";
|
||||
import { finalizeAgentTools } from "../../agent-tools.finalize.js";
|
||||
import type { EmbeddedAgentRunResult } from "../../embedded-agent.js";
|
||||
import { createAgentsWaitTool } from "../../tools/agents-wait-tool.js";
|
||||
import {
|
||||
createAdmittedGatewayToolCallerIdentity,
|
||||
|
|
@ -260,3 +266,157 @@ export function createBoundSpawnInvocation(
|
|||
),
|
||||
);
|
||||
}
|
||||
|
||||
type BoundParent = Awaited<ReturnType<typeof createSpawnBoundaryParent>>;
|
||||
type GatewayRuntime = ReturnType<typeof createGatewayInstanceRuntime>;
|
||||
|
||||
export function registerYieldedRequesterBatchCase(options: {
|
||||
createBoundParent: () => Promise<BoundParent>;
|
||||
createBoundGateway: (bound: BoundParent) => Promise<{
|
||||
context: GatewayRequestContext;
|
||||
runtime: GatewayRuntime;
|
||||
}>;
|
||||
closeBoundGateway: (
|
||||
bound: BoundParent,
|
||||
runtime: GatewayRuntime,
|
||||
childRunId?: string,
|
||||
) => Promise<unknown[]>;
|
||||
waitForEmbeddedRun: (
|
||||
bound: BoundParent,
|
||||
runId: string,
|
||||
started?: Promise<void>,
|
||||
calls?: number,
|
||||
) => Promise<void>;
|
||||
runEmbeddedAgent: Mock<typeof import("../../embedded-agent.js").runEmbeddedAgent>;
|
||||
throwBoundFailures: (failures: unknown[]) => void;
|
||||
}) {
|
||||
it("continues a yielded nested parent once through its accepted child batch", async () => {
|
||||
const registry = await import("../registry/subagent-registry.js");
|
||||
const { subagentRuns } = await import("../registry/subagent-registry-memory.js");
|
||||
const { loadSubagentRegistryFromSqlite } =
|
||||
await import("../registry/subagent-registry.store.sqlite.js");
|
||||
const { settleSubagentRegistryPersistenceWork } =
|
||||
await import("../registry/subagent-registry.persistence.test-support.js");
|
||||
const announce = await import("../announce/subagent-announce.js");
|
||||
const nativeAnnounce = await vi.importActual<typeof announce>(
|
||||
"../announce/subagent-announce.js",
|
||||
);
|
||||
vi.mocked(announce.runSubagentAnnounceFlow).mockImplementation(
|
||||
nativeAnnounce.runSubagentAnnounceFlow,
|
||||
);
|
||||
const bound = await options.createBoundParent();
|
||||
const { context, runtime } = await options.createBoundGateway(bound);
|
||||
const childStarted = createDeferred();
|
||||
const childResult = createDeferred<EmbeddedAgentRunResult>();
|
||||
const parentStarted = createDeferred();
|
||||
const parentCalls: Array<{
|
||||
runId: string;
|
||||
childSessionKey: string | undefined;
|
||||
hasAuthority: boolean;
|
||||
prompt: string;
|
||||
}> = [];
|
||||
const admissionFailures: unknown[] = [];
|
||||
options.runEmbeddedAgent.mockImplementation(async (params) => {
|
||||
try {
|
||||
const admitted = await resolvePreparedRunAdmission({
|
||||
runId: params.runId,
|
||||
runtimeKind: "embedded",
|
||||
admittedRunContext: params.admittedRunContext,
|
||||
preparedRunAdmission: params.preparedRunAdmission,
|
||||
});
|
||||
await params.onExecutionStarted?.();
|
||||
if (params.sessionKey !== bound.parentSessionKey) {
|
||||
childStarted.resolve();
|
||||
return await childResult.promise;
|
||||
}
|
||||
const current = subagentRuns.get(params.runId);
|
||||
parentCalls.push({
|
||||
runId: params.runId,
|
||||
childSessionKey: current?.childSessionKey,
|
||||
hasAuthority: getAdmittedRunDelegatedAuthority(admitted) !== undefined,
|
||||
prompt: params.prompt,
|
||||
});
|
||||
parentStarted.resolve();
|
||||
return {
|
||||
payloads: [{ text: "Nested parent is complete." }],
|
||||
meta: { durationMs: 1, finalAssistantVisibleText: "Nested parent is complete." },
|
||||
};
|
||||
} catch (error) {
|
||||
admissionFailures.push(error);
|
||||
(params.sessionKey === bound.parentSessionKey ? parentStarted : childStarted).resolve();
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
let childRunId: string | undefined;
|
||||
const failures: unknown[] = [];
|
||||
try {
|
||||
await registry.registerSubagentRun({
|
||||
runId: bound.parentRunId,
|
||||
childSessionKey: bound.parentSessionKey,
|
||||
requesterSessionKey: "agent:main:main",
|
||||
requesterAgentId: "main",
|
||||
requesterDisplayKey: "main",
|
||||
task: "Continue after an accepted child completes",
|
||||
cleanup: "keep",
|
||||
expectsCompletionMessage: false,
|
||||
gatewayContextResolver: () => context,
|
||||
});
|
||||
const spawned = await createBoundSpawnInvocation(bound, { context: "isolated" })();
|
||||
const accepted = expectDefined(
|
||||
normalizeAcceptedSessionSpawnResult(spawned),
|
||||
"actual accepted spawn",
|
||||
);
|
||||
expect(accepted.expectsCompletionMessage).toBe(true);
|
||||
childRunId = accepted.runId;
|
||||
await options.waitForEmbeddedRun(bound, childRunId, childStarted.promise);
|
||||
expect(
|
||||
await registry.markRequesterTurnYielded({
|
||||
requesterSessionKey: bound.parentSessionKey,
|
||||
requesterAgentId: "main",
|
||||
requesterTurnRunId: bound.parentRunId,
|
||||
}),
|
||||
).toBe(1);
|
||||
expect(
|
||||
await registry.settleRequesterAfterSessionSpawns({
|
||||
requesterSessionKey: bound.parentSessionKey,
|
||||
requesterTurnRunId: bound.parentRunId,
|
||||
requesterYielded: true,
|
||||
acceptedSessionSpawns: [accepted],
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(subagentRuns.get(bound.parentRunId)?.pauseReason).toBe("sessions_yield");
|
||||
expect(subagentRuns.get(childRunId)?.requesterSettleWake).toMatchObject({
|
||||
requesterYieldBatch: true,
|
||||
rearmGeneration: 1,
|
||||
batchRunIds: [childRunId],
|
||||
});
|
||||
bound.admission.close();
|
||||
bound.parent.cleanup();
|
||||
childResult.resolve({
|
||||
payloads: [{ text: "Nested child result." }],
|
||||
meta: { durationMs: 1, finalAssistantVisibleText: "Nested child result." },
|
||||
});
|
||||
await options.waitForEmbeddedRun(bound, bound.parentRunId, parentStarted.promise, 2);
|
||||
await settleSubagentRegistryPersistenceWork();
|
||||
expect(admissionFailures).toEqual([]);
|
||||
expect(parentCalls).toHaveLength(1);
|
||||
expect(parentCalls[0]).toMatchObject({
|
||||
childSessionKey: bound.parentSessionKey,
|
||||
hasAuthority: true,
|
||||
});
|
||||
expect(parentCalls[0]?.prompt).toContain("Nested child result.");
|
||||
const durable = loadSubagentRegistryFromSqlite().get(childRunId);
|
||||
expect(durable?.delivery?.status).toBe("delivered");
|
||||
expect(durable?.requesterSettleWake).toBeUndefined();
|
||||
} catch (error) {
|
||||
failures.push(error);
|
||||
} finally {
|
||||
childResult.resolve({
|
||||
payloads: [{ text: "Nested child result." }],
|
||||
meta: { durationMs: 1 },
|
||||
});
|
||||
failures.push(...(await options.closeBoundGateway(bound, runtime, childRunId)));
|
||||
options.throwBoundFailures(failures);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ import {
|
|||
createBoundWorker,
|
||||
createSpawnBoundaryParent,
|
||||
createSpawnOperatorSource,
|
||||
registerYieldedRequesterBatchCase,
|
||||
} from "./subagent-spawn.production-boundary.test-support.js";
|
||||
import { registerOperatorSpawnRollbackCases } from "./subagent-spawn.rollback.test-support.js";
|
||||
|
||||
|
|
@ -308,15 +309,16 @@ async function waitForEmbeddedRun(
|
|||
bound: Awaited<ReturnType<typeof createBoundParent>>,
|
||||
childRunId: string,
|
||||
started?: Promise<void>,
|
||||
calls = 1,
|
||||
) {
|
||||
try {
|
||||
if (started) {
|
||||
await withTimeout(started, COLD_MODEL_ENTRY_TIMEOUT_MS, {
|
||||
message: "embedded execution entry timed out",
|
||||
});
|
||||
expect(runEmbeddedAgent).toHaveBeenCalledOnce();
|
||||
expect(runEmbeddedAgent).toHaveBeenCalledTimes(calls);
|
||||
} else {
|
||||
await vi.waitFor(() => expect(runEmbeddedAgent).toHaveBeenCalledOnce(), {
|
||||
await vi.waitFor(() => expect(runEmbeddedAgent).toHaveBeenCalledTimes(calls), {
|
||||
timeout: 15_000,
|
||||
});
|
||||
}
|
||||
|
|
@ -391,6 +393,14 @@ describe("recursive spawn production boundary", () => {
|
|||
parentSessionKey,
|
||||
parentRunId,
|
||||
});
|
||||
registerYieldedRequesterBatchCase({
|
||||
createBoundParent,
|
||||
createBoundGateway,
|
||||
closeBoundGateway,
|
||||
waitForEmbeddedRun,
|
||||
runEmbeddedAgent,
|
||||
throwBoundFailures,
|
||||
});
|
||||
registerOperatorSpawnRollbackCases({
|
||||
createBoundParent,
|
||||
createBoundGateway,
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ export function registerRecoveredSubagentSessionEventTest({
|
|||
publication.onCommitted?.();
|
||||
},
|
||||
clearPendingLifecycleError: vi.fn(),
|
||||
countPendingDescendantRuns: () => 0,
|
||||
countPendingDescendantRuns: async () => 0,
|
||||
getLatestRunForChildSession: () => null,
|
||||
suppressAnnounceForSteerRestart: () => false,
|
||||
shouldEmitEndedHookForRun: () => false,
|
||||
|
|
|
|||
|
|
@ -173,6 +173,8 @@ export const databaseWorkerCoreTestFiles = [
|
|||
"src/agents/subagents/announce/subagent-announce.requester-settle-results.test.ts",
|
||||
"src/agents/subagents/announce/subagent-announce.requester-settle-wake.test.ts",
|
||||
"src/agents/subagents/registry/subagent-control.accounting.test.ts",
|
||||
"src/agents/subagents/registry/subagent-announcement.worker.test.ts",
|
||||
"src/agents/subagents/registry/subagent-announcement-delivery.worker.test.ts",
|
||||
"src/agents/subagents/registry/subagent-registry.persistence.test.ts",
|
||||
"src/agents/subagents/registry/subagent-registry-lifecycle.test.ts",
|
||||
"src/agents/subagents/registry/subagent-registry.test.ts",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue