diff --git a/extensions/codex/src/app-server/settled-turn-finalizer.test-support.ts b/extensions/codex/src/app-server/settled-turn-finalizer.test-support.ts new file mode 100644 index 000000000000..cb4de1222c36 --- /dev/null +++ b/extensions/codex/src/app-server/settled-turn-finalizer.test-support.ts @@ -0,0 +1,101 @@ +import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api"; +import { createPluginRuntimeMock } from "openclaw/plugin-sdk/plugin-test-runtime"; +import { expect, vi } from "vitest"; +import plugin from "../../index.js"; +import * as boundedTurn from "./bounded-turn.js"; +import { + buildEmptyToolTelemetry, + createParams, + createProjector, + forCurrentTurn, + TURN_ID, +} from "./event-projector.test-harness.js"; +import { CodexSettledTurnContext } from "./settled-turn-context.js"; +import { projectSettledCodexMessages } from "./settled-turn-projection.js"; +import { createCodexRuntimePlanFixture } from "./thread-lifecycle.test-fixtures.js"; +import { codexTranscriptMirrorRuntime } from "./transcript-mirror.js"; + +export { registerCodexEventProjectorTestLifecycle } from "./event-projector.test-harness.js"; + +export async function createCodexSettledFinalizerTestFixture( + options: { failedTool?: boolean } = {}, +) { + const runBounded = vi + .spyOn(boundedTurn, "runBoundedCodexAppServerTurn") + .mockImplementation(vi.fn()); + const mirror = vi.spyOn(codexTranscriptMirrorRuntime, "mirror"); + const params = await createParams(); + params.runId = "run-settled"; + params.sessionId = "session-settled"; + params.runtimePlan = createCodexRuntimePlanFixture(); + params.runtimePlan.auth = { + providerForAuth: "openai", + authProfileProviderForAuth: "openai", + modelRoute: { + provider: "openai", + modelId: params.modelId, + api: "openai-responses", + baseUrl: "https://api.openai.com/v1", + authRequirement: "api-key", + requestTransportOverrides: "none", + }, + }; + params.resolvedApiKey = "synthetic-finalizer-key"; + params.authProfileStore = { version: 1, profiles: {} }; + params.timeoutMs = 60_000; + const projector = await createProjector(params); + const command = { + type: "commandExecution", + id: "command-settled", + command: "printf 'settled'", + cwd: "/workspace", + processId: null, + source: "agent", + commandActions: [], + }; + await projector.handleNotification( + forCurrentTurn("item/started", { + item: { ...command, status: "inProgress", aggregatedOutput: null, exitCode: null }, + }), + ); + await projector.handleNotification( + forCurrentTurn("item/completed", { + item: { + ...command, + status: options.failedTool ? "failed" : "completed", + aggregatedOutput: options.failedTool ? "command failed" : "settled", + exitCode: options.failedTool ? 1 : 0, + durationMs: 1, + }, + }), + ); + await projector.handleNotification( + forCurrentTurn("turn/completed", { + turn: { + id: TURN_ID, + status: "failed", + items: [], + error: { message: "Provider overloaded", codexErrorInfo: "serverOverloaded" }, + }, + }), + ); + expect(projector.settledTurnFailureFinalizationAllowed).toBe(true); + const attempt = projector.buildResult(buildEmptyToolTelemetry()); + await projector.closeProjection(); + const context = new CodexSettledTurnContext( + projectSettledCodexMessages(attempt.messagesSnapshot), + { model: params.modelId, modelProvider: "openai" }, + ); + attempt.settledTurnFinalizationContext = context; + const registerAgentHarness = + vi.fn["registerAgentHarness"]>(); + plugin.register( + createTestPluginApi({ + id: "codex", + runtime: createPluginRuntimeMock(), + registerAgentHarness, + }), + ); + expect(registerAgentHarness).toHaveBeenCalledOnce(); + return { attempt, params, harness: registerAgentHarness.mock.calls[0]![0], runBounded, mirror }; +} diff --git a/extensions/codex/src/app-server/settled-turn-finalizer.test.ts b/extensions/codex/src/app-server/settled-turn-finalizer.test.ts index 794f6bb126f8..373ffacdcd17 100644 --- a/extensions/codex/src/app-server/settled-turn-finalizer.test.ts +++ b/extensions/codex/src/app-server/settled-turn-finalizer.test.ts @@ -359,8 +359,8 @@ describe("runCodexSettledTurnFinalization", () => { }, ); - it.each([" ", "NO_REPLY"])( - "returns completed-empty output with native attribution for %j without transcript mutation", + it.each([" ", "NO_REPLY", " NO_REPLY\n", "no_reply"])( + "preserves non-visible output with native attribution for %j without transcript mutation", async (text) => { mocks.runBounded.mockResolvedValue({ ...boundedResult(), text }); @@ -373,7 +373,7 @@ describe("runCodexSettledTurnFinalization", () => { assistant: { provider: "openai", model: "synthetic-summary-model", - content: [{ type: "text", text: "" }], + content: [{ type: "text", text: text.trim() }], }, }); expect(mocks.runBounded).toHaveBeenCalledOnce(); @@ -453,6 +453,41 @@ describe("runCodexSettledTurnFinalization", () => { expect(mocks.mirror).toHaveBeenCalledOnce(); }); + it.each(["NO_REPLY", " "])( + "preserves %j after an attested managed Stop-hook continuation without mirroring", + async (text) => { + mocks.runBounded.mockResolvedValue({ + ...boundedResult(), + text, + managedHooksEnabled: true, + items: [ + { id: "draft", type: "agentMessage", text: "An earlier draft." }, + { + id: "hook", + type: "hookPrompt", + fragments: [{ text: "Revise the answer.", hookRunId: "managed-stop-1" }], + }, + { id: "answer", type: "agentMessage", text }, + ], + }); + + await expect( + runCodexSettledTurnFinalization( + { attempt: createAttempt(), settledAttempt: createSettledAttempt() }, + {}, + ), + ).resolves.toMatchObject({ + assistant: { + provider: "openai", + model: "synthetic-summary-model", + content: [{ type: "text", text: text.trim() }], + }, + }); + expect(mocks.runBounded).toHaveBeenCalledOnce(); + expect(mocks.mirror).not.toHaveBeenCalled(); + }, + ); + it.each([undefined, false])( "rejects unattested hook continuations before transcript mutation (%s)", async (managedHooksEnabled) => { diff --git a/extensions/codex/src/app-server/settled-turn-finalizer.ts b/extensions/codex/src/app-server/settled-turn-finalizer.ts index 2033b070d1d5..63924cb0dea3 100644 --- a/extensions/codex/src/app-server/settled-turn-finalizer.ts +++ b/extensions/codex/src/app-server/settled-turn-finalizer.ts @@ -96,13 +96,15 @@ export async function runCodexSettledTurnFinalization( assertCodexPassiveTurnItems(bounded.items, attempt.prompt, "settled-turn finalization", { allowManagedHookPrompts: bounded.managedHooksEnabled, }); - const text = isSilentReplyText(bounded.text) ? "" : bounded.text.trim(); + const text = bounded.text.trim(); const assistant = createAttributedCodexAssistantMessage(attribution, text, { tokenUsage: bounded.usage, aborted: false, promptError: null, }); - if (!text) { + // The host distinguishes authored silence from missing output. Preserve the + // sentinel for its optional/required policy without mirroring a non-answer. + if (!text || isSilentReplyText(text)) { return { assistant, ...(bounded.usage ? { usage: bounded.usage } : {}) }; } diff --git a/extensions/codex/test-api.ts b/extensions/codex/test-api.ts index 08947f72e714..4e0731cf8060 100644 --- a/extensions/codex/test-api.ts +++ b/extensions/codex/test-api.ts @@ -42,6 +42,10 @@ export async function createCodexSessionInitializationFixtureForTest(params: { return await createCodexSessionInitializationFixture(params); } +// Host finalizer fixtures opt into Vitest hooks without affecting snapshot consumers. +export const loadCodexSettledFinalizerTestFixture = () => + import("./src/app-server/settled-turn-finalizer.test-support.js"); + type CodexHarnessPromptSnapshot = { developerInstructions: string; parentLocalInstructions: string | null; diff --git a/test/codex-settled-turn-finalization.integration.test.ts b/test/codex-settled-turn-finalization.integration.test.ts new file mode 100644 index 000000000000..6296de168be5 --- /dev/null +++ b/test/codex-settled-turn-finalization.integration.test.ts @@ -0,0 +1,199 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { loadCodexSettledFinalizerTestFixture } from "../extensions/codex/test-api.js"; +import { + prepareSystemAgentRunAdmission, + type AdmittedRunContext, +} from "../src/agents/admitted-run-context.js"; +import { + normalizeAgentRunAttemptTerminal, + projectAgentRunAttemptTerminal, +} from "../src/agents/agent-run-terminal-outcome.js"; +import { resolveSettledToolBatchEvidence } from "../src/agents/embedded-agent-runner/run/incomplete-turn-recovery.js"; +import { resolveReplayInvalidFlag } from "../src/agents/embedded-agent-runner/run/incomplete-turn-resolution.js"; +import { prepareTerminalWithSettledTurnFinalization } from "../src/agents/embedded-agent-runner/run/settled-turn-finalization.js"; +import { createSettledFinalizationTestInput } from "../src/agents/embedded-agent-runner/run/settled-turn-finalization.test-support.js"; +import { isEmbeddedRunTerminalTimeout } from "../src/agents/embedded-agent-runner/run/terminal-outcome.js"; +import { resolveEmbeddedRunTerminalTimeout } from "../src/agents/embedded-agent-runner/run/terminal-timeout.js"; + +const { createCodexSettledFinalizerTestFixture, registerCodexEventProjectorTestLifecycle } = + await loadCodexSettledFinalizerTestFixture(); + +registerCodexEventProjectorTestLifecycle(); + +describe("registered Codex finalizer host silence contract", () => { + let admission: ReturnType; + let admittedRunContext: AdmittedRunContext; + let fixture: Awaited>; + + beforeEach(async () => { + admission = prepareSystemAgentRunAdmission({}, "run-settled", "main", "codex-finalizer-test"); + admittedRunContext = await admission.admit("embedded"); + }); + + afterEach(() => admission.close()); + + async function createInput(options: { failedTool?: boolean; timedOut?: boolean } = {}) { + fixture = await createCodexSettledFinalizerTestFixture(options); + const { attempt, params } = fixture; + expect(attempt.terminal.kind).toBe("failed"); + expect(attempt.itemLifecycle).toMatchObject({ activeCount: 0, completedCount: 1 }); + expect(attempt.didSendViaMessagingTool).toBe(false); + expect(attempt.messagingToolSentTexts).toEqual([]); + expect(attempt.messagingToolSentMediaUrls).toEqual([]); + expect(resolveSettledToolBatchEvidence(attempt)).toMatchObject({ + allToolCallsRecorded: true, + allToolsProvenSettled: true, + hasUnsettledToolError: false, + }); + expect(attempt.settledTurnFinalizationContext).toMatchObject({ + data: expect.arrayContaining([expect.objectContaining({ type: "function_call_output" })]), + }); + if (options.timedOut) { + // Deadline ownership follows native projection; a timeout must block finalization. + attempt.terminal = normalizeAgentRunAttemptTerminal({ + ...projectAgentRunAttemptTerminal(attempt.terminal), + timedOut: true, + }); + } + const input = createSettledFinalizationTestInput(attempt, admittedRunContext); + input.finalization.harness = fixture.harness; + input.finalization.preparedAttempt = { ...input.finalization.preparedAttempt, ...params }; + input.finalization.modelApi = params.model.api; + input.terminalBase.provider = params.provider; + input.terminalBase.model = params.modelId; + input.terminalBase.activeErrorContext = { provider: params.provider, model: params.modelId }; + Object.assign(input.terminalBase.runParams, { + trigger: "heartbeat", + terminalReplyExpectation: "optional", + sourceReplyDeliveryMode: "automatic", + }); + return input; + } + + function returnBoundedText(text: string) { + const result = { + text, + items: [], + model: "synthetic-finalizer-model", + nativeSelection: { model: "synthetic-finalizer-model", modelProvider: "openai" }, + managedHooksEnabled: false, + }; + fixture.runBounded.mockResolvedValue(result); + return result; + } + + it.each([true, false, undefined])( + "honors optional authored silence independently of empty-reply permission (%s)", + async (allowed) => { + const input = await createInput(); + input.terminalBase.runParams.allowEmptyAssistantReplyAsSilent = allowed; + returnBoundedText(" NO_REPLY\n"); + + const result = await prepareTerminalWithSettledTurnFinalization(input); + + expect(fixture.runBounded).toHaveBeenCalledOnce(); + expect(fixture.runBounded).toHaveBeenCalledWith( + expect.objectContaining({ + isolation: "private-stdio", + requireNoExternalCapabilities: true, + }), + ); + expect(result.finalizationOutcome).toBe("answered"); + expect(result.attempt.assistantTexts).toEqual(["NO_REPLY"]); + expect(result.prepared.payloadsWithToolMedia ?? []).toEqual([]); + expect(fixture.mirror).not.toHaveBeenCalled(); + }, + ); + + it.each([ + { text: "no_reply", expectation: "optional", silent: true }, + { text: "NO_REPLY", expectation: "required", silent: false }, + { text: " ", expectation: "optional", silent: false }, + { text: " ", expectation: "required", silent: false }, + ] as const)("distinguishes $expectation $text output", async ({ text, expectation, silent }) => { + const input = await createInput(); + input.terminalBase.runParams.terminalReplyExpectation = expectation; + input.terminalBase.runParams.allowEmptyAssistantReplyAsSilent = true; + returnBoundedText(text); + + const result = await prepareTerminalWithSettledTurnFinalization(input); + + expect(fixture.runBounded).toHaveBeenCalledTimes(silent ? 1 : 2); + expect(result.finalizationOutcome).toBe(silent ? "answered" : "completed-empty"); + if (silent) { + expect(result.attempt.assistantTexts).toEqual([text]); + expect(result.prepared.payloadsWithToolMedia ?? []).toEqual([]); + } else { + expect(result.prepared.payloadsWithToolMedia).toEqual([ + expect.objectContaining({ + text: "The tool run finished, but no final summary was produced. I did not repeat any completed actions.", + }), + ]); + } + expect(fixture.mirror).not.toHaveBeenCalled(); + }); + + it.each([{ failedTool: true }, { timedOut: true }])( + "does not hide an original failure with authored silence: %j", + async (options) => { + const input = await createInput(options); + input.terminalBase.runParams.allowEmptyAssistantReplyAsSilent = true; + returnBoundedText("NO_REPLY"); + + const result = await prepareTerminalWithSettledTurnFinalization(input); + + expect(fixture.runBounded).toHaveBeenCalledTimes(options.timedOut ? 0 : 2); + expect(result.finalizationOutcome).toBe(options.timedOut ? "not-attempted" : "failed"); + expect(result.attempt).toBe(input.initial.attempt); + if (options.timedOut) { + expect(isEmbeddedRunTerminalTimeout(result.terminalState.outcome)).toBe(true); + expect(result.prepared.timedOutDuringPrompt).toBe(true); + expect(result.prepared.payloadsWithToolMedia ?? []).toEqual([]); + // The run loop presents timeout errors after finalization preparation. + const setTerminalLifecycleMeta = vi.fn(); + const timeout = resolveEmbeddedRunTerminalTimeout({ + terminalPrepared: result.prepared, + attempt: result.attempt, + terminalState: result.terminalState, + resolveReplayInvalid: (incompleteTurnText) => + resolveReplayInvalidFlag({ attempt: result.attempt, incompleteTurnText }), + setTerminalLifecycleMeta, + startedAtMs: Date.now(), + }); + expect(timeout?.payloads).toEqual([ + { text: expect.stringContaining("timed out"), isError: true }, + ]); + expect(timeout?.meta).toMatchObject({ + replayInvalid: true, + modelFallbackStopReason: "agent_run_terminal_timeout", + error: { kind: "incomplete_turn", fallbackSafe: false }, + }); + expect(setTerminalLifecycleMeta).toHaveBeenCalledOnce(); + expect(setTerminalLifecycleMeta).toHaveBeenCalledWith( + expect.objectContaining({ replayInvalid: true, livenessState: "blocked" }), + ); + } else { + expect(result.prepared.payloadsWithToolMedia?.[0]).toMatchObject({ isError: true }); + } + expect(fixture.mirror).not.toHaveBeenCalled(); + }, + ); + + it("preserves cancellation while the bounded finalizer returns authored silence", async () => { + const input = await createInput(); + const controller = new AbortController(); + input.finalization.abortSignal = controller.signal; + const boundedResult = returnBoundedText("NO_REPLY"); + fixture.runBounded.mockImplementation(async () => { + controller.abort(new Error("cancelled")); + return boundedResult; + }); + + const result = await prepareTerminalWithSettledTurnFinalization(input); + + expect(fixture.runBounded).toHaveBeenCalledOnce(); + expect(result.finalizationOutcome).toBe("failed"); + expect(result.attempt).toBe(input.initial.attempt); + expect(fixture.mirror).not.toHaveBeenCalled(); + }); +});