vrr/openclaw
2
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-02 11:35:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 16

fix(agents): finalize verified Code Mode turns

Browse source
This commit is contained in:
Vincent Koc 2026-07-31 20:58:16 +02:00
parent 69f2c627d3
commit 3f729252dc
No known key found for this signature in database
5 changed files with 362 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

316
src/agents/embedded-agent-runner/run.incomplete-turn.test.ts
View file

@ -39,6 +39,7 @@ import {
resolveRunLivenessState,
resolveSilentToolResultReplyPayload,
resolveSettledToolTerminalContinuationInstruction,
shouldLatchCodeModeReadOnlyForRun,
shouldRetryMissingAssistantTurn,
shouldRetrySilentErrorAssistantTurn,
shouldTreatEmptyAssistantReplyAsSilent,
@ -56,6 +57,13 @@ const NORMAL_CODE_MODE_CONTINUATION_INSTRUCTION = resolveCodeModeContinuationIns
targetlessSideEffectEvidence: null,
toolPolicy: "normal",
});
const VERIFIED_CODE_MODE_MUTATION_CONTINUATION_INSTRUCTION = resolveCodeModeContinuationInstruction(
{
mutationVerificationRequired: false,
targetlessSideEffectEvidence: false,
toolPolicy: "normal",
},
);
const VERIFY_CODE_MODE_MUTATION_CONTINUATION_INSTRUCTION = resolveCodeModeContinuationInstruction({
mutationVerificationRequired: true,
targetlessSideEffectEvidence: false,
@ -142,6 +150,21 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
).onUserMessagePersisted?.({ role: "user", content: "test prompt" });
}
it("latches read-only tools for confirmed or unresolved targetless Code Mode effects", () => {
expect(
shouldLatchCodeModeReadOnlyForRun({
mutationVerificationRequired: true,
targetlessSideEffectEvidence: null,
}),
).toBe(true);
expect(
shouldLatchCodeModeReadOnlyForRun({
mutationVerificationRequired: true,
targetlessSideEffectEvidence: true,
}),
).toBe(true);
});
it("counts failed tool results in trace tool summaries", async () => {
mockedRunEmbeddedAttempt.mockResolvedValueOnce(
makeAttemptResult({
@ -1303,8 +1326,8 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
const emptyStopAssistant = {
role: "assistant",
stopReason: "stop",
provider: "huggingface",
model: "Qwen/Qwen3.5-9B",
provider: "openai",
model: "gpt-5.6",
content: [],
} as unknown as NonNullable<EmbeddedRunAttemptResult["lastAssistant"]>;
mockedClassifyFailoverReason.mockReturnValue(null);
@ -1357,6 +1380,211 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
expectNoWarnMessageWith("settled post-tool turn lacked a final answer");
});
it("uses answer-only finalization after one verified Code Mode completion continuation", async () => {
const emptyStopAssistant = {
role: "assistant",
stopReason: "stop",
provider: "openai",
model: "gpt-5.6",
content: [],
} as unknown as NonNullable<EmbeddedRunAttemptResult["lastAssistant"]>;
mockedClassifyFailoverReason.mockReturnValue(null);
mockedRunEmbeddedAttempt
.mockResolvedValueOnce(
makeAttemptResult({
assistantTexts: [],
codeModeEngaged: true,
toolMetas: [
{
toolName: "edit",
replaySafe: false,
sideEffectFree: false,
codeModeHadTargetlessSideEffects: false,
codeModeUnverifiedMutationFileTargets: [{ path: "result.txt" }],
},
{
toolName: "read",
replaySafe: true,
sideEffectFree: true,
fileTarget: { path: "result.txt" },
fileTargetVerified: true,
},
],
replayMetadata: { hadPotentialSideEffects: true, replaySafe: false },
currentAttemptReplayMetadata: { hadPotentialSideEffects: true, replaySafe: false },
itemLifecycle: { startedCount: 2, completedCount: 2, activeCount: 0 },
lastAssistant: emptyStopAssistant,
currentAttemptAssistant: emptyStopAssistant,
}),
)
.mockResolvedValueOnce(
makeAttemptResult({
assistantTexts: [],
codeModeEngaged: true,
toolMetas: [{ toolName: "read", replaySafe: true, sideEffectFree: true }],
replayMetadata: { hadPotentialSideEffects: false, replaySafe: true },
currentAttemptReplayMetadata: { hadPotentialSideEffects: false, replaySafe: true },
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
lastAssistant: emptyStopAssistant,
currentAttemptAssistant: emptyStopAssistant,
}),
)
.mockResolvedValueOnce(
makeAttemptResult({
assistantTexts: ["CM-FRONTIER"],
lastAssistant: {
role: "assistant",
stopReason: "stop",
provider: "openai",
model: "gpt-5.6",
content: [{ type: "text", text: "CM-FRONTIER" }],
} as unknown as EmbeddedRunAttemptResult["lastAssistant"],
}),
);
mockedBuildEmbeddedRunPayloads
.mockReturnValueOnce([])
.mockReturnValueOnce([])
.mockReturnValueOnce([{ text: "CM-FRONTIER" }]);
const result = await runEmbeddedAgent({
...overflowBaseRunParams,
provider: "openai",
model: "gpt-5.6",
runId: "run-code-mode-answer-only-finalization",
});
expect(runAttemptCall(1).operation).toBe("attempt");
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(3);
expect(result.payloads?.[0]?.text).toBe("CM-FRONTIER");
expect(runAttemptCall(1).prompt).toBe(VERIFIED_CODE_MODE_MUTATION_CONTINUATION_INSTRUCTION);
expect(runAttemptCall(1).disableTools).not.toBe(true);
expect(runAttemptCall(2).operation).toBe("settled-tool-finalization");
expect(runAttemptCall(2).disableTools).toBe(true);
expect(runAttemptCall(2).skipPreparedUserTurnMessage).toBe(true);
expectWarnMessageWith("settled post-tool turn lacked a final answer");
});
it("surfaces a bounded incomplete turn when Code Mode finalization is unavailable", async () => {
registerAgentHarness({
id: "legacy-code-mode",
label: "Legacy Code Mode harness without settled-turn finalization",
supports: () => ({ supported: true, priority: 100 }),
runAttempt: async (params) => await mockedRunEmbeddedAttempt(params),
});
try {
const emptyStopAssistant = {
role: "assistant",
stopReason: "stop",
provider: "openai",
model: "gpt-5.6",
content: [],
} as unknown as NonNullable<EmbeddedRunAttemptResult["lastAssistant"]>;
mockedClassifyFailoverReason.mockReturnValue(null);
mockedRunEmbeddedAttempt.mockResolvedValue(
makeAttemptResult({
assistantTexts: [],
codeModeEngaged: true,
toolMetas: [{ toolName: "read", replaySafe: true, sideEffectFree: true }],
replayMetadata: { hadPotentialSideEffects: false, replaySafe: true },
currentAttemptReplayMetadata: { hadPotentialSideEffects: false, replaySafe: true },
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
lastAssistant: emptyStopAssistant,
currentAttemptAssistant: emptyStopAssistant,
}),
);
mockedBuildEmbeddedRunPayloads.mockReturnValue([]);
const result = await runEmbeddedAgent({
...overflowBaseRunParams,
provider: "openai",
model: "gpt-5.6",
agentHarnessId: "legacy-code-mode",
runId: "run-code-mode-no-finalization-capability",
});
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
expect(runAttemptCall(1).operation).toBe("attempt");
expect(result.payloads?.[0]).toMatchObject({ isError: true });
expect(result.payloads?.[0]?.text).toContain("stopped before producing a final answer");
expectWarnMessageWith("Code Mode completion retries exhausted");
} finally {
resetRunOverflowCompactionHarnessMocks();
}
});
it("uses answer-only finalization after a run-latched read-only completion", async () => {
const emptyStopAssistant = {
role: "assistant",
stopReason: "stop",
provider: "openai",
model: "gpt-5.6",
content: [],
} as unknown as NonNullable<EmbeddedRunAttemptResult["lastAssistant"]>;
mockedClassifyFailoverReason.mockReturnValue(null);
mockedRunEmbeddedAttempt
.mockResolvedValueOnce(
makeAttemptResult({
assistantTexts: [],
codeModeEngaged: true,
toolMetas: [
{
toolName: "exec",
replaySafe: false,
sideEffectFree: false,
codeModeHadTargetlessSideEffects: true,
},
],
replayMetadata: { hadPotentialSideEffects: true, replaySafe: false },
currentAttemptReplayMetadata: { hadPotentialSideEffects: true, replaySafe: false },
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
lastAssistant: emptyStopAssistant,
currentAttemptAssistant: emptyStopAssistant,
}),
)
.mockResolvedValueOnce(
makeAttemptResult({
assistantTexts: [],
codeModeEngaged: true,
toolMetas: [{ toolName: "read", replaySafe: true, sideEffectFree: true }],
replayMetadata: { hadPotentialSideEffects: false, replaySafe: true },
currentAttemptReplayMetadata: { hadPotentialSideEffects: false, replaySafe: true },
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
lastAssistant: emptyStopAssistant,
currentAttemptAssistant: emptyStopAssistant,
}),
)
.mockResolvedValueOnce(
makeAttemptResult({
assistantTexts: ["Finished safely."],
lastAssistant: {
role: "assistant",
stopReason: "stop",
provider: "openai",
model: "gpt-5.6",
content: [{ type: "text", text: "Finished safely." }],
} as unknown as EmbeddedRunAttemptResult["lastAssistant"],
}),
);
mockedBuildEmbeddedRunPayloads
.mockReturnValueOnce([])
.mockReturnValueOnce([])
.mockReturnValueOnce([{ text: "Finished safely." }]);
const result = await runEmbeddedAgent({
...overflowBaseRunParams,
provider: "openai",
model: "gpt-5.6",
runId: "run-targetless-answer-only-finalization",
});
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(3);
expect(result.payloads?.[0]?.text).toBe("Finished safely.");
expect(runAttemptCall(1).forceReadOnlyTools).toBe(true);
expect(runAttemptCall(1).prompt).toBe(RESTART_SAFE_CODE_MODE_CONTINUATION_INSTRUCTION);
expect(runAttemptCall(2).operation).toBe("settled-tool-finalization");
expect(runAttemptCall(2).disableTools).toBe(true);
});
it("keeps only read-only tools across uncertain Code Mode mutation retries", async () => {
const emptyStopAssistant = {
role: "assistant",
@ -1436,14 +1664,14 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(3);
expect(result.payloads?.[0]?.text).toBe("Verified the completed write.");
const secondCall = runAttemptCall(1);
expect(secondCall.prompt).toBe(VERIFY_CODE_MODE_MUTATION_CONTINUATION_INSTRUCTION);
expect(secondCall.prompt).toBe(RESTART_SAFE_CODE_MODE_CONTINUATION_INSTRUCTION);
expect(secondCall.suppressNextUserMessagePersistence).toBe(true);
expect(secondCall.disableTools).not.toBe(true);
expect(secondCall.forceRestartSafeTools).toBeFalsy();
expect(secondCall.forceReadOnlyTools).toBe(true);
expect(secondCall.operation).toBe("attempt");
const thirdCall = runAttemptCall(2);
expect(thirdCall.prompt).toBe(VERIFY_CODE_MODE_MUTATION_CONTINUATION_INSTRUCTION);
expect(thirdCall.prompt).toBe(RESTART_SAFE_CODE_MODE_CONTINUATION_INSTRUCTION);
expect(thirdCall.disableTools).not.toBe(true);
expect(thirdCall.forceRestartSafeTools).toBeFalsy();
expect(thirdCall.forceReadOnlyTools).toBe(true);
@ -1545,6 +1773,84 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
expect(runAttemptCall(2).forceReadOnlyTools).toBe(false);
});
it("keeps unknown targetless evidence read-only after targeted mutation verification", async () => {
const emptyStopAssistant = {
role: "assistant",
stopReason: "stop",
provider: "huggingface",
model: "Qwen/Qwen3.5-9B",
content: [],
} as unknown as NonNullable<EmbeddedRunAttemptResult["lastAssistant"]>;
mockedClassifyFailoverReason.mockReturnValue(null);
mockedRunEmbeddedAttempt
.mockResolvedValueOnce(
makeAttemptResult({
assistantTexts: [],
codeModeEngaged: true,
toolMetas: [
{
toolName: "exec",
replaySafe: false,
sideEffectFree: false,
codeModeUnverifiedMutationFileTargets: [{ path: "result.txt" }],
},
],
replayMetadata: { hadPotentialSideEffects: true, replaySafe: false },
currentAttemptReplayMetadata: { hadPotentialSideEffects: true, replaySafe: false },
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
lastAssistant: emptyStopAssistant,
currentAttemptAssistant: emptyStopAssistant,
}),
)
.mockResolvedValueOnce(
makeAttemptResult({
assistantTexts: [],
codeModeEngaged: true,
toolMetas: [
{
toolName: "read",
replaySafe: true,
sideEffectFree: true,
fileTarget: { path: "result.txt" },
fileTargetVerified: true,
},
],
replayMetadata: { hadPotentialSideEffects: false, replaySafe: true },
currentAttemptReplayMetadata: { hadPotentialSideEffects: false, replaySafe: true },
itemLifecycle: { startedCount: 1, completedCount: 1, activeCount: 0 },
lastAssistant: emptyStopAssistant,
currentAttemptAssistant: emptyStopAssistant,
}),
)
.mockResolvedValueOnce(
makeAttemptResult({
assistantTexts: ["Verified and finished."],
lastAssistant: {
role: "assistant",
stopReason: "stop",
provider: "huggingface",
model: "Qwen/Qwen3.5-9B",
content: [{ type: "text", text: "Verified and finished." }],
} as unknown as EmbeddedRunAttemptResult["lastAssistant"],
}),
);
mockedBuildEmbeddedRunPayloads
.mockReturnValueOnce([])
.mockReturnValueOnce([])
.mockReturnValueOnce([{ text: "Verified and finished." }]);
const result = await runEmbeddedAgent({
...overflowBaseRunParams,
provider: "huggingface",
model: "Qwen/Qwen3.5-9B",
runId: "run-release-unknown-targetless-latch",
});
expect(result.payloads?.[0]?.text).toBe("Verified and finished.");
expect(runAttemptCall(1).forceReadOnlyTools).toBe(true);
expect(runAttemptCall(2).forceReadOnlyTools).toBe(true);
});
it("keeps mixed targeted and targetless mutations read-only after file verification", async () => {
const emptyStopAssistant = {
role: "assistant",
@ -1758,7 +2064,7 @@ describe("runEmbeddedAgent incomplete-turn safety", () => {
expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(2);
expect(result.payloads?.[0]?.text).toBe("CM-EXAMPLE");
const verificationCall = runAttemptCall(1);
expect(verificationCall.prompt).toBe(VERIFY_CODE_MODE_MUTATION_CONTINUATION_INSTRUCTION);
expect(verificationCall.prompt).toBe(RESTART_SAFE_CODE_MODE_CONTINUATION_INSTRUCTION);
expect(verificationCall.forceReadOnlyTools).toBe(true);
expectWarnMessageWith("settled Code Mode work stopped before final verification");
});

4
src/agents/embedded-agent-runner/run/incomplete-turn.ts
View file

@ -246,14 +246,14 @@ export function resolveCodeModeTargetlessSideEffectEvidence(attempt: {
return coveredPotentialSideEffect ? false : null;
}
/** Keeps unknown or observed targetless effects latched while file verification is pending. */
/** Keeps confirmed or unresolved targetless effects read-only for the rest of the run. */
export function shouldLatchCodeModeReadOnlyForRun(params: {
mutationVerificationRequired: boolean;
targetlessSideEffectEvidence: boolean | null;
}): boolean {
return (
params.targetlessSideEffectEvidence === true ||
(params.mutationVerificationRequired && params.targetlessSideEffectEvidence !== false)
(params.mutationVerificationRequired && params.targetlessSideEffectEvidence === null)
);
}

16
src/agents/embedded-agent-runner/run/settled-turn-finalization.ts
View file

@ -115,6 +115,13 @@ export async function prepareTerminalWithSettledTurnFinalization(input: {
attemptContinuationToolPolicy && input.finalization.retryState.forceReadOnlyToolsForRun
? "read-only"
: attemptContinuationToolPolicy;
const settledTurnFinalizationAvailable =
typeof input.finalization.harness.finalizeSettledTurn === "function";
const codeModeCompletionContinuationExhausted =
!codeModeMutationVerificationRequired &&
input.finalization.retryState.codeModeCompletionContinuationAttempts >= 1;
const useIsolatedFinalization =
codeModeCompletionContinuationExhausted && settledTurnFinalizationAvailable;
const prompt = resolveSettledTurnFinalizationRequest({
runParams: input.terminalBase.runParams,
attempt,
@ -126,9 +133,9 @@ export async function prepareTerminalWithSettledTurnFinalization(input: {
prepared.recoveredFinalAssistantPayloadsAfterPromptTimeout,
hasTerminalToolPresentation: input.finalization.hasTerminalToolPresentation,
terminalState: initial.terminalState,
settledTurnFinalizationAvailable:
typeof input.finalization.harness.finalizeSettledTurn === "function",
toolCapableContinuationAvailable: codeModeContinuationToolPolicy !== null,
settledTurnFinalizationAvailable,
toolCapableContinuationAvailable:
codeModeContinuationToolPolicy !== null && !useIsolatedFinalization,
requireCodeModeMutationVerification: codeModeMutationVerificationRequired,
});
if (!prompt) {
@ -142,7 +149,7 @@ export async function prepareTerminalWithSettledTurnFinalization(input: {
toolCapableContinuation: null,
};
}
if (codeModeContinuationToolPolicy) {
if (codeModeContinuationToolPolicy && !useIsolatedFinalization) {
// Keep unfinished Code Mode work on the ordinary continuation path.
// Mutating turns expose only host-enforced read-only tools.
const readOnlyToolsScope: "run" | "verification" | null = input.finalization.retryState
@ -161,6 +168,7 @@ export async function prepareTerminalWithSettledTurnFinalization(input: {
finalizationAttempted: false,
finalizationSucceeded: false,
toolCapableContinuation: {
kind: codeModeMutationVerificationRequired ? "verification" : "completion",
instruction: resolveCodeModeContinuationInstruction({
mutationVerificationRequired: codeModeMutationVerificationRequired,
targetlessSideEffectEvidence: codeModeTargetlessSideEffectEvidence,

40
src/agents/embedded-agent-runner/run/terminal-resolution.ts
View file

@ -50,6 +50,7 @@ import type { EmbeddedRunAttemptResult } from "./types.js";
const MAX_MISSING_ASSISTANT_RETRIES = 1;
const MAX_CODE_MODE_ERROR_CONTINUATIONS = 1;
const MAX_CODE_MODE_COMPLETION_CONTINUATIONS = 1;
const MAX_CODE_MODE_VERIFICATION_CONTINUATIONS = 2;
const COMPACTION_CONTINUATION_RETRY_INSTRUCTION =
"The previous attempt compacted the conversation context before producing a final user-visible answer. Continue from the compacted transcript and produce the final answer now. Do not restart from scratch, do not repeat completed work, and do not rerun tools unless the transcript clearly lacks required evidence.";
@ -184,6 +185,7 @@ export async function resolveEmbeddedRunTerminal(input: {
agentHarnessId: string;
settledTurnFinalizationAttempted: boolean;
toolCapableContinuation?: {
kind: "verification" | "completion";
instruction: string;
readOnlyToolsScope: "verification" | "run" | null;
} | null;
@ -316,10 +318,19 @@ export async function resolveEmbeddedRunTerminal(input: {
return { action: "retry" };
}
if (!nextReasoningOnlyRetryInstruction && input.toolCapableContinuation) {
if (
retryState.codeModeVerificationContinuationAttempts < MAX_CODE_MODE_VERIFICATION_CONTINUATIONS
) {
retryState.codeModeVerificationContinuationAttempts += 1;
const verificationContinuation = input.toolCapableContinuation.kind === "verification";
const continuationAttempts = verificationContinuation
? retryState.codeModeVerificationContinuationAttempts
: retryState.codeModeCompletionContinuationAttempts;
const maxContinuationAttempts = verificationContinuation
? MAX_CODE_MODE_VERIFICATION_CONTINUATIONS
: MAX_CODE_MODE_COMPLETION_CONTINUATIONS;
if (continuationAttempts < maxContinuationAttempts) {
if (verificationContinuation) {
retryState.codeModeVerificationContinuationAttempts += 1;
} else {
retryState.codeModeCompletionContinuationAttempts += 1;
}
if (input.toolCapableContinuation.readOnlyToolsScope === "run") {
retryState.forceReadOnlyToolsForRun = true;
} else if (input.toolCapableContinuation.readOnlyToolsScope === "verification") {
@ -335,11 +346,30 @@ export async function resolveEmbeddedRunTerminal(input: {
);
log.warn(
`settled Code Mode work stopped before final verification: runId=${runParams.runId} sessionId=${runParams.sessionId} ` +
`provider=${input.activeErrorContext.provider}/${input.activeErrorContext.model} — retrying ${retryState.codeModeVerificationContinuationAttempts}/${MAX_CODE_MODE_VERIFICATION_CONTINUATIONS} ` +
`provider=${input.activeErrorContext.provider}/${input.activeErrorContext.model} — retrying ${
verificationContinuation
? retryState.codeModeVerificationContinuationAttempts
: retryState.codeModeCompletionContinuationAttempts
}/${maxContinuationAttempts} ` +
`with ${forceReadOnlyTools ? "read-only" : "normal"} tools`,
);
return { action: "retry" };
}
if (!verificationContinuation) {
const incompletePayloadText =
"⚠️ Agent completed Code Mode work but stopped before producing a final answer. Please inspect the changes and try again.";
log.warn(
`Code Mode completion retries exhausted: runId=${runParams.runId} sessionId=${runParams.sessionId} ` +
`provider=${input.activeErrorContext.provider}/${input.activeErrorContext.model} ` +
`attempts=${retryState.codeModeCompletionContinuationAttempts}/${MAX_CODE_MODE_COMPLETION_CONTINUATIONS} — surfacing incomplete-turn error`,
);
return surfaceIncompleteTurn({
...input,
text: incompletePayloadText,
payloadCount,
availableTerminalToolPresentation,
});
}
const incompletePayloadText =
"⚠️ Agent stopped before completing Code Mode verification. Please inspect the changes and try again.";
log.warn(

2
src/agents/embedded-agent-runner/run/terminal-retry-state.ts
View file

@ -10,6 +10,7 @@ export type EmbeddedRunTerminalRetryState = {
reasoningOnlyAttempts: number;
emptyResponseAttempts: number;
codeModeErrorContinuationAttempts: number;
codeModeCompletionContinuationAttempts: number;
codeModeVerificationContinuationAttempts: number;
missingAssistantAttempts: number;
compactionContinuationAttempts: number;
@ -26,6 +27,7 @@ export function createEmbeddedRunTerminalRetryState(): EmbeddedRunTerminalRetryS
reasoningOnlyAttempts: 0,
emptyResponseAttempts: 0,
codeModeErrorContinuationAttempts: 0,
codeModeCompletionContinuationAttempts: 0,
codeModeVerificationContinuationAttempts: 0,
missingAssistantAttempts: 0,
compactionContinuationAttempts: 0,