From 9474e5507699484f1caaaeb0895d986b02b03467 Mon Sep 17 00:00:00 2001 From: Vito Cappello Date: Sat, 15 Aug 2026 23:49:08 -0400 Subject: [PATCH] fix: consume prepared images once in CLI-backed turns (#120721) * fix(agents): preserve images across CLI dispatch * test(agents): format CLI dispatch coverage * fix(reply): consume prepared images once * test: cover CLI inbound image reply path * fix(auto-reply): preserve prepared image state * fix(auto-reply): preserve queued image fallback * fix(agents): keep CLI image marker internal * fix(agents): keep image admission marker private * fix(agents): preserve internal image admission state * fix(auto-reply): keep image admission marker internal * test: keep image preparation marker internal * fix(cli): preserve queued image metadata * test(cli): cover filtered media image indexes * test: restore media fixture cleanup * test: align filtered media fixture with resolver mock * style: format CLI image forwarding changes * fix(auto-reply): preserve queued media slot type * fix(auto-reply): avoid spread in media slot mapping --------- Co-authored-by: VACInc <3279061+VACInc@users.noreply.github.com> Co-authored-by: Patrick Erichsen --- ...claude-cli-inbound-image-subscription.yaml | 114 +++++++++++++++ src/agents/cli-runner.helpers.test.ts | 25 ++++ src/agents/cli-runner.spawn.test.ts | 43 +++--- src/agents/cli-runner/execute.ts | 12 +- src/agents/cli-runner/helpers.ts | 4 + .../cli-backend-dispatch.test.ts | 8 +- .../cli-backend-dispatch.ts | 2 + .../run/prompt-image-metadata.ts | 2 +- .../reply/agent-runner-cli-candidate.ts | 1 + .../reply/agent-runner-cli-dispatch.ts | 7 +- ...gent-runner-execution-cli-sessions.test.ts | 136 ++++++++++++++++++ .../reply/agent-runner-execution.ts | 39 +++-- src/auto-reply/reply/current-turn-images.ts | 17 ++- src/auto-reply/reply/get-reply-run-execute.ts | 16 +++ .../reply/get-reply-run.media-only.test.ts | 71 +++++++++ src/auto-reply/reply/queue.collect.test.ts | 86 ++++++++++- src/auto-reply/reply/queue/drain.ts | 47 +++++- 17 files changed, 582 insertions(+), 48 deletions(-) create mode 100644 qa/scenarios/models/claude-cli-inbound-image-subscription.yaml diff --git a/qa/scenarios/models/claude-cli-inbound-image-subscription.yaml b/qa/scenarios/models/claude-cli-inbound-image-subscription.yaml new file mode 100644 index 00000000000..27fc4fe424f --- /dev/null +++ b/qa/scenarios/models/claude-cli-inbound-image-subscription.yaml @@ -0,0 +1,114 @@ +title: Claude CLI inbound image subscription + +scenario: + id: claude-cli-inbound-image-subscription + surface: model-provider + coverage: + primary: + - anthropic.claude-cli-compatibility-provider-capabilities + - channels.inbound-routing-qa-channel + objective: Verify a qa-channel inbound image survives reply admission and reaches a Claude CLI subscription turn exactly once. + successCriteria: + - The selected live provider is claude-cli with subscription auth. + - A qa-channel inbound image receives a visible description through the normal reply path. + - The description mentions the expected red and blue regions. + - Gateway logs prove the turn entered the Claude CLI backend without a structured-image hydration failure. + docsRefs: + - docs/gateway/cli-backends.md + - docs/channels/qa-channel.md + - docs/help/testing.md + codeRefs: + - extensions/qa-channel/src/inbound.ts + - src/auto-reply/reply/get-reply-run-execute.ts + - src/auto-reply/reply/agent-runner-execution.ts + - src/agents/cli-runner/helpers.ts + execution: + kind: flow + providerMode: live-frontier + channel: qa-channel + summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --cli-auth-mode subscription --model claude-cli/claude-sonnet-4-6 --alt-model claude-cli/claude-sonnet-4-6 --scenario claude-cli-inbound-image-subscription`. + config: + authMode: subscription + requiredProvider: claude-cli + conversationId: qa-claude-cli-inbound-image-subscription + prompt: "Inbound Claude CLI image check: describe the top and bottom colors in the attached image in one short sentence." + requiredColorGroups: + - [red, scarlet, crimson] + - [blue, azure, teal, cyan, aqua] + +flow: + steps: + - name: describes an inbound image through the Claude CLI reply path + actions: + - set: selected + value: + expr: splitModelRef(env.primaryModel) + - set: preserveEnv + value: + expr: "String(env.gateway.runtimeEnv.OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV ?? '')" + - assert: + expr: "env.providerMode === 'live-frontier' && selected?.provider === config.requiredProvider" + message: + expr: "`expected live provider ${config.requiredProvider}, got mode=${env.providerMode} model=${env.primaryModel}`" + - assert: + expr: "env.gateway.runtimeEnv.OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE === config.authMode" + message: + expr: "`expected Claude CLI auth mode ${config.authMode}, got ${env.gateway.runtimeEnv.OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE ?? 'unset'}`" + - assert: + expr: "!preserveEnv.includes('ANTHROPIC_API_KEY')" + message: + expr: "`expected subscription auth without ANTHROPIC_API_KEY preservation, got ${preserveEnv}`" + - call: waitForGatewayHealthy + args: + - ref: env + - 60000 + - call: waitForQaChannelReady + args: + - ref: env + - 60000 + - call: reset + - set: gatewayLogCursor + value: + expr: markGatewayLogCursor() + - set: outboundStartIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + accountId: default + conversation: + id: + expr: config.conversationId + kind: direct + senderId: qa-operator + senderName: QA Operator + text: + expr: config.prompt + attachments: + - kind: image + mimeType: image/png + fileName: claude-cli-inbound-red-top-blue-bottom.png + altText: red on top blue on bottom + contentBase64: + expr: imageUnderstandingValidPngBase64 + - call: waitForOutboundMessage + saveAs: outbound + args: + - ref: state + - lambda: + params: [candidate] + expr: "candidate.conversation.id === config.conversationId && config.requiredColorGroups.every((group) => group.some((color) => normalizeLowercaseStringOrEmpty(candidate.text).includes(color)))" + - expr: resolveQaLiveTurnTimeoutMs(env, 90000, env.primaryModel) + - sinceIndex: + ref: outboundStartIndex + - set: gatewayLog + value: + expr: "String(readGatewayLogs() ?? '').slice(gatewayLogCursor)" + - assert: + expr: "gatewayLog.includes('cli exec: provider=claude-cli')" + message: + expr: "`expected Claude CLI execution trace; log tail=${gatewayLog.slice(-2000)}`" + - assert: + expr: "!gatewayLog.includes('failed to hydrate')" + message: + expr: "`unexpected structured image hydration failure; log tail=${gatewayLog.slice(-2000)}`" + detailsExpr: "`provider=${selected?.provider} model=${selected?.model} auth=${config.authMode} reply=${JSON.stringify(outbound.text)} cliTrace=true`" diff --git a/src/agents/cli-runner.helpers.test.ts b/src/agents/cli-runner.helpers.test.ts index fb6eb946e69..2644f3ed36a 100644 --- a/src/agents/cli-runner.helpers.test.ts +++ b/src/agents/cli-runner.helpers.test.ts @@ -194,6 +194,27 @@ describe("prepareCliPromptImagePayload prompt references", () => { } }); + it("does not hydrate media suppressed during current-turn admission", async () => { + await expect( + prepareCliPromptImagePayload({ + backend: { command: "claude" }, + prompt: "describe the attachment", + imagePrompt: "describe the attachment", + workspaceDir: "/workspace", + images: [], + imageOrder: [], + mediaImageLayout: { slots: [], suppressedFactIndexes: [0] }, + media: [ + { + path: "/openclaw-test-missing/current.png", + contentType: "image/png", + hydrationSuppressed: true, + }, + ], + }), + ).resolves.toEqual({ prompt: "describe the attachment" }); + }); + it("delivers readable structured images when an unresolved attachment is hydration-suppressed", async () => { const workspaceDir = await fs.mkdtemp( path.join(resolvePreferredOpenClawTmpDir(), "openclaw-cli-mixed-media-"), @@ -676,6 +697,10 @@ describe("writeCliImages", () => { }, ], imageOrder: ["offloaded", "inline"], + mediaImageLayout: { + slots: [{ kind: "offloaded", factIndex: 0 }, { kind: "inline" }], + suppressedFactIndexes: [], + }, media: [{ url: `media://inbound/${mediaId}`, contentType: "image/png" }], }); diff --git a/src/agents/cli-runner.spawn.test.ts b/src/agents/cli-runner.spawn.test.ts index 1af307eb538..70e08aaa9ab 100644 --- a/src/agents/cli-runner.spawn.test.ts +++ b/src/agents/cli-runner.spawn.test.ts @@ -678,26 +678,14 @@ describe("runCliAgent spawn path", () => { expect(invokeNode).not.toHaveBeenCalled(); }); - it("allows non-hydratable image facts on a text-only node turn", async () => { - const invokeNode = vi.fn(async (params: Parameters[0]) => { - params.onProgress( - [ - JSON.stringify({ type: "system", subtype: "init", session_id: "node-text-only" }), - JSON.stringify({ type: "result", session_id: "node-text-only", result: "ok" }), - "", - ].join("\n"), - ); - return { - ok: true, - payloadJSON: JSON.stringify({ exitCode: 0, stderrTail: "", truncated: false }), - }; - }); + it("rejects prepared offloaded images before invoking a node-placed Claude session", async () => { + const invokeNode = vi.fn(); setCliRunnerExecuteTestDeps({ invokeNodeClaudeCliRun: invokeNode }); const context = buildPreparedCliRunContext({ provider: "claude-cli", model: "claude-opus-4-8", - runId: "run-node-text-only-media-facts", - prompt: "already described", + runId: "run-node-offloaded-media-facts", + prompt: "describe the attachment", sessionEntry: { sessionId: "openclaw-session", updatedAt: 1, @@ -705,13 +693,24 @@ describe("runCliAgent spawn path", () => { execNode: "node-a", }, }); - context.params.media = [ - { kind: "image" }, - { kind: "image", url: "https://example.test/described.png" }, - ]; + const preparedParams = context.params as typeof context.params & { + mediaImageLayout?: { + slots: Array<{ kind: "inline" | "offloaded"; factIndex?: number }>; + suppressedFactIndexes: number[]; + }; + }; + preparedParams.mediaImageLayout = { + slots: [{ kind: "offloaded", factIndex: 0 }], + suppressedFactIndexes: [], + }; + context.params.images = []; + context.params.imageOrder = ["offloaded"]; + context.params.media = [{ kind: "image", path: "/tmp/offloaded.png" }]; - await expect(executePreparedCliRun(context)).resolves.toMatchObject({ text: "ok" }); - expect(invokeNode).toHaveBeenCalledOnce(); + await expect(executePreparedCliRun(context)).rejects.toThrow( + "paired-node Claude CLI sessions do not support attachments or images", + ); + expect(invokeNode).not.toHaveBeenCalled(); }); it("does not inject hardcoded 'Tools are disabled' text into CLI arguments", async () => { diff --git a/src/agents/cli-runner/execute.ts b/src/agents/cli-runner/execute.ts index c7828311aab..69798479718 100644 --- a/src/agents/cli-runner/execute.ts +++ b/src/agents/cli-runner/execute.ts @@ -20,6 +20,7 @@ import { detectImageReferences, hasHydratableMediaImages, } from "../embedded-agent-runner/run/images.js"; +import type { MediaImageLayout } from "../embedded-agent-runner/run/prompt-image-metadata.js"; import { applyPluginTextReplacements } from "../plugin-text-transforms.js"; import { prepareCliBundleMcpCaptureAttempt } from "./bundle-mcp.js"; import { buildClaudeOwnerKey, closeClaudeSession } from "./claude-live-registry.js"; @@ -117,13 +118,17 @@ type ExecutePreparedCliRunOptions = { onPhase?: (phase: "send" | "resolve" | "cleanup") => void; }; +type PreparedCliRunInternalParams = PreparedCliRunContext["params"] & { + mediaImageLayout?: MediaImageLayout; +}; + /** Executes a prepared CLI run context and returns normalized CLI output. */ export async function executePreparedCliRun( context: PreparedCliRunContext, cliSessionIdToUse?: string, options?: ExecutePreparedCliRunOptions, ): Promise { - const params = context.params; + const params = context.params as PreparedCliRunInternalParams; if (params.abortSignal?.aborted) { throw createCliAbortError(); } @@ -166,7 +171,9 @@ export async function executePreparedCliRun( if ( nodePlacement && ((params.images?.length ?? 0) > 0 || - hasHydratableMediaImages(params.media) || + (params.mediaImageLayout + ? params.mediaImageLayout.slots.length > 0 + : hasHydratableMediaImages(params.media)) || (params.imagePrompt ? detectImageReferences(params.imagePrompt).length > 0 : false)) ) { throw new Error("paired-node Claude CLI sessions do not support attachments or images"); @@ -181,6 +188,7 @@ export async function executePreparedCliRun( localRoots: getAgentScopedMediaLocalRoots(params.config ?? {}, params.agentId), images: params.images, imageOrder: params.imageOrder, + mediaImageLayout: params.mediaImageLayout, media: params.media, }); prompt = imagePayload.prompt; diff --git a/src/agents/cli-runner/helpers.ts b/src/agents/cli-runner/helpers.ts index 0dcedb85f08..084a8bb2bf7 100644 --- a/src/agents/cli-runner/helpers.ts +++ b/src/agents/cli-runner/helpers.ts @@ -34,6 +34,7 @@ import { detectAndLoadPromptImages, detectImageReferences, } from "../embedded-agent-runner/run/images.js"; +import type { MediaImageLayout } from "../embedded-agent-runner/run/prompt-image-metadata.js"; import { resolveDefaultModelForAgent } from "../model-selection.js"; import type { AgentTool } from "../runtime/index.js"; import { detectRuntimeShell } from "../shell-utils.js"; @@ -391,6 +392,7 @@ export async function prepareCliPromptImagePayload(params: { localRoots?: readonly string[]; images?: ImageContent[]; imageOrder?: PromptImageOrderEntry[]; + mediaImageLayout?: MediaImageLayout; media?: MediaFact[]; }): Promise<{ prompt: string; @@ -402,6 +404,7 @@ export async function prepareCliPromptImagePayload(params: { const needsHydration = params.imagePrompt !== undefined || Boolean(params.media?.length) || + Boolean(params.mediaImageLayout) || (!params.images?.length && detectImageReferences(imagePrompt).length > 0); const imageResult = needsHydration ? await detectAndLoadPromptImages({ @@ -411,6 +414,7 @@ export async function prepareCliPromptImagePayload(params: { model: { input: ["text", "image"] }, existingImages: params.images, imageOrder: params.imageOrder, + mediaImageLayout: params.mediaImageLayout, maxBytes: MAX_IMAGE_BYTES, localRoots: params.localRoots, }) diff --git a/src/agents/embedded-agent-runner/cli-backend-dispatch.test.ts b/src/agents/embedded-agent-runner/cli-backend-dispatch.test.ts index b93279a915f..a73092a95bd 100644 --- a/src/agents/embedded-agent-runner/cli-backend-dispatch.test.ts +++ b/src/agents/embedded-agent-runner/cli-backend-dispatch.test.ts @@ -411,13 +411,17 @@ describe("runEmbeddedAgentViaCliBackendIfEligible execution", () => { expect(onExecutionStarted).toHaveBeenCalledWith({ lifecycleGeneration: "gen-1" }); }); - it("retains prompt media facts through the embedded-to-CLI bridge", async () => { + it("retains ordered prompt images and media facts through the embedded-to-CLI bridge", async () => { + const images = [{ type: "image" as const, data: "aGVsbG8=", mimeType: "image/png" }]; + const imageOrder = ["inline" as const]; const media = [{ path: "/tmp/recall.png", contentType: "image/png" }]; - await runEmbeddedAgentViaCliBackendIfEligible(baseRunParams({ media })); + await runEmbeddedAgentViaCliBackendIfEligible(baseRunParams({ images, imageOrder, media })); expect(runCliAgent.mock.calls[0]?.[0]).toMatchObject({ prompt: "recall prompt", + images, + imageOrder, media, }); }); diff --git a/src/agents/embedded-agent-runner/cli-backend-dispatch.ts b/src/agents/embedded-agent-runner/cli-backend-dispatch.ts index 663fb920166..4e1b60357dc 100644 --- a/src/agents/embedded-agent-runner/cli-backend-dispatch.ts +++ b/src/agents/embedded-agent-runner/cli-backend-dispatch.ts @@ -224,6 +224,8 @@ async function runEmbeddedAgentViaCliBackend( config: params.config, prompt: params.prompt, imagePrompt: params.prompt, + images: params.images, + imageOrder: params.imageOrder, media: params.media, provider: dispatch.provider, model: params.model, diff --git a/src/agents/embedded-agent-runner/run/prompt-image-metadata.ts b/src/agents/embedded-agent-runner/run/prompt-image-metadata.ts index b5a7e4de7b7..8d9eb070469 100644 --- a/src/agents/embedded-agent-runner/run/prompt-image-metadata.ts +++ b/src/agents/embedded-agent-runner/run/prompt-image-metadata.ts @@ -4,7 +4,7 @@ export type ImageFactIndex = number | null; export type MediaImageLayout = { slots: Array<{ kind: "inline" | "offloaded"; factIndex?: number }>; - suppressedFactIndexes: number[]; + suppressedFactIndexes?: number[]; }; export function readPersistedImageBlockFactIndexes( diff --git a/src/auto-reply/reply/agent-runner-cli-candidate.ts b/src/auto-reply/reply/agent-runner-cli-candidate.ts index 88bd37035ec..5579c9269ca 100644 --- a/src/auto-reply/reply/agent-runner-cli-candidate.ts +++ b/src/auto-reply/reply/agent-runner-cli-candidate.ts @@ -418,6 +418,7 @@ export async function runCliFallbackCandidate(params: { ], images: params.currentTurnImages.images, imageOrder: params.currentTurnImages.imageOrder, + mediaImageLayout: params.currentTurnImages.mediaImageLayout, skillsSnapshot: turn.followupRun.run.skillsSnapshot, messageChannel: turn.followupRun.originatingChannel ?? undefined, messageProvider: hookMessageProvider, diff --git a/src/auto-reply/reply/agent-runner-cli-dispatch.ts b/src/auto-reply/reply/agent-runner-cli-dispatch.ts index 48e1b4f6d69..a61e0a94a23 100644 --- a/src/auto-reply/reply/agent-runner-cli-dispatch.ts +++ b/src/auto-reply/reply/agent-runner-cli-dispatch.ts @@ -4,6 +4,7 @@ import { normalizeOptionalString } from "@openclaw/normalization-core/string-coe import { runCliAgent } from "../../agents/cli-runner.js"; import type { RunCliAgentParams } from "../../agents/cli-runner/types.js"; import { clearCliSession, getCliSessionBinding } from "../../agents/cli-session.js"; +import type { MediaImageLayout } from "../../agents/embedded-agent-runner/run/prompt-image-metadata.js"; import { extractToolResultText } from "../../agents/embedded-agent-tool-results.js"; import type { EmbeddedAgentRunResult } from "../../agents/embedded-agent.js"; import { @@ -34,6 +35,10 @@ import { } from "./agent-event-bridge.js"; import { resolveAgentLifecycleTerminalMetadata } from "./agent-lifecycle-terminal.js"; +type RunCliAgentInternalParams = RunCliAgentParams & { + mediaImageLayout?: MediaImageLayout; +}; + type AgentEventBridge = { unsubscribe: () => void; drain: () => Promise; @@ -422,7 +427,7 @@ type RunCliAgentWithLifecycleParams = { runId: string; lifecycleGeneration?: string; provider: string; - runParams: RunCliAgentParams; + runParams: RunCliAgentInternalParams; startedAt?: number; emitLifecycleTerminal?: boolean; onAgentRunStart?: () => void; diff --git a/src/auto-reply/reply/agent-runner-execution-cli-sessions.test.ts b/src/auto-reply/reply/agent-runner-execution-cli-sessions.test.ts index 37558cfece5..47f99ed2559 100644 --- a/src/auto-reply/reply/agent-runner-execution-cli-sessions.test.ts +++ b/src/auto-reply/reply/agent-runner-execution-cli-sessions.test.ts @@ -1,4 +1,7 @@ import { afterEach, describe, expect, it } from "vitest"; +import { prepareCliPromptImagePayload } from "../../agents/cli-runner/helpers.js"; +import type { RunCliAgentParams } from "../../agents/cli-runner/types.js"; +import { detectAndLoadPromptImages } from "../../agents/embedded-agent-runner/run/images.js"; import { FailoverError } from "../../agents/failover-error.js"; import { installSessionPlacementAdmissionProvider } from "../../agents/session-placement-admission.js"; import type { SessionEntry } from "../../config/sessions.js"; @@ -23,6 +26,139 @@ const state = setupAgentRunnerExecutionTestState(); afterEach(resetGeneratedMediaTaskActivityForTests); describe("executeAgentTurn: CLI session routing", () => { + it("preserves queued image fields from runs created before the prepared marker", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("claude-cli", "claude-opus-5"), + provider: "claude-cli", + model: "claude-opus-5", + attempts: [], + })); + state.runCliAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "described" }], + meta: {}, + }); + const images = [ + { + type: "image" as const, + data: "iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAFElEQVR4nGP4z8Dwn4GBgYGJAQoAHxcCAr7cGDwAAAAASUVORK5CYII=", + mimeType: "image/png", + }, + ]; + const imageOrder = ["inline" as const]; + const followupRun = createFollowupRun(); + followupRun.run.provider = "claude-cli"; + followupRun.run.model = "claude-opus-5"; + followupRun.images = images; + followupRun.imageOrder = imageOrder; + + const executeAgentTurn = await getExecuteAgentTurnForTest(); + const result = await executeAgentTurn( + createMinimalRunAgentTurnParams({ + followupRun, + sessionCtx: { + Provider: "telegram", + MessageSid: "msg", + } as unknown as TemplateContext, + }), + ); + + expect(result.kind).toBe("success"); + expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", { + images, + imageOrder, + }); + }); + + it("keeps prepared current-turn images aligned with CLI media facts", async () => { + state.isCliProviderMock.mockReturnValue(true); + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("claude-cli", "claude-opus-5"), + provider: "claude-cli", + model: "claude-opus-5", + attempts: [], + })); + const images = [ + { + type: "image" as const, + data: "iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAFElEQVR4nGP4z8Dwn4GBgYGJAQoAHxcCAr7cGDwAAAAASUVORK5CYII=", + mimeType: "image/png", + }, + ]; + const imageOrder = ["inline" as const]; + const media = [{ path: "/openclaw-test-missing/current.png", contentType: "image/png" }]; + const mediaImageLayout = { + slots: [{ kind: "inline" as const, factIndex: 0 }], + suppressedFactIndexes: [], + }; + state.runCliAgentMock.mockImplementationOnce(async (params: RunCliAgentParams) => { + const internalParams = params as RunCliAgentParams & { + mediaImageLayout?: typeof mediaImageLayout; + }; + await expect( + prepareCliPromptImagePayload({ + backend: { command: "claude" }, + prompt: params.prompt, + imagePrompt: params.prompt, + workspaceDir: params.workspaceDir, + images: [...images, ...images], + imageOrder: [...imageOrder, ...imageOrder], + media, + }), + ).rejects.toThrow("failed to hydrate 1 structured image attachment"); + + const reconciled = await detectAndLoadPromptImages({ + prompt: params.prompt, + media: params.media, + workspaceDir: params.workspaceDir, + model: { input: ["text", "image"] }, + existingImages: params.images, + imageOrder: params.imageOrder, + mediaImageLayout: internalParams.mediaImageLayout, + }); + expect(reconciled).toMatchObject({ + failedMediaCount: 0, + images, + }); + return { + payloads: [{ text: "described" }], + meta: {}, + }; + }); + const followupRun = createFollowupRun(); + followupRun.run.provider = "claude-cli"; + followupRun.run.model = "claude-opus-5"; + const preparedFollowupRun = followupRun as typeof followupRun & { + currentTurnImagesPrepared?: true; + mediaImageLayout?: typeof mediaImageLayout; + }; + preparedFollowupRun.currentTurnImagesPrepared = true; + preparedFollowupRun.mediaImageLayout = mediaImageLayout; + followupRun.images = images; + followupRun.imageOrder = imageOrder; + followupRun.media = media; + + const executeAgentTurn = await getExecuteAgentTurnForTest(); + const result = await executeAgentTurn( + createMinimalRunAgentTurnParams({ + followupRun, + sessionCtx: { + Provider: "telegram", + MessageSid: "msg", + media, + } as unknown as TemplateContext, + }), + ); + + expect(result.kind).toBe("success"); + expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", { + images, + imageOrder, + mediaImageLayout, + media, + }); + }); + it("forwards the static extra system prompt to CLI backends", async () => { state.isCliProviderMock.mockReturnValue(true); state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ diff --git a/src/auto-reply/reply/agent-runner-execution.ts b/src/auto-reply/reply/agent-runner-execution.ts index 216e22698e8..c5250bbb4dc 100644 --- a/src/auto-reply/reply/agent-runner-execution.ts +++ b/src/auto-reply/reply/agent-runner-execution.ts @@ -64,7 +64,7 @@ import { createAgentTurnTimingTracker } from "./agent-runner-turn-timing.js"; import { resolveQueuedReplyRuntimeConfig } from "./agent-runner-utils.js"; import { prepareChannelRunAdmission } from "./channel-run-admission.js"; import { shouldNotifyUserAboutCompaction } from "./compaction-notice.js"; -import { resolveCurrentTurnImages } from "./current-turn-images.js"; +import { type CurrentTurnImages, resolveCurrentTurnImages } from "./current-turn-images.js"; import type { FollowupRun } from "./queue.js"; import type { ReplyMediaContext } from "./reply-media-paths.js"; import { createReplyMediaContext } from "./reply-media-paths.runtime.js"; @@ -74,6 +74,12 @@ import { } from "./reply-operation-abort.js"; import { isReplyProfilerEnabled } from "./reply-timing-tracker.js"; +type InternalFollowupRun = FollowupRun & { + /** Keep admission state out of the public plugin-facing FollowupRun contract. */ + currentTurnImagesPrepared?: true; + mediaImageLayout?: CurrentTurnImages["mediaImageLayout"]; +}; + function resolveRunStartupPhase( phase: EmbeddedAgentExecutionPhase, ): ChatRunStartupPhase | undefined { @@ -180,7 +186,7 @@ async function executeAgentTurnInternalWithRetryState( }); } let replyMediaContext: ReplyMediaContext; - let currentTurnImages: Awaited>; + let currentTurnImages: CurrentTurnImages; try { replyMediaContext = params.replyMediaContext ?? @@ -201,14 +207,27 @@ async function executeAgentTurnInternalWithRetryState( requesterSenderE164: params.followupRun.run.senderE164, }), ); - currentTurnImages = await agentTurnTiming.measure("current_turn_images", () => - resolveCurrentTurnImages({ - ctx: params.sessionCtx, - cfg: runtimeConfig, - images: params.followupRun.images ?? params.opts?.images, - imageOrder: params.followupRun.imageOrder ?? params.opts?.imageOrder, - }), - ); + const internalFollowupRun = params.followupRun as InternalFollowupRun; + const hasQueuedCurrentTurnImages = + internalFollowupRun.currentTurnImagesPrepared === true || + Object.hasOwn(params.followupRun, "images") || + Object.hasOwn(params.followupRun, "imageOrder"); + // Queue admission owns current-turn materialization, including empty results. + // Re-scanning here can resurrect suppressed media or duplicate loaded images. + currentTurnImages = hasQueuedCurrentTurnImages + ? { + images: params.followupRun.images, + imageOrder: params.followupRun.imageOrder, + mediaImageLayout: internalFollowupRun.mediaImageLayout, + } + : await agentTurnTiming.measure("current_turn_images", () => + resolveCurrentTurnImages({ + ctx: params.sessionCtx, + cfg: runtimeConfig, + images: params.opts?.images, + imageOrder: params.opts?.imageOrder, + }), + ); } catch (error) { clearAgentRunContext(runId, lifecycleGeneration); throw error; diff --git a/src/auto-reply/reply/current-turn-images.ts b/src/auto-reply/reply/current-turn-images.ts index 18768f96f88..0d768a0dc20 100644 --- a/src/auto-reply/reply/current-turn-images.ts +++ b/src/auto-reply/reply/current-turn-images.ts @@ -1,5 +1,6 @@ // Tracks image attachments that belong to the current reply turn. import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; +import type { MediaImageLayout } from "../../agents/embedded-agent-runner/run/prompt-image-metadata.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { logVerbose } from "../../globals.js"; import { formatErrorMessage } from "../../infra/errors.js"; @@ -26,6 +27,15 @@ type OrderedTurnImage = { sequence: number; }; +export type CurrentTurnImages = { + images?: ImageContent[]; + imageOrder?: PromptImageOrderEntry[]; + imageSourceIndexes?: Array; + unresolvedSourceIndexes?: number[]; + /** Admission-owned slot-to-media identity used by later runtime adapters. */ + mediaImageLayout?: MediaImageLayout; +}; + function collectCurrentImageAttachments(ctx: MsgContext): CurrentImageAttachment[] { return normalizeAttachments(ctx).flatMap((attachment) => { const mediaPath = normalizeOptionalString(attachment.path); @@ -130,12 +140,7 @@ export async function resolveCurrentTurnImages(params: { images?: ImageContent[]; imageOrder?: PromptImageOrderEntry[]; extractedFileImages?: ExtractedFileImage[]; -}): Promise<{ - images?: ImageContent[]; - imageOrder?: PromptImageOrderEntry[]; - imageSourceIndexes?: Array; - unresolvedSourceIndexes?: number[]; -}> { +}): Promise { const entries: OrderedTurnImage[] = []; appendOrderedImages({ entries, diff --git a/src/auto-reply/reply/get-reply-run-execute.ts b/src/auto-reply/reply/get-reply-run-execute.ts index d16e380dc09..49a13743510 100644 --- a/src/auto-reply/reply/get-reply-run-execute.ts +++ b/src/auto-reply/reply/get-reply-run-execute.ts @@ -229,6 +229,20 @@ export async function executePreparedReplyRun(state: PreparedReplyRunAdmission) imageOrder: currentTurnImages.imageOrder, imageSourceIndexes: currentTurnImages.imageSourceIndexes, }); + const promptMediaSourceIndexes = currentTurnImages.imageSourceIndexes?.map((sourceIndex) => { + if (sourceIndex === undefined) { + return undefined; + } + const promptIndex = inboundMediaIndexes.indexOf(sourceIndex); + return promptIndex >= 0 ? promptIndex : undefined; + }); + const promptMediaImageLayout = buildPersistedMediaImageLayout({ + ctx: {}, + media: promptMediaForRun, + ctxMediaCount: inboundMediaIndexes.length, + imageOrder: currentTurnImages.imageOrder, + imageSourceIndexes: promptMediaSourceIndexes, + }); const inputProvenance = ctx.InputProvenance ?? sessionCtx.InputProvenance; const userTurnTimestamp = normalizeMessageTimestampMs(ctx.Timestamp); // prompt-prelude substitutes MEDIA_ONLY_USER_TEXT as transcriptBody for @@ -357,8 +371,10 @@ export async function executePreparedReplyRun(state: PreparedReplyRunAdmission) ...(queuedToolsAllow !== undefined ? { toolsAllow: queuedToolsAllow } : {}), ...(opts?.disableTools !== undefined ? { disableTools: opts.disableTools } : {}), enqueuedAt: Date.now(), + currentTurnImagesPrepared: true as const, images: currentTurnImages.images, imageOrder: currentTurnImages.imageOrder, + mediaImageLayout: promptMediaImageLayout, media: promptMediaForRun, // Originating channel for reply routing. originatingChannel: replyRoute.channel, diff --git a/src/auto-reply/reply/get-reply-run.media-only.test.ts b/src/auto-reply/reply/get-reply-run.media-only.test.ts index e2136c26117..4243d1ab14d 100644 --- a/src/auto-reply/reply/get-reply-run.media-only.test.ts +++ b/src/auto-reply/reply/get-reply-run.media-only.test.ts @@ -1655,6 +1655,13 @@ describe("runPreparedReply media-only handling", () => { }, }); expect(call.followupRun.imageOrder).toEqual(["inline"]); + expect( + ( + call.followupRun as typeof call.followupRun & { + currentTurnImagesPrepared?: true; + } + ).currentTurnImagesPrepared, + ).toBe(true); expect(resolveCurrentTurnImagesMock).toHaveBeenCalledWith({ ctx: expect.objectContaining({ media: [{ path: imagePath, workspaceDir: "/tmp" }], @@ -1897,6 +1904,70 @@ describe("runPreparedReply media-only handling", () => { expect(call.followupRun.prompt).toContain("a tiny dot image"); }); + it("indexes the runtime image layout against filtered prompt media", async () => { + const imageData = Buffer.from("runtime image bytes"); + const imagePath = "/tmp/current.png"; + resolveCurrentTurnImagesMock.mockResolvedValueOnce({ + images: [{ type: "image", data: imageData.toString("base64"), mimeType: "image/png" }], + imageOrder: ["inline"], + imageSourceIndexes: [1], + }); + + const result = await runPrepared({ + ctx: { + ...createInboundBody("describe the image"), + media: [ + { + path: "/tmp/voice.ogg", + contentType: "audio/ogg", + transcribed: true, + }, + { path: imagePath, contentType: "image/png", workspaceDir: "/tmp" }, + ], + OriginatingChannel: "webchat", + OriginatingTo: "webchat:local", + ChatType: "direct", + }, + sessionCtx: { + ...createSessionBody("describe the image"), + Provider: "webchat", + OriginatingChannel: "webchat", + OriginatingTo: "webchat:local", + ChatType: "direct", + }, + }); + + expect(result).toEqual({ text: "ok" }); + const call = requireRunReplyAgentCall(); + expect(call.followupRun.media).toHaveLength(1); + expect(call.followupRun.media?.[0]).toMatchObject({ + path: imagePath, + contentType: "image/png", + workspaceDir: "/tmp", + }); + expect(call.followupRun.images).toEqual([ + { + type: "image", + data: imageData.toString("base64"), + mimeType: "image/png", + }, + ]); + expect( + ( + call.followupRun as typeof call.followupRun & { + mediaImageLayout?: { slots: Array<{ kind: string; factIndex?: number }> }; + } + ).mediaImageLayout, + ).toEqual({ slots: [{ kind: "inline", factIndex: 0 }] }); + expect( + ( + call.followupRun.userTurnTranscriptRecorder?.message as unknown as Record + )?.["__openclaw"], + ).toMatchObject({ + mediaImageLayout: { slots: [{ kind: "inline", factIndex: 1 }] }, + }); + }); + it("does not send a standalone reset notice for reply-producing /new turns", async () => { await runPrepared({ ctx: { diff --git a/src/auto-reply/reply/queue.collect.test.ts b/src/auto-reply/reply/queue.collect.test.ts index 4e1dd8c03c9..2dbf81af3b0 100644 --- a/src/auto-reply/reply/queue.collect.test.ts +++ b/src/auto-reply/reply/queue.collect.test.ts @@ -32,6 +32,14 @@ import { import { resolveFollowupDeliveryContextKey } from "./queue/drain.js"; import { clearFollowupQueue, getExistingFollowupQueue } from "./queue/state.js"; +type InternalFollowupRun = FollowupRun & { + currentTurnImagesPrepared?: true; + mediaImageLayout?: { + slots: Array<{ kind: "inline" | "offloaded"; factIndex?: number }>; + suppressedFactIndexes: number[]; + }; +}; + installQueueRuntimeErrorSilencer(); function createQueueSettings(overrides: Partial = {}): QueueSettings { @@ -53,7 +61,7 @@ function enqueueTestRun( } function createDrainRecorder(expectedCalls = 1) { - const calls: FollowupRun[] = []; + const calls: Array = []; const done = createDeferred(); const runFollowup = async (run: FollowupRun) => { calls.push(run); @@ -2019,6 +2027,82 @@ describe("followup queue collect routing", () => { expect(calls[0]?.imageOrder).toEqual(["inline", "inline"]); }); + it("preserves prepared empty image state across collected batches", async () => { + const key = `test-collect-prepared-empty-images-${Date.now()}`; + const { calls, done, runFollowup } = createDrainRecorder(); + const settings = createQueueSettings(); + const missingMedia = { + path: "/openclaw-test-missing/current.png", + contentType: "image/png", + hydrationSuppressed: true, + }; + + for (const prompt of ["one", "two"]) { + const preparedRun: InternalFollowupRun = { + ...createRun({ + prompt, + originatingChannel: "slack", + originatingTo: "channel:A", + }), + currentTurnImagesPrepared: true, + images: [], + imageOrder: [], + media: [missingMedia], + mediaImageLayout: { slots: [], suppressedFactIndexes: [0] }, + }; + enqueueFollowupRun(key, preparedRun, settings); + } + + scheduleFollowupDrain(key, runFollowup); + await done.promise; + + const collected = calls[0] as InternalFollowupRun | undefined; + expect(collected?.currentTurnImagesPrepared).toBe(true); + expect(collected?.images).toEqual([]); + expect(collected?.imageOrder).toEqual([]); + expect(collected?.media).toEqual([missingMedia, missingMedia]); + expect(collected?.mediaImageLayout).toEqual({ + slots: [], + suppressedFactIndexes: [0, 1], + }); + }); + + it("offsets prepared media layout fact indexes across collected batches", async () => { + const key = `test-collect-prepared-image-layout-${Date.now()}`; + const { calls, done, runFollowup } = createDrainRecorder(); + const settings = createQueueSettings(); + + for (const [index, prompt] of ["one", "two"].entries()) { + const preparedRun: InternalFollowupRun = { + ...createRun({ + prompt, + originatingChannel: "slack", + originatingTo: "channel:A", + }), + currentTurnImagesPrepared: true, + images: [], + imageOrder: ["offloaded"], + media: [{ path: `/tmp/offloaded-${index}.png`, contentType: "image/png" }], + mediaImageLayout: { + slots: [{ kind: "offloaded", factIndex: 0 }], + suppressedFactIndexes: [], + }, + }; + enqueueFollowupRun(key, preparedRun, settings); + } + + scheduleFollowupDrain(key, runFollowup); + await done.promise; + + expect((calls[0] as InternalFollowupRun | undefined)?.mediaImageLayout).toEqual({ + slots: [ + { kind: "offloaded", factIndex: 0 }, + { kind: "offloaded", factIndex: 1 }, + ], + suppressedFactIndexes: [], + }); + }); + it("splits collect batches when sender authorization changes", async () => { const { key, calls, done, runFollowup, settings } = createQueueCase( `test-collect-auth-split-${Date.now()}`, diff --git a/src/auto-reply/reply/queue/drain.ts b/src/auto-reply/reply/queue/drain.ts index f89945482e8..11632b613c3 100644 --- a/src/auto-reply/reply/queue/drain.ts +++ b/src/auto-reply/reply/queue/drain.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import { expectDefined } from "@openclaw/normalization-core"; import { stableStringify } from "@openclaw/normalization-core"; import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; +import type { MediaImageLayout } from "../../../agents/embedded-agent-runner/run/prompt-image-metadata.js"; import { runAgentHarnessBeforeMessageWriteHook } from "../../../agents/harness/hook-helpers.js"; import { readToolAllowlistIntersection } from "../../../agents/tool-policy.js"; import { normalizeChatType } from "../../../channels/chat-type.js"; @@ -45,6 +46,17 @@ import { type FollowupRun, } from "./types.js"; +type InternalFollowupRun = FollowupRun & { + /** Keep admission state out of the public plugin-facing FollowupRun contract. */ + currentTurnImagesPrepared?: true; + /** Admission-owned layout; fact indexes are relative to this run's media array. */ + mediaImageLayout?: MediaImageLayout; +}; + +function hasPreparedCurrentTurnImages(run: FollowupRun): boolean { + return (run as InternalFollowupRun).currentTurnImagesPrepared === true; +} + // Persists the most recent runFollowup callback per queue key so that // enqueueFollowupRun can restart a drain that finished and deleted the queue. const FOLLOWUP_DRAIN_CALLBACKS_KEY = Symbol.for("openclaw.followupDrainCallbacks"); @@ -225,6 +237,7 @@ export function resolveFollowupDeliveryContextKey(run: FollowupRun): string { accountId: run.originatingAccountId, threadId: run.originatingThreadId, }), + hasPreparedCurrentTurnImages(run), run.originatingChatId ?? "", resolveFollowupReplyAnchor(run) ?? "", run.originatingReplyToMode ?? "", @@ -336,24 +349,52 @@ function renderCollectItemPrompt(item: FollowupRun, idx: number, prompt: string) function collectQueuedPromptMedia( items: FollowupRun[], -): Pick { +): Pick & + Pick { const images: NonNullable = []; const imageOrder: NonNullable = []; const media: NonNullable = []; + const mediaImageSlots: MediaImageLayout["slots"] = []; + const suppressedFactIndexes: number[] = []; + const currentTurnImagesPrepared = items.every(hasPreparedCurrentTurnImages); for (const item of items) { + const mediaOffset = media.length; + const internalItem = item as InternalFollowupRun; if (item.images) { images.push(...item.images); } if (item.imageOrder) { imageOrder.push(...item.imageOrder); } + if (currentTurnImagesPrepared) { + const itemSlots: MediaImageLayout["slots"] = + internalItem.mediaImageLayout?.slots ?? item.imageOrder?.map((kind) => ({ kind })) ?? []; + mediaImageSlots.push( + ...itemSlots.map((slot) => + slot.factIndex === undefined + ? { kind: slot.kind } + : { kind: slot.kind, factIndex: slot.factIndex + mediaOffset }, + ), + ); + suppressedFactIndexes.push( + ...(internalItem.mediaImageLayout?.suppressedFactIndexes ?? []).map( + (factIndex) => factIndex + mediaOffset, + ), + ); + } if (item.media) { media.push(...item.media); } } + const mediaImageLayout = + mediaImageSlots.length > 0 || suppressedFactIndexes.length > 0 + ? { slots: mediaImageSlots, suppressedFactIndexes } + : undefined; return { - ...(images.length > 0 ? { images } : {}), - ...(imageOrder.length > 0 ? { imageOrder } : {}), + ...(currentTurnImagesPrepared ? { currentTurnImagesPrepared: true as const } : {}), + ...(currentTurnImagesPrepared || images.length > 0 ? { images } : {}), + ...(currentTurnImagesPrepared || imageOrder.length > 0 ? { imageOrder } : {}), + ...(mediaImageLayout ? { mediaImageLayout } : {}), ...(media.length > 0 ? { media } : {}), }; }