fix: consume prepared images once in CLI-backed turns (#120721)

* fix(agents): preserve images across CLI dispatch

* test(agents): format CLI dispatch coverage

* fix(reply): consume prepared images once

* test: cover CLI inbound image reply path

* fix(auto-reply): preserve prepared image state

* fix(auto-reply): preserve queued image fallback

* fix(agents): keep CLI image marker internal

* fix(agents): keep image admission marker private

* fix(agents): preserve internal image admission state

* fix(auto-reply): keep image admission marker internal

* test: keep image preparation marker internal

* fix(cli): preserve queued image metadata

* test(cli): cover filtered media image indexes

* test: restore media fixture cleanup

* test: align filtered media fixture with resolver mock

* style: format CLI image forwarding changes

* fix(auto-reply): preserve queued media slot type

* fix(auto-reply): avoid spread in media slot mapping

---------

Co-authored-by: VACInc <3279061+VACInc@users.noreply.github.com>
Co-authored-by: Patrick Erichsen <patrick.a.erichsen@gmail.com>
This commit is contained in:
Vito Cappello 2026-08-15 23:49:08 -04:00 committed by GitHub
parent 717f77678a
commit 9474e55076
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 582 additions and 48 deletions

View file

@ -0,0 +1,114 @@
title: Claude CLI inbound image subscription
scenario:
id: claude-cli-inbound-image-subscription
surface: model-provider
coverage:
primary:
- anthropic.claude-cli-compatibility-provider-capabilities
- channels.inbound-routing-qa-channel
objective: Verify a qa-channel inbound image survives reply admission and reaches a Claude CLI subscription turn exactly once.
successCriteria:
- The selected live provider is claude-cli with subscription auth.
- A qa-channel inbound image receives a visible description through the normal reply path.
- The description mentions the expected red and blue regions.
- Gateway logs prove the turn entered the Claude CLI backend without a structured-image hydration failure.
docsRefs:
- docs/gateway/cli-backends.md
- docs/channels/qa-channel.md
- docs/help/testing.md
codeRefs:
- extensions/qa-channel/src/inbound.ts
- src/auto-reply/reply/get-reply-run-execute.ts
- src/auto-reply/reply/agent-runner-execution.ts
- src/agents/cli-runner/helpers.ts
execution:
kind: flow
providerMode: live-frontier
channel: qa-channel
summary: Run with `pnpm openclaw qa suite --provider-mode live-frontier --cli-auth-mode subscription --model claude-cli/claude-sonnet-4-6 --alt-model claude-cli/claude-sonnet-4-6 --scenario claude-cli-inbound-image-subscription`.
config:
authMode: subscription
requiredProvider: claude-cli
conversationId: qa-claude-cli-inbound-image-subscription
prompt: "Inbound Claude CLI image check: describe the top and bottom colors in the attached image in one short sentence."
requiredColorGroups:
- [red, scarlet, crimson]
- [blue, azure, teal, cyan, aqua]
flow:
steps:
- name: describes an inbound image through the Claude CLI reply path
actions:
- set: selected
value:
expr: splitModelRef(env.primaryModel)
- set: preserveEnv
value:
expr: "String(env.gateway.runtimeEnv.OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV ?? '')"
- assert:
expr: "env.providerMode === 'live-frontier' && selected?.provider === config.requiredProvider"
message:
expr: "`expected live provider ${config.requiredProvider}, got mode=${env.providerMode} model=${env.primaryModel}`"
- assert:
expr: "env.gateway.runtimeEnv.OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE === config.authMode"
message:
expr: "`expected Claude CLI auth mode ${config.authMode}, got ${env.gateway.runtimeEnv.OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE ?? 'unset'}`"
- assert:
expr: "!preserveEnv.includes('ANTHROPIC_API_KEY')"
message:
expr: "`expected subscription auth without ANTHROPIC_API_KEY preservation, got ${preserveEnv}`"
- call: waitForGatewayHealthy
args:
- ref: env
- 60000
- call: waitForQaChannelReady
args:
- ref: env
- 60000
- call: reset
- set: gatewayLogCursor
value:
expr: markGatewayLogCursor()
- set: outboundStartIndex
value:
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
- sendInbound:
accountId: default
conversation:
id:
expr: config.conversationId
kind: direct
senderId: qa-operator
senderName: QA Operator
text:
expr: config.prompt
attachments:
- kind: image
mimeType: image/png
fileName: claude-cli-inbound-red-top-blue-bottom.png
altText: red on top blue on bottom
contentBase64:
expr: imageUnderstandingValidPngBase64
- call: waitForOutboundMessage
saveAs: outbound
args:
- ref: state
- lambda:
params: [candidate]
expr: "candidate.conversation.id === config.conversationId && config.requiredColorGroups.every((group) => group.some((color) => normalizeLowercaseStringOrEmpty(candidate.text).includes(color)))"
- expr: resolveQaLiveTurnTimeoutMs(env, 90000, env.primaryModel)
- sinceIndex:
ref: outboundStartIndex
- set: gatewayLog
value:
expr: "String(readGatewayLogs() ?? '').slice(gatewayLogCursor)"
- assert:
expr: "gatewayLog.includes('cli exec: provider=claude-cli')"
message:
expr: "`expected Claude CLI execution trace; log tail=${gatewayLog.slice(-2000)}`"
- assert:
expr: "!gatewayLog.includes('failed to hydrate')"
message:
expr: "`unexpected structured image hydration failure; log tail=${gatewayLog.slice(-2000)}`"
detailsExpr: "`provider=${selected?.provider} model=${selected?.model} auth=${config.authMode} reply=${JSON.stringify(outbound.text)} cliTrace=true`"

View file

@ -194,6 +194,27 @@ describe("prepareCliPromptImagePayload prompt references", () => {
}
});
it("does not hydrate media suppressed during current-turn admission", async () => {
await expect(
prepareCliPromptImagePayload({
backend: { command: "claude" },
prompt: "describe the attachment",
imagePrompt: "describe the attachment",
workspaceDir: "/workspace",
images: [],
imageOrder: [],
mediaImageLayout: { slots: [], suppressedFactIndexes: [0] },
media: [
{
path: "/openclaw-test-missing/current.png",
contentType: "image/png",
hydrationSuppressed: true,
},
],
}),
).resolves.toEqual({ prompt: "describe the attachment" });
});
it("delivers readable structured images when an unresolved attachment is hydration-suppressed", async () => {
const workspaceDir = await fs.mkdtemp(
path.join(resolvePreferredOpenClawTmpDir(), "openclaw-cli-mixed-media-"),
@ -676,6 +697,10 @@ describe("writeCliImages", () => {
},
],
imageOrder: ["offloaded", "inline"],
mediaImageLayout: {
slots: [{ kind: "offloaded", factIndex: 0 }, { kind: "inline" }],
suppressedFactIndexes: [],
},
media: [{ url: `media://inbound/${mediaId}`, contentType: "image/png" }],
});

View file

@ -678,26 +678,14 @@ describe("runCliAgent spawn path", () => {
expect(invokeNode).not.toHaveBeenCalled();
});
it("allows non-hydratable image facts on a text-only node turn", async () => {
const invokeNode = vi.fn(async (params: Parameters<typeof invokeNodeClaudeCliRun>[0]) => {
params.onProgress(
[
JSON.stringify({ type: "system", subtype: "init", session_id: "node-text-only" }),
JSON.stringify({ type: "result", session_id: "node-text-only", result: "ok" }),
"",
].join("\n"),
);
return {
ok: true,
payloadJSON: JSON.stringify({ exitCode: 0, stderrTail: "", truncated: false }),
};
});
it("rejects prepared offloaded images before invoking a node-placed Claude session", async () => {
const invokeNode = vi.fn();
setCliRunnerExecuteTestDeps({ invokeNodeClaudeCliRun: invokeNode });
const context = buildPreparedCliRunContext({
provider: "claude-cli",
model: "claude-opus-4-8",
runId: "run-node-text-only-media-facts",
prompt: "already described",
runId: "run-node-offloaded-media-facts",
prompt: "describe the attachment",
sessionEntry: {
sessionId: "openclaw-session",
updatedAt: 1,
@ -705,13 +693,24 @@ describe("runCliAgent spawn path", () => {
execNode: "node-a",
},
});
context.params.media = [
{ kind: "image" },
{ kind: "image", url: "https://example.test/described.png" },
];
const preparedParams = context.params as typeof context.params & {
mediaImageLayout?: {
slots: Array<{ kind: "inline" | "offloaded"; factIndex?: number }>;
suppressedFactIndexes: number[];
};
};
preparedParams.mediaImageLayout = {
slots: [{ kind: "offloaded", factIndex: 0 }],
suppressedFactIndexes: [],
};
context.params.images = [];
context.params.imageOrder = ["offloaded"];
context.params.media = [{ kind: "image", path: "/tmp/offloaded.png" }];
await expect(executePreparedCliRun(context)).resolves.toMatchObject({ text: "ok" });
expect(invokeNode).toHaveBeenCalledOnce();
await expect(executePreparedCliRun(context)).rejects.toThrow(
"paired-node Claude CLI sessions do not support attachments or images",
);
expect(invokeNode).not.toHaveBeenCalled();
});
it("does not inject hardcoded 'Tools are disabled' text into CLI arguments", async () => {

View file

@ -20,6 +20,7 @@ import {
detectImageReferences,
hasHydratableMediaImages,
} from "../embedded-agent-runner/run/images.js";
import type { MediaImageLayout } from "../embedded-agent-runner/run/prompt-image-metadata.js";
import { applyPluginTextReplacements } from "../plugin-text-transforms.js";
import { prepareCliBundleMcpCaptureAttempt } from "./bundle-mcp.js";
import { buildClaudeOwnerKey, closeClaudeSession } from "./claude-live-registry.js";
@ -117,13 +118,17 @@ type ExecutePreparedCliRunOptions = {
onPhase?: (phase: "send" | "resolve" | "cleanup") => void;
};
type PreparedCliRunInternalParams = PreparedCliRunContext["params"] & {
mediaImageLayout?: MediaImageLayout;
};
/** Executes a prepared CLI run context and returns normalized CLI output. */
export async function executePreparedCliRun(
context: PreparedCliRunContext,
cliSessionIdToUse?: string,
options?: ExecutePreparedCliRunOptions,
): Promise<CliOutput> {
const params = context.params;
const params = context.params as PreparedCliRunInternalParams;
if (params.abortSignal?.aborted) {
throw createCliAbortError();
}
@ -166,7 +171,9 @@ export async function executePreparedCliRun(
if (
nodePlacement &&
((params.images?.length ?? 0) > 0 ||
hasHydratableMediaImages(params.media) ||
(params.mediaImageLayout
? params.mediaImageLayout.slots.length > 0
: hasHydratableMediaImages(params.media)) ||
(params.imagePrompt ? detectImageReferences(params.imagePrompt).length > 0 : false))
) {
throw new Error("paired-node Claude CLI sessions do not support attachments or images");
@ -181,6 +188,7 @@ export async function executePreparedCliRun(
localRoots: getAgentScopedMediaLocalRoots(params.config ?? {}, params.agentId),
images: params.images,
imageOrder: params.imageOrder,
mediaImageLayout: params.mediaImageLayout,
media: params.media,
});
prompt = imagePayload.prompt;

View file

@ -34,6 +34,7 @@ import {
detectAndLoadPromptImages,
detectImageReferences,
} from "../embedded-agent-runner/run/images.js";
import type { MediaImageLayout } from "../embedded-agent-runner/run/prompt-image-metadata.js";
import { resolveDefaultModelForAgent } from "../model-selection.js";
import type { AgentTool } from "../runtime/index.js";
import { detectRuntimeShell } from "../shell-utils.js";
@ -391,6 +392,7 @@ export async function prepareCliPromptImagePayload(params: {
localRoots?: readonly string[];
images?: ImageContent[];
imageOrder?: PromptImageOrderEntry[];
mediaImageLayout?: MediaImageLayout;
media?: MediaFact[];
}): Promise<{
prompt: string;
@ -402,6 +404,7 @@ export async function prepareCliPromptImagePayload(params: {
const needsHydration =
params.imagePrompt !== undefined ||
Boolean(params.media?.length) ||
Boolean(params.mediaImageLayout) ||
(!params.images?.length && detectImageReferences(imagePrompt).length > 0);
const imageResult = needsHydration
? await detectAndLoadPromptImages({
@ -411,6 +414,7 @@ export async function prepareCliPromptImagePayload(params: {
model: { input: ["text", "image"] },
existingImages: params.images,
imageOrder: params.imageOrder,
mediaImageLayout: params.mediaImageLayout,
maxBytes: MAX_IMAGE_BYTES,
localRoots: params.localRoots,
})

View file

@ -411,13 +411,17 @@ describe("runEmbeddedAgentViaCliBackendIfEligible execution", () => {
expect(onExecutionStarted).toHaveBeenCalledWith({ lifecycleGeneration: "gen-1" });
});
it("retains prompt media facts through the embedded-to-CLI bridge", async () => {
it("retains ordered prompt images and media facts through the embedded-to-CLI bridge", async () => {
const images = [{ type: "image" as const, data: "aGVsbG8=", mimeType: "image/png" }];
const imageOrder = ["inline" as const];
const media = [{ path: "/tmp/recall.png", contentType: "image/png" }];
await runEmbeddedAgentViaCliBackendIfEligible(baseRunParams({ media }));
await runEmbeddedAgentViaCliBackendIfEligible(baseRunParams({ images, imageOrder, media }));
expect(runCliAgent.mock.calls[0]?.[0]).toMatchObject({
prompt: "recall prompt",
images,
imageOrder,
media,
});
});

View file

@ -224,6 +224,8 @@ async function runEmbeddedAgentViaCliBackend(
config: params.config,
prompt: params.prompt,
imagePrompt: params.prompt,
images: params.images,
imageOrder: params.imageOrder,
media: params.media,
provider: dispatch.provider,
model: params.model,

View file

@ -4,7 +4,7 @@ export type ImageFactIndex = number | null;
export type MediaImageLayout = {
slots: Array<{ kind: "inline" | "offloaded"; factIndex?: number }>;
suppressedFactIndexes: number[];
suppressedFactIndexes?: number[];
};
export function readPersistedImageBlockFactIndexes(

View file

@ -418,6 +418,7 @@ export async function runCliFallbackCandidate(params: {
],
images: params.currentTurnImages.images,
imageOrder: params.currentTurnImages.imageOrder,
mediaImageLayout: params.currentTurnImages.mediaImageLayout,
skillsSnapshot: turn.followupRun.run.skillsSnapshot,
messageChannel: turn.followupRun.originatingChannel ?? undefined,
messageProvider: hookMessageProvider,

View file

@ -4,6 +4,7 @@ import { normalizeOptionalString } from "@openclaw/normalization-core/string-coe
import { runCliAgent } from "../../agents/cli-runner.js";
import type { RunCliAgentParams } from "../../agents/cli-runner/types.js";
import { clearCliSession, getCliSessionBinding } from "../../agents/cli-session.js";
import type { MediaImageLayout } from "../../agents/embedded-agent-runner/run/prompt-image-metadata.js";
import { extractToolResultText } from "../../agents/embedded-agent-tool-results.js";
import type { EmbeddedAgentRunResult } from "../../agents/embedded-agent.js";
import {
@ -34,6 +35,10 @@ import {
} from "./agent-event-bridge.js";
import { resolveAgentLifecycleTerminalMetadata } from "./agent-lifecycle-terminal.js";
type RunCliAgentInternalParams = RunCliAgentParams & {
mediaImageLayout?: MediaImageLayout;
};
type AgentEventBridge = {
unsubscribe: () => void;
drain: () => Promise<void>;
@ -422,7 +427,7 @@ type RunCliAgentWithLifecycleParams = {
runId: string;
lifecycleGeneration?: string;
provider: string;
runParams: RunCliAgentParams;
runParams: RunCliAgentInternalParams;
startedAt?: number;
emitLifecycleTerminal?: boolean;
onAgentRunStart?: () => void;

View file

@ -1,4 +1,7 @@
import { afterEach, describe, expect, it } from "vitest";
import { prepareCliPromptImagePayload } from "../../agents/cli-runner/helpers.js";
import type { RunCliAgentParams } from "../../agents/cli-runner/types.js";
import { detectAndLoadPromptImages } from "../../agents/embedded-agent-runner/run/images.js";
import { FailoverError } from "../../agents/failover-error.js";
import { installSessionPlacementAdmissionProvider } from "../../agents/session-placement-admission.js";
import type { SessionEntry } from "../../config/sessions.js";
@ -23,6 +26,139 @@ const state = setupAgentRunnerExecutionTestState();
afterEach(resetGeneratedMediaTaskActivityForTests);
describe("executeAgentTurn: CLI session routing", () => {
it("preserves queued image fields from runs created before the prepared marker", async () => {
state.isCliProviderMock.mockReturnValue(true);
state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({
result: await params.run("claude-cli", "claude-opus-5"),
provider: "claude-cli",
model: "claude-opus-5",
attempts: [],
}));
state.runCliAgentMock.mockResolvedValueOnce({
payloads: [{ text: "described" }],
meta: {},
});
const images = [
{
type: "image" as const,
data: "iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAFElEQVR4nGP4z8Dwn4GBgYGJAQoAHxcCAr7cGDwAAAAASUVORK5CYII=",
mimeType: "image/png",
},
];
const imageOrder = ["inline" as const];
const followupRun = createFollowupRun();
followupRun.run.provider = "claude-cli";
followupRun.run.model = "claude-opus-5";
followupRun.images = images;
followupRun.imageOrder = imageOrder;
const executeAgentTurn = await getExecuteAgentTurnForTest();
const result = await executeAgentTurn(
createMinimalRunAgentTurnParams({
followupRun,
sessionCtx: {
Provider: "telegram",
MessageSid: "msg",
} as unknown as TemplateContext,
}),
);
expect(result.kind).toBe("success");
expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", {
images,
imageOrder,
});
});
it("keeps prepared current-turn images aligned with CLI media facts", async () => {
state.isCliProviderMock.mockReturnValue(true);
state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({
result: await params.run("claude-cli", "claude-opus-5"),
provider: "claude-cli",
model: "claude-opus-5",
attempts: [],
}));
const images = [
{
type: "image" as const,
data: "iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAFElEQVR4nGP4z8Dwn4GBgYGJAQoAHxcCAr7cGDwAAAAASUVORK5CYII=",
mimeType: "image/png",
},
];
const imageOrder = ["inline" as const];
const media = [{ path: "/openclaw-test-missing/current.png", contentType: "image/png" }];
const mediaImageLayout = {
slots: [{ kind: "inline" as const, factIndex: 0 }],
suppressedFactIndexes: [],
};
state.runCliAgentMock.mockImplementationOnce(async (params: RunCliAgentParams) => {
const internalParams = params as RunCliAgentParams & {
mediaImageLayout?: typeof mediaImageLayout;
};
await expect(
prepareCliPromptImagePayload({
backend: { command: "claude" },
prompt: params.prompt,
imagePrompt: params.prompt,
workspaceDir: params.workspaceDir,
images: [...images, ...images],
imageOrder: [...imageOrder, ...imageOrder],
media,
}),
).rejects.toThrow("failed to hydrate 1 structured image attachment");
const reconciled = await detectAndLoadPromptImages({
prompt: params.prompt,
media: params.media,
workspaceDir: params.workspaceDir,
model: { input: ["text", "image"] },
existingImages: params.images,
imageOrder: params.imageOrder,
mediaImageLayout: internalParams.mediaImageLayout,
});
expect(reconciled).toMatchObject({
failedMediaCount: 0,
images,
});
return {
payloads: [{ text: "described" }],
meta: {},
};
});
const followupRun = createFollowupRun();
followupRun.run.provider = "claude-cli";
followupRun.run.model = "claude-opus-5";
const preparedFollowupRun = followupRun as typeof followupRun & {
currentTurnImagesPrepared?: true;
mediaImageLayout?: typeof mediaImageLayout;
};
preparedFollowupRun.currentTurnImagesPrepared = true;
preparedFollowupRun.mediaImageLayout = mediaImageLayout;
followupRun.images = images;
followupRun.imageOrder = imageOrder;
followupRun.media = media;
const executeAgentTurn = await getExecuteAgentTurnForTest();
const result = await executeAgentTurn(
createMinimalRunAgentTurnParams({
followupRun,
sessionCtx: {
Provider: "telegram",
MessageSid: "msg",
media,
} as unknown as TemplateContext,
}),
);
expect(result.kind).toBe("success");
expectMockCallArgFields(state.runCliAgentMock, 0, "CLI run params", {
images,
imageOrder,
mediaImageLayout,
media,
});
});
it("forwards the static extra system prompt to CLI backends", async () => {
state.isCliProviderMock.mockReturnValue(true);
state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({

View file

@ -64,7 +64,7 @@ import { createAgentTurnTimingTracker } from "./agent-runner-turn-timing.js";
import { resolveQueuedReplyRuntimeConfig } from "./agent-runner-utils.js";
import { prepareChannelRunAdmission } from "./channel-run-admission.js";
import { shouldNotifyUserAboutCompaction } from "./compaction-notice.js";
import { resolveCurrentTurnImages } from "./current-turn-images.js";
import { type CurrentTurnImages, resolveCurrentTurnImages } from "./current-turn-images.js";
import type { FollowupRun } from "./queue.js";
import type { ReplyMediaContext } from "./reply-media-paths.js";
import { createReplyMediaContext } from "./reply-media-paths.runtime.js";
@ -74,6 +74,12 @@ import {
} from "./reply-operation-abort.js";
import { isReplyProfilerEnabled } from "./reply-timing-tracker.js";
type InternalFollowupRun = FollowupRun & {
/** Keep admission state out of the public plugin-facing FollowupRun contract. */
currentTurnImagesPrepared?: true;
mediaImageLayout?: CurrentTurnImages["mediaImageLayout"];
};
function resolveRunStartupPhase(
phase: EmbeddedAgentExecutionPhase,
): ChatRunStartupPhase | undefined {
@ -180,7 +186,7 @@ async function executeAgentTurnInternalWithRetryState(
});
}
let replyMediaContext: ReplyMediaContext;
let currentTurnImages: Awaited<ReturnType<typeof resolveCurrentTurnImages>>;
let currentTurnImages: CurrentTurnImages;
try {
replyMediaContext =
params.replyMediaContext ??
@ -201,14 +207,27 @@ async function executeAgentTurnInternalWithRetryState(
requesterSenderE164: params.followupRun.run.senderE164,
}),
);
currentTurnImages = await agentTurnTiming.measure("current_turn_images", () =>
resolveCurrentTurnImages({
ctx: params.sessionCtx,
cfg: runtimeConfig,
images: params.followupRun.images ?? params.opts?.images,
imageOrder: params.followupRun.imageOrder ?? params.opts?.imageOrder,
}),
);
const internalFollowupRun = params.followupRun as InternalFollowupRun;
const hasQueuedCurrentTurnImages =
internalFollowupRun.currentTurnImagesPrepared === true ||
Object.hasOwn(params.followupRun, "images") ||
Object.hasOwn(params.followupRun, "imageOrder");
// Queue admission owns current-turn materialization, including empty results.
// Re-scanning here can resurrect suppressed media or duplicate loaded images.
currentTurnImages = hasQueuedCurrentTurnImages
? {
images: params.followupRun.images,
imageOrder: params.followupRun.imageOrder,
mediaImageLayout: internalFollowupRun.mediaImageLayout,
}
: await agentTurnTiming.measure("current_turn_images", () =>
resolveCurrentTurnImages({
ctx: params.sessionCtx,
cfg: runtimeConfig,
images: params.opts?.images,
imageOrder: params.opts?.imageOrder,
}),
);
} catch (error) {
clearAgentRunContext(runId, lifecycleGeneration);
throw error;

View file

@ -1,5 +1,6 @@
// Tracks image attachments that belong to the current reply turn.
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import type { MediaImageLayout } from "../../agents/embedded-agent-runner/run/prompt-image-metadata.js";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import { logVerbose } from "../../globals.js";
import { formatErrorMessage } from "../../infra/errors.js";
@ -26,6 +27,15 @@ type OrderedTurnImage = {
sequence: number;
};
export type CurrentTurnImages = {
images?: ImageContent[];
imageOrder?: PromptImageOrderEntry[];
imageSourceIndexes?: Array<number | undefined>;
unresolvedSourceIndexes?: number[];
/** Admission-owned slot-to-media identity used by later runtime adapters. */
mediaImageLayout?: MediaImageLayout;
};
function collectCurrentImageAttachments(ctx: MsgContext): CurrentImageAttachment[] {
return normalizeAttachments(ctx).flatMap((attachment) => {
const mediaPath = normalizeOptionalString(attachment.path);
@ -130,12 +140,7 @@ export async function resolveCurrentTurnImages(params: {
images?: ImageContent[];
imageOrder?: PromptImageOrderEntry[];
extractedFileImages?: ExtractedFileImage[];
}): Promise<{
images?: ImageContent[];
imageOrder?: PromptImageOrderEntry[];
imageSourceIndexes?: Array<number | undefined>;
unresolvedSourceIndexes?: number[];
}> {
}): Promise<CurrentTurnImages> {
const entries: OrderedTurnImage[] = [];
appendOrderedImages({
entries,

View file

@ -229,6 +229,20 @@ export async function executePreparedReplyRun(state: PreparedReplyRunAdmission)
imageOrder: currentTurnImages.imageOrder,
imageSourceIndexes: currentTurnImages.imageSourceIndexes,
});
const promptMediaSourceIndexes = currentTurnImages.imageSourceIndexes?.map((sourceIndex) => {
if (sourceIndex === undefined) {
return undefined;
}
const promptIndex = inboundMediaIndexes.indexOf(sourceIndex);
return promptIndex >= 0 ? promptIndex : undefined;
});
const promptMediaImageLayout = buildPersistedMediaImageLayout({
ctx: {},
media: promptMediaForRun,
ctxMediaCount: inboundMediaIndexes.length,
imageOrder: currentTurnImages.imageOrder,
imageSourceIndexes: promptMediaSourceIndexes,
});
const inputProvenance = ctx.InputProvenance ?? sessionCtx.InputProvenance;
const userTurnTimestamp = normalizeMessageTimestampMs(ctx.Timestamp);
// prompt-prelude substitutes MEDIA_ONLY_USER_TEXT as transcriptBody for
@ -357,8 +371,10 @@ export async function executePreparedReplyRun(state: PreparedReplyRunAdmission)
...(queuedToolsAllow !== undefined ? { toolsAllow: queuedToolsAllow } : {}),
...(opts?.disableTools !== undefined ? { disableTools: opts.disableTools } : {}),
enqueuedAt: Date.now(),
currentTurnImagesPrepared: true as const,
images: currentTurnImages.images,
imageOrder: currentTurnImages.imageOrder,
mediaImageLayout: promptMediaImageLayout,
media: promptMediaForRun,
// Originating channel for reply routing.
originatingChannel: replyRoute.channel,

View file

@ -1655,6 +1655,13 @@ describe("runPreparedReply media-only handling", () => {
},
});
expect(call.followupRun.imageOrder).toEqual(["inline"]);
expect(
(
call.followupRun as typeof call.followupRun & {
currentTurnImagesPrepared?: true;
}
).currentTurnImagesPrepared,
).toBe(true);
expect(resolveCurrentTurnImagesMock).toHaveBeenCalledWith({
ctx: expect.objectContaining({
media: [{ path: imagePath, workspaceDir: "/tmp" }],
@ -1897,6 +1904,70 @@ describe("runPreparedReply media-only handling", () => {
expect(call.followupRun.prompt).toContain("a tiny dot image");
});
it("indexes the runtime image layout against filtered prompt media", async () => {
const imageData = Buffer.from("runtime image bytes");
const imagePath = "/tmp/current.png";
resolveCurrentTurnImagesMock.mockResolvedValueOnce({
images: [{ type: "image", data: imageData.toString("base64"), mimeType: "image/png" }],
imageOrder: ["inline"],
imageSourceIndexes: [1],
});
const result = await runPrepared({
ctx: {
...createInboundBody("describe the image"),
media: [
{
path: "/tmp/voice.ogg",
contentType: "audio/ogg",
transcribed: true,
},
{ path: imagePath, contentType: "image/png", workspaceDir: "/tmp" },
],
OriginatingChannel: "webchat",
OriginatingTo: "webchat:local",
ChatType: "direct",
},
sessionCtx: {
...createSessionBody("describe the image"),
Provider: "webchat",
OriginatingChannel: "webchat",
OriginatingTo: "webchat:local",
ChatType: "direct",
},
});
expect(result).toEqual({ text: "ok" });
const call = requireRunReplyAgentCall();
expect(call.followupRun.media).toHaveLength(1);
expect(call.followupRun.media?.[0]).toMatchObject({
path: imagePath,
contentType: "image/png",
workspaceDir: "/tmp",
});
expect(call.followupRun.images).toEqual([
{
type: "image",
data: imageData.toString("base64"),
mimeType: "image/png",
},
]);
expect(
(
call.followupRun as typeof call.followupRun & {
mediaImageLayout?: { slots: Array<{ kind: string; factIndex?: number }> };
}
).mediaImageLayout,
).toEqual({ slots: [{ kind: "inline", factIndex: 0 }] });
expect(
(
call.followupRun.userTurnTranscriptRecorder?.message as unknown as Record<string, unknown>
)?.["__openclaw"],
).toMatchObject({
mediaImageLayout: { slots: [{ kind: "inline", factIndex: 1 }] },
});
});
it("does not send a standalone reset notice for reply-producing /new turns", async () => {
await runPrepared({
ctx: {

View file

@ -32,6 +32,14 @@ import {
import { resolveFollowupDeliveryContextKey } from "./queue/drain.js";
import { clearFollowupQueue, getExistingFollowupQueue } from "./queue/state.js";
type InternalFollowupRun = FollowupRun & {
currentTurnImagesPrepared?: true;
mediaImageLayout?: {
slots: Array<{ kind: "inline" | "offloaded"; factIndex?: number }>;
suppressedFactIndexes: number[];
};
};
installQueueRuntimeErrorSilencer();
function createQueueSettings(overrides: Partial<QueueSettings> = {}): QueueSettings {
@ -53,7 +61,7 @@ function enqueueTestRun(
}
function createDrainRecorder(expectedCalls = 1) {
const calls: FollowupRun[] = [];
const calls: Array<FollowupRun & { currentTurnImagesPrepared?: true }> = [];
const done = createDeferred();
const runFollowup = async (run: FollowupRun) => {
calls.push(run);
@ -2019,6 +2027,82 @@ describe("followup queue collect routing", () => {
expect(calls[0]?.imageOrder).toEqual(["inline", "inline"]);
});
it("preserves prepared empty image state across collected batches", async () => {
const key = `test-collect-prepared-empty-images-${Date.now()}`;
const { calls, done, runFollowup } = createDrainRecorder();
const settings = createQueueSettings();
const missingMedia = {
path: "/openclaw-test-missing/current.png",
contentType: "image/png",
hydrationSuppressed: true,
};
for (const prompt of ["one", "two"]) {
const preparedRun: InternalFollowupRun = {
...createRun({
prompt,
originatingChannel: "slack",
originatingTo: "channel:A",
}),
currentTurnImagesPrepared: true,
images: [],
imageOrder: [],
media: [missingMedia],
mediaImageLayout: { slots: [], suppressedFactIndexes: [0] },
};
enqueueFollowupRun(key, preparedRun, settings);
}
scheduleFollowupDrain(key, runFollowup);
await done.promise;
const collected = calls[0] as InternalFollowupRun | undefined;
expect(collected?.currentTurnImagesPrepared).toBe(true);
expect(collected?.images).toEqual([]);
expect(collected?.imageOrder).toEqual([]);
expect(collected?.media).toEqual([missingMedia, missingMedia]);
expect(collected?.mediaImageLayout).toEqual({
slots: [],
suppressedFactIndexes: [0, 1],
});
});
it("offsets prepared media layout fact indexes across collected batches", async () => {
const key = `test-collect-prepared-image-layout-${Date.now()}`;
const { calls, done, runFollowup } = createDrainRecorder();
const settings = createQueueSettings();
for (const [index, prompt] of ["one", "two"].entries()) {
const preparedRun: InternalFollowupRun = {
...createRun({
prompt,
originatingChannel: "slack",
originatingTo: "channel:A",
}),
currentTurnImagesPrepared: true,
images: [],
imageOrder: ["offloaded"],
media: [{ path: `/tmp/offloaded-${index}.png`, contentType: "image/png" }],
mediaImageLayout: {
slots: [{ kind: "offloaded", factIndex: 0 }],
suppressedFactIndexes: [],
},
};
enqueueFollowupRun(key, preparedRun, settings);
}
scheduleFollowupDrain(key, runFollowup);
await done.promise;
expect((calls[0] as InternalFollowupRun | undefined)?.mediaImageLayout).toEqual({
slots: [
{ kind: "offloaded", factIndex: 0 },
{ kind: "offloaded", factIndex: 1 },
],
suppressedFactIndexes: [],
});
});
it("splits collect batches when sender authorization changes", async () => {
const { key, calls, done, runFollowup, settings } = createQueueCase(
`test-collect-auth-split-${Date.now()}`,

View file

@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
import { expectDefined } from "@openclaw/normalization-core";
import { stableStringify } from "@openclaw/normalization-core";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import type { MediaImageLayout } from "../../../agents/embedded-agent-runner/run/prompt-image-metadata.js";
import { runAgentHarnessBeforeMessageWriteHook } from "../../../agents/harness/hook-helpers.js";
import { readToolAllowlistIntersection } from "../../../agents/tool-policy.js";
import { normalizeChatType } from "../../../channels/chat-type.js";
@ -45,6 +46,17 @@ import {
type FollowupRun,
} from "./types.js";
type InternalFollowupRun = FollowupRun & {
/** Keep admission state out of the public plugin-facing FollowupRun contract. */
currentTurnImagesPrepared?: true;
/** Admission-owned layout; fact indexes are relative to this run's media array. */
mediaImageLayout?: MediaImageLayout;
};
function hasPreparedCurrentTurnImages(run: FollowupRun): boolean {
return (run as InternalFollowupRun).currentTurnImagesPrepared === true;
}
// Persists the most recent runFollowup callback per queue key so that
// enqueueFollowupRun can restart a drain that finished and deleted the queue.
const FOLLOWUP_DRAIN_CALLBACKS_KEY = Symbol.for("openclaw.followupDrainCallbacks");
@ -225,6 +237,7 @@ export function resolveFollowupDeliveryContextKey(run: FollowupRun): string {
accountId: run.originatingAccountId,
threadId: run.originatingThreadId,
}),
hasPreparedCurrentTurnImages(run),
run.originatingChatId ?? "",
resolveFollowupReplyAnchor(run) ?? "",
run.originatingReplyToMode ?? "",
@ -336,24 +349,52 @@ function renderCollectItemPrompt(item: FollowupRun, idx: number, prompt: string)
function collectQueuedPromptMedia(
items: FollowupRun[],
): Pick<FollowupRun, "images" | "imageOrder" | "media"> {
): Pick<FollowupRun, "images" | "imageOrder" | "media"> &
Pick<InternalFollowupRun, "currentTurnImagesPrepared" | "mediaImageLayout"> {
const images: NonNullable<FollowupRun["images"]> = [];
const imageOrder: NonNullable<FollowupRun["imageOrder"]> = [];
const media: NonNullable<FollowupRun["media"]> = [];
const mediaImageSlots: MediaImageLayout["slots"] = [];
const suppressedFactIndexes: number[] = [];
const currentTurnImagesPrepared = items.every(hasPreparedCurrentTurnImages);
for (const item of items) {
const mediaOffset = media.length;
const internalItem = item as InternalFollowupRun;
if (item.images) {
images.push(...item.images);
}
if (item.imageOrder) {
imageOrder.push(...item.imageOrder);
}
if (currentTurnImagesPrepared) {
const itemSlots: MediaImageLayout["slots"] =
internalItem.mediaImageLayout?.slots ?? item.imageOrder?.map((kind) => ({ kind })) ?? [];
mediaImageSlots.push(
...itemSlots.map((slot) =>
slot.factIndex === undefined
? { kind: slot.kind }
: { kind: slot.kind, factIndex: slot.factIndex + mediaOffset },
),
);
suppressedFactIndexes.push(
...(internalItem.mediaImageLayout?.suppressedFactIndexes ?? []).map(
(factIndex) => factIndex + mediaOffset,
),
);
}
if (item.media) {
media.push(...item.media);
}
}
const mediaImageLayout =
mediaImageSlots.length > 0 || suppressedFactIndexes.length > 0
? { slots: mediaImageSlots, suppressedFactIndexes }
: undefined;
return {
...(images.length > 0 ? { images } : {}),
...(imageOrder.length > 0 ? { imageOrder } : {}),
...(currentTurnImagesPrepared ? { currentTurnImagesPrepared: true as const } : {}),
...(currentTurnImagesPrepared || images.length > 0 ? { images } : {}),
...(currentTurnImagesPrepared || imageOrder.length > 0 ? { imageOrder } : {}),
...(mediaImageLayout ? { mediaImageLayout } : {}),
...(media.length > 0 ? { media } : {}),
};
}