From 73fc0f5c5fa3bce614e362e9f98243d31513d7dc Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 4 Jul 2026 20:28:52 -0400 Subject: [PATCH] fix(channels): expose inbound media download failures [AI-assisted] (#100119) * fix(channels): expose inbound media download failures * fix(msteams): correlate attachment references --- CHANGELOG.md | 1 + extensions/feishu/src/bot-content.ts | 85 ++++++++--- extensions/feishu/src/bot.helpers.test.ts | 14 +- extensions/feishu/src/bot.test.ts | 143 ++++++++++++++++++ extensions/feishu/src/bot.ts | 30 +++- extensions/feishu/src/post.test.ts | 4 + extensions/feishu/src/post.ts | 20 ++- .../imessage/src/monitor.media-policy.test.ts | 70 ++++++++- .../src/monitor/inbound-processing.test.ts | 55 +++++++ .../src/monitor/inbound-processing.ts | 5 +- .../src/monitor/media-staging.test.ts | 18 ++- .../imessage/src/monitor/media-staging.ts | 12 +- .../imessage/src/monitor/monitor-provider.ts | 134 +++++++++++----- extensions/line/src/bot-handlers.test.ts | 25 +++ extensions/line/src/bot-handlers.ts | 3 + .../line/src/bot-message-context.test.ts | 24 +++ extensions/line/src/bot-message-context.ts | 29 +++- .../src/mattermost/monitor-resources.test.ts | 26 +++- .../src/mattermost/monitor-resources.ts | 18 +++ .../mattermost/src/mattermost/monitor.ts | 19 ++- .../msteams/src/attachments.helpers.test.ts | 60 ++++++++ extensions/msteams/src/attachments.ts | 1 + extensions/msteams/src/attachments/html.ts | 111 +++++++++++++- extensions/msteams/src/attachments/types.ts | 1 + .../src/monitor-handler/inbound-media.test.ts | 35 ++++- .../src/monitor-handler/inbound-media.ts | 23 +++ .../src/monitor-handler/message-handler.ts | 25 ++- .../event-handler.inbound-context.test.ts | 108 +++++++++++++ .../signal/src/monitor/event-handler.ts | 28 +++- ...o-reply.connection-and-logging.e2e.test.ts | 11 ++ extensions/whatsapp/src/auto-reply/monitor.ts | 5 +- .../src/auto-reply/monitor/group-gating.ts | 8 +- .../monitor/process-message.test.ts | 24 ++- .../src/auto-reply/monitor/process-message.ts | 16 +- extensions/whatsapp/src/inbound.media.test.ts | 40 ++++- .../whatsapp/src/inbound/media.node.test.ts | 11 ++ extensions/whatsapp/src/inbound/media.ts | 51 +++---- extensions/whatsapp/src/inbound/monitor.ts | 36 ++++- extensions/whatsapp/src/inbound/types.ts | 1 + .../zalo/src/monitor.image.polling.test.ts | 40 ++++- extensions/zalo/src/monitor.ts | 19 ++- .../test-support/lifecycle-test-support.ts | 3 +- scripts/plugin-sdk-surface-report.mjs | 4 +- src/channels/inbound-event/media.test.ts | 18 +++ src/channels/inbound-event/media.ts | 18 +++ src/plugin-sdk/channel-inbound.ts | 1 + 46 files changed, 1269 insertions(+), 164 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 946fed1f4aa..9e3b9fa9f19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ Docs: https://docs.openclaw.ai - **ClawRouter auth profiles:** resolve credential-scoped catalog models during agent runs when the proxy key is stored in an auth profile, and document plugin and model allowlists. - **Telegram durability:** recover stalled ingress claims, retry restart-dropped media, survive transient polling errors, dead-letter poison updates, preserve forwarded rich text, route plugin callbacks correctly, keep progress updates in one stable multi-line window, map self-hosted Bot API container paths through trusted host roots, and fall back safely when Telegram rejects rich final replies. (#97118, #98102, #98735, #98775, #98776, #97174, #98907, #91984, #98786) Thanks @vincentkoc, @luoyanglang, @DaveArcher18, @obviyus, @goldmar, @Marvinthebored, @Dizesales, and @shakkernerd. +- **Cross-channel inbound media:** preserve captions and expose unavailable-attachment notices when WhatsApp, LINE, Signal, iMessage, Microsoft Teams, Feishu, Mattermost, or Zalo cannot materialize inbound media, instead of dispatching phantom placeholders or dropping media-only turns. (#100092) - **Agent and context reliability:** preserve runtime overrides, steered subagent tasks, fallback tool-call hints, and legacy reseed attachments; soft-resume CLI sessions across prompt-only drift; improve harness-aware context estimation and compaction prechecks; time out silent local streams; recover mid-stream failures; and cap Gateway run-cache growth. (#92237, #77539, #99851, #99839, #99822, #97928, #97861, #98525, #95430, #77973) Thanks @sercada, @amittell, @obviyus, @liuhao1024, @yetval, @osolmaz, @lzyyzznl, @vincentkoc, @alexelgier, and @fede-kamel. - **Provider and network safety:** bound oversized or malformed responses across Moonshot, MiniMax, Anthropic OAuth, Discord, Matrix, SMS, browser, update, embeddings, Tlön, and Inworld paths. (#96502, #96322, #96644, #97693, #97662, #97999, #98455, #98508, #98554, #98496, #98660) Thanks @hugenshen, @cursoragent, @lsr911, @solodmd, @Alix-007, @wings1029, @lzyyzznl, @sunlit-deng, @vincentkoc, and @Pandah97. - **Channel delivery and routing:** keep Slack replies in the active thread, preserve account-bound delivery routes, apply response prefixes, suppress internal traces and unwanted fallback replies, and retain WeChat session routing for opaque account ids. (#97168, #98240, #89949, #93639, #97989, #80928, #93686) Thanks @LiuwqGit, @gorkem2020, @yetval, @wangwllu, @ZengWen-DT, @alexuser, @UnClouded77, @zhangguiping-xydt, @htkillermax-gif, and @vincentkoc. diff --git a/extensions/feishu/src/bot-content.ts b/extensions/feishu/src/bot-content.ts index 9123632e78d..dec08a6bb09 100644 --- a/extensions/feishu/src/bot-content.ts +++ b/extensions/feishu/src/bot-content.ts @@ -141,17 +141,8 @@ export function parseMessageContent(content: string, messageType: string): strin if (messageType === "text") { return parsed.text || ""; } - if (["image", "file", "audio", "video", "media", "sticker"].includes(messageType)) { - if (messageType === "audio") { - const speechToText = - typeof parsed.speech_to_text === "string" ? parsed.speech_to_text.trim() : ""; - if (speechToText) { - return speechToText; - } - } - const placeholder = inferPlaceholder(messageType); - const fileName = typeof parsed.file_name === "string" ? parsed.file_name.trim() : ""; - return fileName ? `${placeholder} (${fileName})` : placeholder; + if (FEISHU_MEDIA_MESSAGE_TYPES.has(messageType)) { + return formatFeishuMediaContent(parsed, messageType).body; } if (messageType === "share_chat") { if (parsed && typeof parsed === "object") { @@ -177,6 +168,60 @@ export function parseMessageContent(content: string, messageType: string): strin } } +const FEISHU_MEDIA_MESSAGE_TYPES = new Set(["image", "file", "audio", "video", "media", "sticker"]); + +function formatFeishuMediaContent( + parsed: Record, + messageType: string, +): { body: string; mediaPlaceholder?: string; unavailableBody?: string } { + const speechToText = + messageType === "audio" && typeof parsed.speech_to_text === "string" + ? parsed.speech_to_text.trim() + : ""; + if (speechToText) { + return { body: speechToText }; + } + + const placeholder = inferPlaceholder(messageType); + const fileName = typeof parsed.file_name === "string" ? parsed.file_name.trim() : ""; + const body = fileName ? `${placeholder} (${fileName})` : placeholder; + return { + body, + mediaPlaceholder: placeholder, + unavailableBody: fileName || undefined, + }; +} + +export function resolveFeishuMediaFailurePresentation( + content: string, + messageType: string, +): { mediaPlaceholder?: string; unavailableBody?: string } { + if (messageType === "post") { + return { + unavailableBody: parsePostContent(content, { + renderMediaPlaceholders: false, + emptyTextFallback: "", + }).textContent, + }; + } + if (!FEISHU_MEDIA_MESSAGE_TYPES.has(messageType)) { + return {}; + } + try { + const parsed: unknown = JSON.parse(content); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + return {}; + } + const presentation = formatFeishuMediaContent(parsed as Record, messageType); + return { + mediaPlaceholder: presentation.mediaPlaceholder, + unavailableBody: presentation.unavailableBody, + }; + } catch { + return {}; + } +} + function formatSubMessageContent(content: string, contentType: string): string { try { const parsed = JSON.parse(content); @@ -380,18 +425,19 @@ export async function resolveFeishuMediaList(params: { maxBytes: number; log?: (msg: string) => void; accountId?: string; -}): Promise { +}): Promise<{ media: FeishuMediaInfo[]; unavailableCount: number }> { const { cfg, messageId, messageType, content, maxBytes, log, accountId } = params; const mediaTypes = ["image", "file", "audio", "video", "media", "sticker", "post"]; if (!mediaTypes.includes(messageType)) { - return []; + return { media: [], unavailableCount: 0 }; } const out: FeishuMediaInfo[] = []; + let unavailableCount = 0; if (messageType === "post") { const { imageKeys, mediaKeys } = parsePostContent(content); if (imageKeys.length === 0 && mediaKeys.length === 0) { - return []; + return { media: [], unavailableCount: 0 }; } if (imageKeys.length > 0) { log?.(`feishu: post message contains ${imageKeys.length} embedded image(s)`); @@ -418,6 +464,7 @@ export async function resolveFeishuMediaList(params: { }); log?.(`feishu: downloaded embedded image ${imageKey}, saved to ${saved.path}`); } catch (err) { + unavailableCount += 1; log?.(`feishu: failed to download embedded image ${imageKey}: ${String(err)}`); } } @@ -445,21 +492,22 @@ export async function resolveFeishuMediaList(params: { }); log?.(`feishu: downloaded embedded media ${media.fileKey}, saved to ${saved.path}`); } catch (err) { + unavailableCount += 1; log?.(`feishu: failed to download embedded media ${media.fileKey}: ${String(err)}`); } } - return out; + return { media: out, unavailableCount }; } const mediaKeys = parseMediaKeys(content, messageType); if (!mediaKeys.imageKey && !mediaKeys.fileKey) { - return []; + return { media: [], unavailableCount: 1 }; } try { const fileKey = mediaKeys.fileKey || mediaKeys.imageKey; if (!fileKey) { - return []; + return { media: [], unavailableCount: 1 }; } const result = await saveMessageResourceFeishu({ cfg, @@ -482,7 +530,8 @@ export async function resolveFeishuMediaList(params: { }); log?.(`feishu: downloaded ${messageType} media, saved to ${saved.path}`); } catch (err) { + unavailableCount += 1; log?.(`feishu: failed to download ${messageType} media: ${String(err)}`); } - return out; + return { media: out, unavailableCount }; } diff --git a/extensions/feishu/src/bot.helpers.test.ts b/extensions/feishu/src/bot.helpers.test.ts index 46c205f70a0..3926cded5a2 100644 --- a/extensions/feishu/src/bot.helpers.test.ts +++ b/extensions/feishu/src/bot.helpers.test.ts @@ -1,7 +1,7 @@ // Feishu tests cover bot.helpers plugin behavior. import { describe, expect, it } from "vitest"; import type { ClawdbotConfig } from "../runtime-api.js"; -import { parseMessageContent } from "./bot-content.js"; +import { parseMessageContent, resolveFeishuMediaFailurePresentation } from "./bot-content.js"; import { buildBroadcastSessionKey, buildFeishuAgentBody, @@ -97,12 +97,24 @@ describe("parseMessageContent media placeholders", () => { "audio", ), ).toBe("spoken words"); + expect( + resolveFeishuMediaFailurePresentation( + JSON.stringify({ file_key: "file_audio", speech_to_text: " spoken words " }), + "audio", + ), + ).toEqual({ mediaPlaceholder: undefined, unavailableBody: undefined }); }); it("keeps media filenames as placeholder context without raw payload fields", () => { expect( parseMessageContent(JSON.stringify({ file_key: "file_doc", file_name: "q1.pdf" }), "file"), ).toBe(" (q1.pdf)"); + expect( + resolveFeishuMediaFailurePresentation( + JSON.stringify({ file_key: "file_doc", file_name: "q1.pdf" }), + "file", + ), + ).toEqual({ mediaPlaceholder: "", unavailableBody: "q1.pdf" }); }); }); diff --git a/extensions/feishu/src/bot.test.ts b/extensions/feishu/src/bot.test.ts index 116b1d82942..f05286e0a63 100644 --- a/extensions/feishu/src/bot.test.ts +++ b/extensions/feishu/src/bot.test.ts @@ -2265,6 +2265,107 @@ describe("handleFeishuMessage command authorization", () => { expect(mockDispatchReplyFromConfig).toHaveBeenCalledTimes(1); }); + it("replaces a failed image download placeholder with an unavailable notice", async () => { + mockShouldComputeCommandAuthorized.mockReturnValue(false); + mockDownloadMessageResourceFeishu.mockRejectedValueOnce(new Error("expired image key")); + + await dispatchMessage({ + cfg: { + channels: { feishu: { dmPolicy: "open" } }, + } as ClawdbotConfig, + event: { + sender: { sender_id: { open_id: "ou-sender" } }, + message: { + message_id: "msg-image-failed", + chat_id: "oc-dm", + chat_type: "p2p", + message_type: "image", + content: JSON.stringify({ image_key: "expired-image" }), + }, + }, + }); + + const context = mockCallArg<{ + BodyForAgent?: string; + CommandBody?: string; + MediaPath?: string; + RawBody?: string; + }>(mockFinalizeInboundContext, 0, 0); + expect(context.RawBody).toBe(""); + expect(context.CommandBody).toBe(""); + expect(context.BodyForAgent).toContain("[feishu attachment unavailable]"); + expect(context.BodyForAgent).not.toContain(""); + expect(context.MediaPath).toBeUndefined(); + }); + + it("preserves an audio transcript when the media download fails", async () => { + mockShouldComputeCommandAuthorized.mockReturnValue(false); + mockDownloadMessageResourceFeishu.mockRejectedValueOnce(new Error("expired audio key")); + + await dispatchMessage({ + cfg: { + channels: { feishu: { dmPolicy: "open" } }, + } as ClawdbotConfig, + event: { + sender: { sender_id: { open_id: "ou-sender" } }, + message: { + message_id: "msg-audio-failed", + chat_id: "oc-dm", + chat_type: "p2p", + message_type: "audio", + content: JSON.stringify({ + file_key: "expired-audio", + speech_to_text: "spoken words", + }), + }, + }, + }); + + const context = mockCallArg<{ + BodyForAgent?: string; + CommandBody?: string; + MediaPath?: string; + RawBody?: string; + }>(mockFinalizeInboundContext, 0, 0); + expect(context.RawBody).toBe("spoken words"); + expect(context.CommandBody).toBe("spoken words"); + expect(context.BodyForAgent).toContain("spoken words\n\n[feishu attachment unavailable]"); + expect(context.MediaPath).toBeUndefined(); + }); + + it("preserves a filename without a phantom placeholder when a file download fails", async () => { + mockShouldComputeCommandAuthorized.mockReturnValue(false); + mockDownloadMessageResourceFeishu.mockRejectedValueOnce(new Error("expired file key")); + + await dispatchMessage({ + cfg: { + channels: { feishu: { dmPolicy: "open" } }, + } as ClawdbotConfig, + event: { + sender: { sender_id: { open_id: "ou-sender" } }, + message: { + message_id: "msg-file-failed", + chat_id: "oc-dm", + chat_type: "p2p", + message_type: "file", + content: JSON.stringify({ file_key: "expired-file", file_name: "q1.pdf" }), + }, + }, + }); + + const context = mockCallArg<{ + BodyForAgent?: string; + CommandBody?: string; + MediaPath?: string; + RawBody?: string; + }>(mockFinalizeInboundContext, 0, 0); + expect(context.RawBody).toBe(" (q1.pdf)"); + expect(context.CommandBody).toBe(" (q1.pdf)"); + expect(context.BodyForAgent).toContain("q1.pdf\n\n[feishu attachment unavailable]"); + expect(context.BodyForAgent).not.toContain(""); + expect(context.MediaPath).toBeUndefined(); + }); + it("drops group image message when groupPolicy is open but requireMention is explicitly true", async () => { mockShouldComputeCommandAuthorized.mockReturnValue(false); @@ -2750,6 +2851,48 @@ describe("handleFeishuMessage command authorization", () => { expect(typeof mockCallArg(mockSaveMediaBuffer, 0, 3)).toBe("number"); }); + it("removes failed rich-post media markers while preserving post text", async () => { + mockShouldComputeCommandAuthorized.mockReturnValue(false); + mockDownloadMessageResourceFeishu.mockRejectedValueOnce(new Error("expired image key")); + + await dispatchMessage({ + cfg: { + channels: { feishu: { dmPolicy: "open" } }, + } as ClawdbotConfig, + event: { + sender: { sender_id: { open_id: "ou-sender" } }, + message: { + message_id: "msg-post-image-failed", + chat_id: "oc-dm", + chat_type: "p2p", + message_type: "post", + content: JSON.stringify({ + title: "Rich text", + content: [ + [ + { tag: "text", text: "Before " }, + { tag: "img", image_key: "expired-image" }, + { tag: "text", text: " after" }, + ], + ], + }), + }, + }, + }); + + const context = mockCallArg<{ + BodyForAgent?: string; + MediaPath?: string; + RawBody?: string; + }>(mockFinalizeInboundContext, 0, 0); + expect(context.RawBody).toBe("Rich text\n\nBefore ![image] after"); + expect(context.BodyForAgent).toContain( + "Rich text\n\nBefore after\n\n[feishu attachment unavailable]", + ); + expect(context.BodyForAgent).not.toContain("![image]"); + expect(context.MediaPath).toBeUndefined(); + }); + it("includes message_id in BodyForAgent on its own line", async () => { mockShouldComputeCommandAuthorized.mockReturnValue(false); diff --git a/extensions/feishu/src/bot.ts b/extensions/feishu/src/bot.ts index 424bb5fc8ec..8f0b48ec469 100644 --- a/extensions/feishu/src/bot.ts +++ b/extensions/feishu/src/bot.ts @@ -1,6 +1,7 @@ // Feishu plugin module implements bot behavior. import { buildChannelInboundEventContext, + formatInboundMediaUnavailableText, toInboundMediaFacts, } from "openclaw/plugin-sdk/channel-inbound"; import { resolveAgentOutboundIdentity } from "openclaw/plugin-sdk/channel-outbound"; @@ -38,6 +39,7 @@ import { parseMessageContent, resolveFeishuGroupSession, resolveFeishuMediaList, + resolveFeishuMediaFailurePresentation, } from "./bot-content.js"; import { evaluateSupplementalContextVisibility, @@ -1018,7 +1020,7 @@ export async function handleFeishuMessage(params: { // Resolve media from message const mediaMaxBytes = (feishuCfg?.mediaMaxMb ?? 30) * 1024 * 1024; // 30MB default - const mediaList = await resolveFeishuMediaList({ + const mediaResolution = await resolveFeishuMediaList({ cfg, messageId: ctx.messageId, messageType: event.message.message_type, @@ -1027,6 +1029,19 @@ export async function handleFeishuMessage(params: { log, accountId: account.accountId, }); + const mediaList = mediaResolution.media; + const mediaFailurePresentation = resolveFeishuMediaFailurePresentation( + event.message.content, + event.message.message_type, + ); + const mediaFailureContent = + mediaResolution.unavailableCount > 0 + ? formatInboundMediaUnavailableText({ + body: mediaFailurePresentation.unavailableBody ?? ctx.content, + mediaPlaceholder: mediaFailurePresentation.mediaPlaceholder, + notice: `[feishu ${mediaResolution.unavailableCount > 1 ? `${mediaResolution.unavailableCount} attachments` : "attachment"} unavailable]`, + }) + : ctx.content; // Fetch quoted/replied message content before the empty-message guard // so a reply with only @bot (no text, no media) is not dropped when // the quoted message carries meaningful content. @@ -1075,7 +1090,7 @@ export async function handleFeishuMessage(params: { // be empty" errors. Logging the skip avoids silent loss without polluting // the agent session. Quoted content is checked too so a reply-only @bot // with quoted context is not dropped. - if (!ctx.content.trim() && mediaList.length === 0 && !quotedContent?.trim()) { + if (!mediaFailureContent.trim() && mediaList.length === 0 && !quotedContent?.trim()) { log( `feishu[${account.accountId}]: skipping empty message (no text, no media, no quoted) from ${ctx.senderOpenId}`, ); @@ -1096,13 +1111,14 @@ export async function handleFeishuMessage(params: { const inboundMedia = toInboundMediaFacts(mediaList, { transcribed: (_media, index) => index === preflightAudioIndex, }); - const agentFacingContent = audioTranscript ?? ctx.content; + const agentFacingContent = audioTranscript ?? mediaFailureContent; + const commandFacingContent = audioTranscript ?? ctx.content; const agentFacingCtx = - audioTranscript === undefined + agentFacingContent === ctx.content ? ctx : { ...ctx, - content: audioTranscript, + content: agentFacingContent, }; const effectiveCommandProbeBody = audioTranscript === undefined @@ -1434,8 +1450,8 @@ export async function handleFeishuMessage(params: { body: combinedBody, bodyForAgent: messageBody, inboundHistory, - rawBody: agentFacingContent, - commandBody: agentFacingContent, + rawBody: commandFacingContent, + commandBody: commandFacingContent, }, access: { mentions: { diff --git a/extensions/feishu/src/post.test.ts b/extensions/feishu/src/post.test.ts index ccd974c1610..5b40c9bf694 100644 --- a/extensions/feishu/src/post.test.ts +++ b/extensions/feishu/src/post.test.ts @@ -72,6 +72,10 @@ describe("parsePostContent", () => { expect(result.textContent).toBe("Before ![image] after\n![image]"); expect(result.imageKeys).toEqual(["img_1", "img_2"]); expect(result.mentionedOpenIds).toStrictEqual([]); + expect( + parsePostContent(content, { renderMediaPlaceholders: false, emptyTextFallback: "" }) + .textContent, + ).toBe("Before after"); }); it("supports locale wrappers", () => { diff --git a/extensions/feishu/src/post.ts b/extensions/feishu/src/post.ts index ad102ecbbb0..0d879e65788 100644 --- a/extensions/feishu/src/post.ts +++ b/extensions/feishu/src/post.ts @@ -130,6 +130,7 @@ function renderElement( imageKeys: string[], mediaKeys: Array<{ fileKey: string; fileName?: string }>, mentionedOpenIds: string[], + renderMediaPlaceholders: boolean, ): string { if (!isRecord(element)) { return escapeMarkdownText(toStringOrEmpty(element)); @@ -155,7 +156,7 @@ function renderElement( if (imageKey) { imageKeys.push(imageKey); } - return "![image]"; + return renderMediaPlaceholders ? "![image]" : ""; } case "media": { const fileKey = normalizeFeishuExternalKey(toStringOrEmpty(element.file_key)); @@ -163,7 +164,7 @@ function renderElement( const fileName = toStringOrEmpty(element.file_name) || undefined; mediaKeys.push({ fileKey, fileName }); } - return "[media]"; + return renderMediaPlaceholders ? "[media]" : ""; } case "emotion": return renderEmotionElement(element); @@ -231,7 +232,10 @@ function resolvePostPayload(parsed: unknown): PostPayload | null { return resolveLocalePayload(parsed); } -export function parsePostContent(content: string): PostParseResult { +export function parsePostContent( + content: string, + options: { renderMediaPlaceholders?: boolean; emptyTextFallback?: string } = {}, +): PostParseResult { try { const parsed = JSON.parse(content); const payload = resolvePostPayload(parsed); @@ -255,7 +259,13 @@ export function parsePostContent(content: string): PostParseResult { } let renderedParagraph = ""; for (const element of paragraph) { - renderedParagraph += renderElement(element, imageKeys, mediaKeys, mentionedOpenIds); + renderedParagraph += renderElement( + element, + imageKeys, + mediaKeys, + mentionedOpenIds, + options.renderMediaPlaceholders !== false, + ); } paragraphs.push(renderedParagraph); } @@ -265,7 +275,7 @@ export function parsePostContent(content: string): PostParseResult { const textContent = [title, body].filter(Boolean).join("\n\n").trim(); return { - textContent: textContent || FALLBACK_POST_TEXT, + textContent: textContent || (options.emptyTextFallback ?? FALLBACK_POST_TEXT), imageKeys, mediaKeys, mentionedOpenIds, diff --git a/extensions/imessage/src/monitor.media-policy.test.ts b/extensions/imessage/src/monitor.media-policy.test.ts index a22ee2a7596..1ad2f219849 100644 --- a/extensions/imessage/src/monitor.media-policy.test.ts +++ b/extensions/imessage/src/monitor.media-policy.test.ts @@ -1,9 +1,13 @@ // Imessage tests cover monitor.media policy plugin behavior. import type { waitForTransportReady } from "openclaw/plugin-sdk/transport-ready-runtime"; -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import type { createIMessageRpcClient } from "./client.js"; import { monitorIMessageProvider } from "./monitor.js"; import type { stageIMessageAttachments } from "./monitor/media-staging.js"; +import { + formatIMessageInboundMediaBody, + resolveIMessageInboundMediaInput, +} from "./monitor/monitor-provider.js"; const waitForTransportReadyMock = vi.hoisted(() => vi.fn(async () => {}), @@ -52,8 +56,14 @@ vi.mock("./monitor/media-staging.js", () => ({ })); describe("iMessage monitor attachment policy", () => { + beforeEach(() => { + createIMessageRpcClientMock.mockReset(); + stageIMessageAttachmentsMock.mockReset(); + readChannelAllowFromStoreMock.mockReset().mockResolvedValue([]); + }); + it("does not stage local attachments for messages dropped by inbound policy", async () => { - stageIMessageAttachmentsMock.mockResolvedValue([]); + stageIMessageAttachmentsMock.mockResolvedValue({ attachments: [], unavailableCount: 0 }); readChannelAllowFromStoreMock.mockResolvedValue([]); const attachmentPath = "/Users/openclaw/Library/Messages/Attachments/AA/BB/photo.heic"; @@ -115,4 +125,60 @@ describe("iMessage monitor attachment policy", () => { await vi.waitFor(() => expect(readChannelAllowFromStoreMock).toHaveBeenCalled()); expect(stageIMessageAttachmentsMock).not.toHaveBeenCalled(); }); + + it("admits attachment-only messages that are marked missing", async () => { + const attachment = { + original_path: "/Users/openclaw/Library/Messages/Attachments/missing.heic", + mime_type: "image/heic", + missing: true, + }; + expect( + resolveIMessageInboundMediaInput({ + messageText: "", + attachments: [attachment], + effectiveAttachmentRoots: [], + }), + ).toEqual({ + bodyText: "", + mediaPlaceholder: "", + mediaCandidates: [attachment], + rawMediaAttachments: [], + }); + }); + + it("uses the first materialized attachment type when earlier media is unavailable", () => { + const missingImage = { + original_path: "/Users/openclaw/Library/Messages/Attachments/missing.heic", + mime_type: "image/heic", + missing: true, + }; + const availableDocument = { + original_path: "/Users/openclaw/Library/Messages/Attachments/report.pdf", + mime_type: "application/pdf", + missing: false, + }; + + expect( + resolveIMessageInboundMediaInput({ + messageText: "", + attachments: [missingImage, availableDocument], + effectiveAttachmentRoots: ["/Users/openclaw/Library/Messages/Attachments"], + }), + ).toMatchObject({ + bodyText: "", + mediaPlaceholder: "", + mediaCandidates: [missingImage, availableDocument], + rawMediaAttachments: [ + { path: availableDocument.original_path, contentType: "application/pdf" }, + ], + }); + expect( + formatIMessageInboundMediaBody({ + messageText: "", + optimisticPlaceholder: "", + mediaAttachments: [{ contentType: "application/pdf" }], + unavailableCount: 1, + }), + ).toBe("\n\n[imessage attachment unavailable]"); + }); }); diff --git a/extensions/imessage/src/monitor/inbound-processing.test.ts b/extensions/imessage/src/monitor/inbound-processing.test.ts index dafab5e35f8..54236f5a9d3 100644 --- a/extensions/imessage/src/monitor/inbound-processing.test.ts +++ b/extensions/imessage/src/monitor/inbound-processing.test.ts @@ -788,6 +788,61 @@ describe("buildIMessageInboundContext", () => { expect(ctxPayload.MessageSidFull).toBe("p:0/GUID-current"); }); + it("keeps generated media notices out of command input", async () => { + const decision = await resolveIMessageInboundDecision({ + cfg: {} as OpenClawConfig, + accountId: "default", + message: { + id: 12347, + guid: "p:0/GUID-media-failure", + sender: "+15555550123", + text: "/reset", + is_from_me: false, + is_group: false, + }, + opts: undefined, + messageText: "/reset", + bodyText: "/reset", + allowFrom: ["*"], + groupAllowFrom: [], + groupPolicy: "open", + dmPolicy: "open", + storeAllowFrom: [], + historyLimit: 0, + groupHistories: new Map(), + echoCache: undefined, + selfChatCache: undefined, + logVerbose: undefined, + }); + expect(decision.kind).toBe("dispatch"); + if (decision.kind !== "dispatch") { + return; + } + + const { ctxPayload } = await buildIMessageInboundContext({ + cfg: {} as OpenClawConfig, + decision: { + ...decision, + agentBodyText: "/reset\n\n[imessage attachment unavailable]", + }, + message: { + id: 12347, + guid: "p:0/GUID-media-failure", + sender: "+15555550123", + text: "/reset", + is_from_me: false, + is_group: false, + }, + historyLimit: 0, + groupHistories: new Map(), + }); + + expect(ctxPayload.RawBody).toBe("/reset"); + expect(ctxPayload.CommandBody).toBe("/reset"); + expect(ctxPayload.BodyForAgent).toBe("/reset\n\n[imessage attachment unavailable]"); + expect(ctxPayload.Body).toContain("/reset\n\n[imessage attachment unavailable]"); + }); + it("prepends direct-message history when supplied", async () => { const decision = await resolveIMessageInboundDecision({ cfg: {} as OpenClawConfig, diff --git a/extensions/imessage/src/monitor/inbound-processing.ts b/extensions/imessage/src/monitor/inbound-processing.ts index dd34d0e5a8a..41dcb98bda4 100644 --- a/extensions/imessage/src/monitor/inbound-processing.ts +++ b/extensions/imessage/src/monitor/inbound-processing.ts @@ -351,6 +351,7 @@ type IMessageInboundDispatchDecision = { senderNormalized: string; route: ReturnType; bodyText: string; + agentBodyText?: string; createdAt?: number; replyContext: IMessageReplyContext | null; effectiveWasMentioned: boolean; @@ -930,7 +931,7 @@ export async function buildIMessageInboundContext(params: { channel: "iMessage", from: fromLabel, timestamp: decision.createdAt, - body: `${decision.bodyText}${replySuffix}`, + body: `${decision.agentBodyText ?? decision.bodyText}${replySuffix}`, chatType: decision.isGroup ? "group" : "direct", sender: { name: decision.senderNormalized, id: decision.sender }, previousTimestamp: params.previousTimestamp, @@ -1028,7 +1029,7 @@ export async function buildIMessageInboundContext(params: { }, message: { body: combinedBody, - bodyForAgent: decision.bodyText, + bodyForAgent: decision.agentBodyText ?? decision.bodyText, inboundHistory, rawBody: decision.bodyText, commandBody: decision.bodyText, diff --git a/extensions/imessage/src/monitor/media-staging.test.ts b/extensions/imessage/src/monitor/media-staging.test.ts index f24bd7c7afd..75bb9c2c562 100644 --- a/extensions/imessage/src/monitor/media-staging.test.ts +++ b/extensions/imessage/src/monitor/media-staging.test.ts @@ -36,7 +36,10 @@ describe("stageIMessageAttachments", () => { [{ original_path: sourcePath, mime_type: "image/png", missing: false }], { maxBytes: 1024, allowedRoots: [tempDir], deps: { saveMediaBuffer } }, ), - ).resolves.toEqual([{ path: "/state/media/inbound/saved.png", contentType: "image/png" }]); + ).resolves.toEqual({ + attachments: [{ path: "/state/media/inbound/saved.png", contentType: "image/png" }], + unavailableCount: 0, + }); expect(saveMediaBuffer).toHaveBeenCalledWith( Buffer.from("png-bytes"), @@ -70,7 +73,7 @@ describe("stageIMessageAttachments", () => { ], { maxBytes: 1024, allowedRoots: [allowedRoot], deps: { saveMediaBuffer, logVerbose } }, ), - ).resolves.toEqual([]); + ).resolves.toEqual({ attachments: [], unavailableCount: 1 }); expect(saveMediaBuffer).not.toHaveBeenCalled(); expect(logVerbose).toHaveBeenCalledWith( @@ -113,9 +116,18 @@ describe("stageIMessageAttachments", () => { [{ original_path: sourcePath, mime_type: "image/png", missing: false }], { maxBytes: 4, deps: { saveMediaBuffer, logVerbose } }, ), - ).resolves.toEqual([]); + ).resolves.toEqual({ attachments: [], unavailableCount: 1 }); expect(saveMediaBuffer).not.toHaveBeenCalled(); expect(logVerbose).toHaveBeenCalledWith(expect.stringContaining("failed to stage")); }); + + it("counts attachments already marked missing", async () => { + await expect( + stageIMessageAttachments( + [{ original_path: "/tmp/missing.heic", mime_type: "image/heic", missing: true }], + { maxBytes: 1024, deps: { saveMediaBuffer: vi.fn() } }, + ), + ).resolves.toEqual({ attachments: [], unavailableCount: 1 }); + }); }); diff --git a/extensions/imessage/src/monitor/media-staging.ts b/extensions/imessage/src/monitor/media-staging.ts index f33a579ee43..125998bd014 100644 --- a/extensions/imessage/src/monitor/media-staging.ts +++ b/extensions/imessage/src/monitor/media-staging.ts @@ -11,6 +11,11 @@ export type StagedIMessageAttachment = { contentType?: string; }; +export type StagedIMessageAttachments = { + attachments: StagedIMessageAttachment[]; + unavailableCount: number; +}; + type SaveMediaBufferImpl = typeof saveMediaBuffer; type StageIMessageAttachmentsDeps = { @@ -135,14 +140,16 @@ export async function stageIMessageAttachments( allowedRoots?: readonly string[]; deps?: StageIMessageAttachmentsDeps; }, -): Promise { +): Promise { const deps = params.deps ?? {}; const save = deps.saveMediaBuffer ?? saveMediaBuffer; const staged: StagedIMessageAttachment[] = []; + let unavailableCount = 0; for (const attachment of attachments) { const attachmentPath = attachment.original_path?.trim(); if (!attachmentPath || attachment.missing) { + unavailableCount += 1; continue; } @@ -163,9 +170,10 @@ export async function stageIMessageAttachments( ); staged.push({ path: saved.path, contentType: saved.contentType }); } catch (err) { + unavailableCount += 1; deps.logVerbose?.(`imessage: failed to stage inbound attachment: ${String(err)}`); } } - return staged; + return { attachments: staged, unavailableCount }; } diff --git a/extensions/imessage/src/monitor/monitor-provider.ts b/extensions/imessage/src/monitor/monitor-provider.ts index 6a30df89499..d969bd7001e 100644 --- a/extensions/imessage/src/monitor/monitor-provider.ts +++ b/extensions/imessage/src/monitor/monitor-provider.ts @@ -7,6 +7,7 @@ import { CHANNEL_APPROVAL_NATIVE_RUNTIME_CONTEXT_CAPABILITY } from "openclaw/plu import { logTypingFailure } from "openclaw/plugin-sdk/channel-feedback"; import { createChannelInboundDebouncer, + formatInboundMediaUnavailableText, resolveEnvelopeFormatOptions, runChannelInboundEvent, shouldDebounceTextInbound, @@ -109,7 +110,7 @@ import { enqueueIMessageReactionSystemEvent } from "./reaction-system-event.js"; import { advanceIMessageRecoveryCursor, loadIMessageRecoveryCursor } from "./recovery-cursor.js"; import { normalizeAllowList, resolveRuntime } from "./runtime.js"; import { createSelfChatCache } from "./self-chat-cache.js"; -import type { IMessagePayload, MonitorIMessageOpts } from "./types.js"; +import type { IMessageAttachment, IMessagePayload, MonitorIMessageOpts } from "./types.js"; import { sanitizeIMessageWatchErrorPayload } from "./watch-error-log.js"; const WATCH_SUBSCRIBE_MAX_ATTEMPTS = 3; @@ -158,6 +159,69 @@ function isIMessagePluginPayloadAttachment(attachment: { ); } +export function resolveIMessageInboundMediaInput(params: { + messageText: string; + attachments: IMessageAttachment[]; + effectiveAttachmentRoots: readonly string[]; + logVerbose?: (message: string) => void; +}) { + // Apple rich-link previews are opaque plugin payloads; the useful URL stays + // in message text. Treating them as media creates phantom attachments and + // keeps split-send URL previews out of the text debounce path. + const mediaCandidates = params.attachments.filter( + (entry) => !isIMessagePluginPayloadAttachment(entry), + ); + const rawMediaAttachments = mediaCandidates.flatMap((attachment) => { + const attachmentPath = attachment.original_path?.trim(); + if (!attachmentPath || attachment.missing) { + return []; + } + if ( + !isInboundPathAllowed({ filePath: attachmentPath, roots: params.effectiveAttachmentRoots }) + ) { + params.logVerbose?.( + `imessage: dropping inbound attachment outside allowed roots: ${attachmentPath}`, + ); + return []; + } + return [{ path: attachmentPath, contentType: attachment.mime_type ?? undefined }]; + }); + const kind = kindFromMime( + rawMediaAttachments[0]?.contentType ?? mediaCandidates[0]?.mime_type ?? undefined, + ); + const mediaPlaceholder = kind + ? `` + : mediaCandidates.length + ? "" + : ""; + return { + bodyText: params.messageText || mediaPlaceholder, + mediaPlaceholder, + mediaCandidates, + rawMediaAttachments, + }; +} + +export function formatIMessageInboundMediaBody(params: { + messageText: string; + optimisticPlaceholder: string; + mediaAttachments: Array<{ contentType?: string }>; + unavailableCount: number; +}): string { + const materializedKind = kindFromMime(params.mediaAttachments[0]?.contentType); + const materializedPlaceholder = materializedKind + ? `` + : params.mediaAttachments.length > 0 + ? "" + : ""; + return formatInboundMediaUnavailableText({ + body: params.messageText || materializedPlaceholder || params.optimisticPlaceholder, + mediaPlaceholder: + params.mediaAttachments.length === 0 ? params.optimisticPlaceholder : undefined, + notice: `[imessage ${params.unavailableCount > 1 ? `${params.unavailableCount} attachments` : "attachment"} unavailable]`, + }); +} + async function detectRemoteHostFromCliPath(cliPath: string): Promise { try { const expanded = cliPath.startsWith("~") @@ -845,42 +909,15 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P const messageText = (pollBody ?? message.text ?? "").trim(); const attachments = includeAttachments ? (message.attachments ?? []) : []; const effectiveAttachmentRoots = remoteHost ? remoteAttachmentRoots : attachmentRoots; - const validAttachments = attachments.filter((entry) => { - if (isIMessagePluginPayloadAttachment(entry)) { - // Apple rich-link previews arrive as opaque .pluginPayloadAttachment - // files. The useful URL remains in message.text/attributedBody; treating - // the preview blob as media creates noisy phantom attachments and can - // keep split-send URL previews out of the text debounce path. - return false; - } - const attachmentPath = entry?.original_path?.trim(); - if (!attachmentPath || entry?.missing) { - return false; - } - if (isInboundPathAllowed({ filePath: attachmentPath, roots: effectiveAttachmentRoots })) { - return true; - } - logVerbose(`imessage: dropping inbound attachment outside allowed roots: ${attachmentPath}`); - return false; + const mediaInput = resolveIMessageInboundMediaInput({ + messageText, + attachments, + effectiveAttachmentRoots, + logVerbose, }); - const rawMediaAttachments = validAttachments.flatMap((a) => { - const attachmentPath = a.original_path?.trim(); - return attachmentPath - ? [{ path: attachmentPath, contentType: a.mime_type ?? undefined }] - : []; - }); - const placeholderMediaType = rawMediaAttachments[0]?.contentType; - const kind = kindFromMime(placeholderMediaType ?? undefined); - const placeholder = kind - ? `` - : validAttachments.length - ? "" - : ""; return { messageText, - bodyText: messageText || placeholder, - validAttachments, - rawMediaAttachments, + ...mediaInput, effectiveAttachmentRoots, }; } @@ -913,7 +950,8 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P const { messageText, bodyText, - validAttachments, + mediaPlaceholder, + mediaCandidates, rawMediaAttachments, effectiveAttachmentRoots, } = resolveIMessageInboundBodyText(message); @@ -1150,20 +1188,36 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P }); }; } - const stagedAttachments = remoteHost - ? [] - : await stageIMessageAttachments(validAttachments, { + const staged = remoteHost + ? { + attachments: rawMediaAttachments, + unavailableCount: mediaCandidates.length - rawMediaAttachments.length, + } + : await stageIMessageAttachments(mediaCandidates, { maxBytes: mediaMaxBytes, allowedRoots: effectiveAttachmentRoots, deps: { logVerbose }, }); - const mediaAttachments = remoteHost ? rawMediaAttachments : stagedAttachments; + const mediaAttachments = staged.attachments; const firstAttachment = mediaAttachments[0]; const mediaPath = firstAttachment?.path ?? undefined; const mediaType = firstAttachment?.contentType ?? undefined; // Build arrays for all attachments (for multi-image support) const mediaPaths = mediaAttachments.map((a) => a.path).filter(Boolean); const mediaTypes = mediaAttachments.map((a) => a.contentType ?? undefined); + const unavailableCount = staged.unavailableCount; + const contextDecision = + unavailableCount > 0 + ? { + ...decision, + agentBodyText: formatIMessageInboundMediaBody({ + messageText, + optimisticPlaceholder: mediaPlaceholder, + mediaAttachments, + unavailableCount, + }), + } + : decision; const previousTimestamp = readSessionUpdatedAt({ storePath, sessionKey: decision.route.sessionKey, @@ -1188,7 +1242,7 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P : undefined; const { ctxPayload, chatTarget, imessageTo } = await buildIMessageInboundContext({ cfg, - decision, + decision: contextDecision, message, previousTimestamp, remoteHost, diff --git a/extensions/line/src/bot-handlers.test.ts b/extensions/line/src/bot-handlers.test.ts index 2f30051585a..dc5b1285a4b 100644 --- a/extensions/line/src/bot-handlers.test.ts +++ b/extensions/line/src/bot-handlers.test.ts @@ -1051,6 +1051,31 @@ describe("handleLineWebhookEvents", () => { expect(processMessage).toHaveBeenCalledTimes(1); }); + it("reports failed media materialization to the message-context owner", async () => { + downloadLineMediaMock.mockRejectedValueOnce(new Error("expired content")); + const processMessage = vi.fn(); + const event = createTestMessageEvent({ + message: { + id: "image-failed-1", + type: "image", + contentProvider: { type: "line" }, + quoteToken: "q-image-failed", + }, + source: { type: "user", userId: "user-image-failed" }, + webhookEventId: "evt-image-failed", + }); + + await handleLineWebhookEvents( + [event], + createLineWebhookTestContext({ processMessage, dmPolicy: "open" }), + ); + + expect(buildLineMessageContextMock).toHaveBeenCalledWith( + expect.objectContaining({ allMedia: [], mediaUnavailable: true }), + ); + expect(processMessage).toHaveBeenCalledTimes(1); + }); + it("allows non-text group messages through when requireMention is set (cannot detect mention)", async () => { // Image message -- LINE only carries mention metadata on text messages. const event = createTestMessageEvent({ diff --git a/extensions/line/src/bot-handlers.ts b/extensions/line/src/bot-handlers.ts index aad6e1d1479..45a2f1d0859 100644 --- a/extensions/line/src/bot-handlers.ts +++ b/extensions/line/src/bot-handlers.ts @@ -467,6 +467,7 @@ async function handleMessageEvent(event: MessageEvent, context: LineHandlerConte } const allMedia: MediaRef[] = []; + let mediaUnavailable = false; if (isDownloadableLineMessageType(message.type)) { try { @@ -480,6 +481,7 @@ async function handleMessageEvent(event: MessageEvent, context: LineHandlerConte contentType: media.contentType, }); } catch (err) { + mediaUnavailable = true; const errMsg = String(err); if (errMsg.includes("exceeds") && errMsg.includes("limit")) { logVerbose(`line: media exceeds size limit for message ${message.id}`); @@ -492,6 +494,7 @@ async function handleMessageEvent(event: MessageEvent, context: LineHandlerConte const messageContext = await buildLineMessageContext({ event, allMedia, + mediaUnavailable, cfg, account, commandAuthorized: decision.commandAccess.authorized, diff --git a/extensions/line/src/bot-message-context.test.ts b/extensions/line/src/bot-message-context.test.ts index f5cc35fd47b..114e6bfc238 100644 --- a/extensions/line/src/bot-message-context.test.ts +++ b/extensions/line/src/bot-message-context.test.ts @@ -113,6 +113,30 @@ describe("buildLineMessageContext", () => { expect(context?.ctxPayload.To).toBe("line:group:group-1"); }); + it("replaces a failed media placeholder with an unavailable notice", async () => { + const event = createMessageEvent({ type: "user", userId: "user-image" }, { + message: { + id: "image-1", + type: "image", + contentProvider: { type: "line" }, + }, + } as Partial); + + const context = await buildLineMessageContext({ + event, + allMedia: [], + mediaUnavailable: true, + cfg, + account, + commandAuthorized: true, + }); + + expect(context?.ctxPayload.RawBody).toBe(""); + expect(context?.ctxPayload.CommandBody).toBe(""); + expect(context?.ctxPayload.BodyForAgent).toBe("[line attachment unavailable]"); + expect(context?.ctxPayload.MediaPath).toBeUndefined(); + }); + it("routes group postback replies to the group id", async () => { const event = createPostbackEvent({ type: "group", groupId: "group-2", userId: "user-2" }); diff --git a/extensions/line/src/bot-message-context.ts b/extensions/line/src/bot-message-context.ts index 354c71c0d41..2992e1d5936 100644 --- a/extensions/line/src/bot-message-context.ts +++ b/extensions/line/src/bot-message-context.ts @@ -2,6 +2,7 @@ import type { webhook } from "@line/bot-sdk"; import { recordChannelActivity } from "openclaw/plugin-sdk/channel-activity-runtime"; import { + formatInboundMediaUnavailableText, formatInboundEnvelope, formatLocationText, resolveInboundSessionEnvelopeContext, @@ -36,6 +37,7 @@ interface MediaRef { interface BuildLineMessageContextParams { event: MessageEvent; allMedia: MediaRef[]; + mediaUnavailable?: boolean; cfg: OpenClawConfig; account: ResolvedLineAccount; commandAuthorized: boolean; @@ -282,6 +284,7 @@ async function finalizeLineInboundContext(params: { route: LineRouteInfo; source: LineSourceInfoWithPeerId; rawBody: string; + agentBody?: string; timestamp: number; messageSid: string; commandAuthorized: boolean; @@ -318,11 +321,12 @@ async function finalizeLineInboundContext(params: { sessionKey: params.route.sessionKey, }); + const agentBody = params.agentBody ?? params.rawBody; const body = formatInboundEnvelope({ channel: "LINE", from: conversationLabel, timestamp: params.timestamp, - body: params.rawBody, + body: agentBody, chatType: params.source.isGroup ? "group" : "direct", sender: { id: senderId, @@ -333,7 +337,7 @@ async function finalizeLineInboundContext(params: { const ctxPayload = finalizeInboundContext({ Body: body, - BodyForAgent: params.rawBody, + BodyForAgent: agentBody, RawBody: params.rawBody, CommandBody: params.rawBody, From: fromAddress, @@ -437,7 +441,16 @@ async function finalizeLineInboundContext(params: { } export async function buildLineMessageContext(params: BuildLineMessageContextParams) { - const { event, allMedia, cfg, account, commandAuthorized, groupHistories, historyLimit } = params; + const { + event, + allMedia, + mediaUnavailable, + cfg, + account, + commandAuthorized, + groupHistories, + historyLimit, + } = params; const source = event.source; const { userId, groupId, roomId, isGroup, peerId, route } = await resolveLineInboundRoute({ @@ -457,8 +470,15 @@ export async function buildLineMessageContext(params: BuildLineMessageContextPar if (!rawBody && allMedia.length > 0) { rawBody = `${allMedia.length > 1 ? ` (${allMedia.length} images)` : ""}`; } + const agentBody = mediaUnavailable + ? formatInboundMediaUnavailableText({ + body: rawBody, + mediaPlaceholder: placeholder, + notice: "[line attachment unavailable]", + }) + : rawBody; - if (!rawBody && allMedia.length === 0) { + if (!agentBody && allMedia.length === 0) { return null; } @@ -489,6 +509,7 @@ export async function buildLineMessageContext(params: BuildLineMessageContextPar route, source: { userId, groupId, roomId, isGroup, peerId }, rawBody, + agentBody, timestamp, messageSid: messageId, commandAuthorized, diff --git a/extensions/mattermost/src/mattermost/monitor-resources.test.ts b/extensions/mattermost/src/mattermost/monitor-resources.test.ts index 51d3ab4db53..de9033badf1 100644 --- a/extensions/mattermost/src/mattermost/monitor-resources.test.ts +++ b/extensions/mattermost/src/mattermost/monitor-resources.test.ts @@ -20,9 +20,33 @@ vi.mock("./interactions.js", () => ({ describe("mattermost monitor resources", () => { let createMattermostMonitorResources: typeof import("./monitor-resources.js").createMattermostMonitorResources; + let formatMattermostInboundMediaText: typeof import("./monitor-resources.js").formatMattermostInboundMediaText; beforeAll(async () => { - ({ createMattermostMonitorResources } = await import("./monitor-resources.js")); + ({ createMattermostMonitorResources, formatMattermostInboundMediaText } = + await import("./monitor-resources.js")); + }); + + it("keeps media-only download failures visible to the agent", () => { + expect( + formatMattermostInboundMediaText({ + body: "", + mediaPlaceholder: "", + expectedCount: 1, + mediaCount: 0, + }), + ).toBe("[mattermost attachment unavailable]"); + }); + + it("preserves successful media placeholders on partial failures", () => { + expect( + formatMattermostInboundMediaText({ + body: " (2 files)", + mediaPlaceholder: " (2 files)", + expectedCount: 2, + mediaCount: 1, + }), + ).toBe(" (2 files)\n\n[mattermost attachment unavailable]"); }); beforeEach(() => { diff --git a/extensions/mattermost/src/mattermost/monitor-resources.ts b/extensions/mattermost/src/mattermost/monitor-resources.ts index a65979512bb..4f67f8c1d74 100644 --- a/extensions/mattermost/src/mattermost/monitor-resources.ts +++ b/extensions/mattermost/src/mattermost/monitor-resources.ts @@ -1,4 +1,5 @@ // Mattermost plugin module implements monitor resources behavior. +import { formatInboundMediaUnavailableText } from "openclaw/plugin-sdk/channel-inbound"; import { asDateTimestampMs, resolveExpiresAtMsFromDurationMs, @@ -23,6 +24,23 @@ export type MattermostMediaInfo = { kind: MattermostMediaKind; }; +export function formatMattermostInboundMediaText(params: { + body: string; + mediaPlaceholder: string; + expectedCount: number; + mediaCount: number; +}): string { + const unavailableCount = Math.max(0, params.expectedCount - params.mediaCount); + if (unavailableCount === 0) { + return params.body; + } + return formatInboundMediaUnavailableText({ + body: params.body, + mediaPlaceholder: params.mediaCount === 0 ? params.mediaPlaceholder : undefined, + notice: `[mattermost ${unavailableCount > 1 ? `${unavailableCount} attachments` : "attachment"} unavailable]`, + }); +} + const CHANNEL_CACHE_TTL_MS = 5 * 60_000; const USER_CACHE_TTL_MS = 10 * 60_000; diff --git a/extensions/mattermost/src/mattermost/monitor.ts b/extensions/mattermost/src/mattermost/monitor.ts index 2eb628e7c80..2ac9530d4ac 100644 --- a/extensions/mattermost/src/mattermost/monitor.ts +++ b/extensions/mattermost/src/mattermost/monitor.ts @@ -73,7 +73,11 @@ import { shouldDropEmptyMattermostBody, } from "./monitor-helpers.js"; import { resolveOncharPrefixes, stripOncharPrefix } from "./monitor-onchar.js"; -import { createMattermostMonitorResources, type MattermostMediaInfo } from "./monitor-resources.js"; +import { + createMattermostMonitorResources, + formatMattermostInboundMediaText, + type MattermostMediaInfo, +} from "./monitor-resources.js"; import { registerMattermostMonitorSlashCommands } from "./monitor-slash.js"; import { createMattermostConnectOnce, @@ -1538,10 +1542,17 @@ export async function monitorMattermostProvider(opts: MonitorMattermostOpts = {} recordPendingHistory(); return; } - const mediaList = await resolveMattermostMedia(post.file_ids); + const fileIds = uniqueStrings(normalizeTrimmedStringList(post.file_ids ?? [])); + const mediaList = await resolveMattermostMedia(fileIds); const mediaPlaceholder = buildMattermostAttachmentPlaceholder(mediaList); const bodySource = oncharTriggered ? oncharResult.stripped : rawText; - const baseText = [bodySource, mediaPlaceholder].filter(Boolean).join("\n").trim(); + const downloadedText = [bodySource, mediaPlaceholder].filter(Boolean).join("\n").trim(); + const baseText = formatMattermostInboundMediaText({ + body: downloadedText, + mediaPlaceholder, + expectedCount: fileIds.length, + mediaCount: mediaList.length, + }); const bodyText = normalizeMention(baseText, botUsername); if (shouldDropEmptyMattermostBody({ bodyText, rawText: rawPostText, botUsername })) { logVerboseMessage( @@ -1612,7 +1623,7 @@ export async function monitorMattermostProvider(opts: MonitorMattermostOpts = {} Body: combinedBody, BodyForAgent: bodyForAgent, InboundHistory: inboundHistory, - RawBody: bodyText, + RawBody: commandBody, CommandBody: commandBody, BodyForCommands: commandBody, From: diff --git a/extensions/msteams/src/attachments.helpers.test.ts b/extensions/msteams/src/attachments.helpers.test.ts index 323be29003e..6d2e40a092c 100644 --- a/extensions/msteams/src/attachments.helpers.test.ts +++ b/extensions/msteams/src/attachments.helpers.test.ts @@ -5,6 +5,7 @@ import { buildMSTeamsAttachmentPlaceholder, buildMSTeamsGraphMessageUrls, buildMSTeamsMediaPayload, + resolveMSTeamsInboundAttachmentPresentation, } from "./attachments.js"; import { setMSTeamsRuntime } from "./runtime.js"; @@ -199,6 +200,65 @@ describe("msteams attachment helpers", () => { }), ).toBe(""); }); + + it("counts advertised files without URLs and ignores mention-only HTML", () => { + expect( + resolveMSTeamsInboundAttachmentPresentation([ + { contentType: "application/pdf", name: "report.pdf" }, + ]), + ).toEqual({ placeholder: "", expectedMediaCount: 1 }); + expect( + resolveMSTeamsInboundAttachmentPresentation([ + { contentType: "text/html", content: "
Bot hello
" }, + ]), + ).toEqual({ placeholder: "", expectedMediaCount: 0 }); + }); + + it("does not count HTML references separately from files or cards", () => { + expect( + resolveMSTeamsInboundAttachmentPresentation([ + createHtmlAttachment(''), + { + id: "file-1", + contentType: CONTENT_TYPE_APPLICATION_PDF, + contentUrl: TEST_URL_PDF, + }, + ]), + ).toEqual({ placeholder: "", expectedMediaCount: 1 }); + + expect( + resolveMSTeamsInboundAttachmentPresentation([ + createHtmlAttachment(''), + { + id: "card-1", + contentType: "application/vnd.microsoft.card.adaptive", + content: { type: "AdaptiveCard" }, + }, + ]), + ).toEqual({ placeholder: "", expectedMediaCount: 0 }); + }); + + it("counts repeated inline URLs once while keeping data images per occurrence", () => { + const repeatedUrl = "https://example.com/repeated.png"; + expect( + resolveMSTeamsInboundAttachmentPresentation([ + { + contentType: "text/html", + content: ``, + }, + ]), + ).toEqual({ placeholder: "", expectedMediaCount: 1 }); + + const dataUrl = "data:image/png;base64,AQ=="; + expect( + resolveMSTeamsInboundAttachmentPresentation([ + { + contentType: "text/html", + content: ``, + }, + ]), + ).toEqual({ placeholder: " (2 images)", expectedMediaCount: 2 }); + }); }); describe("buildMSTeamsGraphMessageUrls", () => { diff --git a/extensions/msteams/src/attachments.ts b/extensions/msteams/src/attachments.ts index af4ae162810..e4984f362f2 100644 --- a/extensions/msteams/src/attachments.ts +++ b/extensions/msteams/src/attachments.ts @@ -8,6 +8,7 @@ export { buildMSTeamsGraphMessageUrls, downloadMSTeamsGraphMedia } from "./attac export { buildMSTeamsAttachmentPlaceholder, extractMSTeamsHtmlAttachmentIds, + resolveMSTeamsInboundAttachmentPresentation, summarizeMSTeamsHtmlAttachments, } from "./attachments/html.js"; export { buildMSTeamsMediaPayload } from "./attachments/payload.js"; diff --git a/extensions/msteams/src/attachments/html.ts b/extensions/msteams/src/attachments/html.ts index f68f748fc64..3bdd6b84ba7 100644 --- a/extensions/msteams/src/attachments/html.ts +++ b/extensions/msteams/src/attachments/html.ts @@ -4,7 +4,9 @@ import { extractHtmlFromAttachment, extractInlineImageCandidates, IMG_SRC_RE, + isDownloadableAttachment, isLikelyImageAttachment, + normalizeContentType, safeHostForUrl, } from "./shared.js"; import type { MSTeamsAttachmentLike, MSTeamsHtmlAttachmentSummary } from "./types.js"; @@ -108,16 +110,111 @@ export function buildMSTeamsAttachmentPlaceholder( attachments: MSTeamsAttachmentLike[] | undefined, limits?: { maxInlineBytes?: number; maxInlineTotalBytes?: number }, ): string { + return resolveMSTeamsInboundAttachmentPresentation(attachments, limits).placeholder; +} + +function isAdvertisedFileAttachment(attachment: MSTeamsAttachmentLike): boolean { + const contentType = normalizeContentType(attachment.contentType) ?? ""; + if ( + contentType.startsWith("text/html") || + contentType.startsWith("application/vnd.microsoft.card.") || + contentType.startsWith("application/vnd.microsoft.teams.card.") + ) { + return false; + } + return Boolean( + isDownloadableAttachment(attachment) || + isLikelyImageAttachment(attachment) || + attachment.name?.trim() || + contentType, + ); +} + +function countDistinctInlineImages(attachments: MSTeamsAttachmentLike[]): number { + const seenReferences = new Set(); + let count = 0; + for (const attachment of attachments) { + const html = extractHtmlFromAttachment(attachment); + if (!html) { + continue; + } + IMG_SRC_RE.lastIndex = 0; + let match: RegExpExecArray | null = IMG_SRC_RE.exec(html); + while (match) { + const src = match[1]?.trim(); + if (src?.startsWith("data:")) { + count += 1; + } else if (src && !seenReferences.has(src)) { + seenReferences.add(src); + count += 1; + } + match = IMG_SRC_RE.exec(html); + } + } + return count; +} + +function countDistinctInlineCandidates( + attachments: MSTeamsAttachmentLike[], + limits?: { maxInlineBytes?: number; maxInlineTotalBytes?: number }, +): number { + const seenUrls = new Set(); + let dataCount = 0; + for (const candidate of extractInlineImageCandidates(attachments, limits)) { + if (candidate.kind === "data") { + dataCount += 1; + } else { + seenUrls.add(candidate.url); + } + } + return dataCount + seenUrls.size; +} + +function countUnrepresentedHtmlAttachmentIds(attachments: MSTeamsAttachmentLike[]): number { + const representedIds = new Set(); + for (const attachment of attachments) { + const contentType = normalizeContentType(attachment.contentType) ?? ""; + if (contentType.startsWith("text/html")) { + continue; + } + const id = attachment.id?.trim(); + if (id) { + representedIds.add(id); + } + } + return extractMSTeamsHtmlAttachmentIds(attachments).filter((id) => !representedIds.has(id)) + .length; +} + +export function resolveMSTeamsInboundAttachmentPresentation( + attachments: MSTeamsAttachmentLike[] | undefined, + limits?: { maxInlineBytes?: number; maxInlineTotalBytes?: number }, +): { placeholder: string; expectedMediaCount: number } { const list = Array.isArray(attachments) ? attachments : []; if (list.length === 0) { - return ""; + return { placeholder: "", expectedMediaCount: 0 }; } - const imageCount = list.filter(isLikelyImageAttachment).length; - const inlineCount = extractInlineImageCandidates(list, limits).length; - const totalImages = imageCount + inlineCount; + const fileAttachments = list.filter(isAdvertisedFileAttachment); + const inlinePlaceholderCount = countDistinctInlineCandidates(list, limits); + const inlineExpectedCount = countDistinctInlineImages(list); + // Teams HTML uses tags as references. A matching attachment + // entry is the same resource (and may be a card), so count only unmatched + // IDs that need the Graph/Bot Framework hosted-content fallback. + const htmlAttachmentCount = countUnrepresentedHtmlAttachmentIds(list); + const expectedMediaCount = fileAttachments.length + inlineExpectedCount + htmlAttachmentCount; + if (expectedMediaCount === 0) { + return { placeholder: "", expectedMediaCount: 0 }; + } + const totalImages = + fileAttachments.filter(isLikelyImageAttachment).length + inlinePlaceholderCount; if (totalImages > 0) { - return `${totalImages > 1 ? ` (${totalImages} images)` : ""}`; + return { + placeholder: `${totalImages > 1 ? ` (${totalImages} images)` : ""}`, + expectedMediaCount, + }; } - const count = list.length; - return `${count > 1 ? ` (${count} files)` : ""}`; + return { + placeholder: `${expectedMediaCount > 1 ? ` (${expectedMediaCount} files)` : ""}`, + expectedMediaCount, + }; } diff --git a/extensions/msteams/src/attachments/types.ts b/extensions/msteams/src/attachments/types.ts index 34626886d4b..96872b74d5c 100644 --- a/extensions/msteams/src/attachments/types.ts +++ b/extensions/msteams/src/attachments/types.ts @@ -1,5 +1,6 @@ // Msteams type declarations define plugin contracts. export type MSTeamsAttachmentLike = { + id?: string | null; contentType?: string | null; contentUrl?: string | null; name?: string | null; diff --git a/extensions/msteams/src/monitor-handler/inbound-media.test.ts b/extensions/msteams/src/monitor-handler/inbound-media.test.ts index b1e93343287..8fcc58e16df 100644 --- a/extensions/msteams/src/monitor-handler/inbound-media.test.ts +++ b/extensions/msteams/src/monitor-handler/inbound-media.test.ts @@ -24,7 +24,7 @@ import { downloadMSTeamsGraphMedia, extractMSTeamsHtmlAttachmentIds, } from "../attachments.js"; -import { resolveMSTeamsInboundMedia } from "./inbound-media.js"; +import { resolveMSTeamsInboundMedia, resolveMSTeamsInboundMediaBody } from "./inbound-media.js"; const baseParams = { maxBytes: 1024 * 1024, @@ -52,6 +52,39 @@ function firstBotFrameworkAttachmentCall() { } describe("resolveMSTeamsInboundMedia graph fallback trigger", () => { + it("replaces a failed attachment placeholder without marking mention-only HTML", () => { + expect( + resolveMSTeamsInboundMediaBody({ + body: "", + mediaPlaceholder: "", + materializedMediaPlaceholder: "", + expectedMediaCount: 1, + mediaCount: 0, + }), + ).toBe("[msteams attachment unavailable]"); + expect( + resolveMSTeamsInboundMediaBody({ + body: "hello", + mediaPlaceholder: "", + materializedMediaPlaceholder: "", + expectedMediaCount: 0, + mediaCount: 0, + }), + ).toBe("hello"); + }); + + it("preserves successful media while exposing partial download failures", () => { + expect( + resolveMSTeamsInboundMediaBody({ + body: " (2 files)", + mediaPlaceholder: " (2 files)", + materializedMediaPlaceholder: "", + expectedMediaCount: 2, + mediaCount: 1, + }), + ).toBe("\n\n[msteams attachment unavailable]"); + }); + it("triggers Graph fallback when HTML contains tags", async () => { vi.mocked(downloadMSTeamsAttachments).mockResolvedValue([]); vi.mocked(extractMSTeamsHtmlAttachmentIds).mockReturnValueOnce(["att-0"]); diff --git a/extensions/msteams/src/monitor-handler/inbound-media.ts b/extensions/msteams/src/monitor-handler/inbound-media.ts index 4e52d947ae8..8ee6034bdb0 100644 --- a/extensions/msteams/src/monitor-handler/inbound-media.ts +++ b/extensions/msteams/src/monitor-handler/inbound-media.ts @@ -1,4 +1,5 @@ // Msteams plugin module implements inbound media behavior. +import { formatInboundMediaUnavailableText } from "openclaw/plugin-sdk/channel-inbound"; import { buildMSTeamsGraphMessageUrls, downloadMSTeamsAttachments, @@ -19,6 +20,28 @@ type MSTeamsLogger = { error?: (message: string, meta?: Record) => void; }; +export function resolveMSTeamsInboundMediaBody(params: { + body: string; + mediaPlaceholder: string; + materializedMediaPlaceholder: string; + expectedMediaCount: number; + mediaCount: number; +}): string { + const unavailableCount = Math.max(0, params.expectedMediaCount - params.mediaCount); + if (unavailableCount === 0) { + return params.body; + } + const body = + params.mediaCount > 0 && params.body === params.mediaPlaceholder + ? params.materializedMediaPlaceholder + : params.body; + return formatInboundMediaUnavailableText({ + body, + mediaPlaceholder: params.mediaCount === 0 ? params.mediaPlaceholder : undefined, + notice: `[msteams ${unavailableCount > 1 ? `${unavailableCount} attachments` : "attachment"} unavailable]`, + }); +} + export async function resolveMSTeamsInboundMedia(params: { attachments: MSTeamsAttachmentLike[]; htmlSummary?: MSTeamsHtmlAttachmentSummary; diff --git a/extensions/msteams/src/monitor-handler/message-handler.ts b/extensions/msteams/src/monitor-handler/message-handler.ts index c002422fcd9..4d14a44bc4e 100644 --- a/extensions/msteams/src/monitor-handler/message-handler.ts +++ b/extensions/msteams/src/monitor-handler/message-handler.ts @@ -23,10 +23,10 @@ import { import { sliceUtf16Safe, truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; import { serializeMSTeamsAdaptiveCardActionValue } from "../adaptive-card-submit.js"; import { - buildMSTeamsAttachmentPlaceholder, buildMSTeamsMediaPayload, - type MSTeamsAttachmentLike, + resolveMSTeamsInboundAttachmentPresentation, summarizeMSTeamsHtmlAttachments, + type MSTeamsAttachmentLike, } from "../attachments.js"; import { isRecord } from "../attachments/shared.js"; import { tryNormalizeBotFrameworkServiceUrl } from "../bot-framework-service-url.js"; @@ -101,7 +101,7 @@ import { wasMSTeamsMessageSentWithPersistence, } from "../sent-message-cache.js"; import { resolveMSTeamsSenderAccess } from "./access.js"; -import { resolveMSTeamsInboundMedia } from "./inbound-media.js"; +import { resolveMSTeamsInboundMedia, resolveMSTeamsInboundMediaBody } from "./inbound-media.js"; import { resolveMSTeamsRouteSessionKey } from "./thread-session.js"; function formatMSTeamsSenderReason(params: { @@ -231,10 +231,11 @@ export function createMSTeamsMessageHandler(deps: MSTeamsMessageHandlerDeps) { const rawText = params.rawText; const text = params.text; const attachments = params.attachments; - const attachmentPlaceholder = buildMSTeamsAttachmentPlaceholder(attachments, { + const attachmentPresentation = resolveMSTeamsInboundAttachmentPresentation(attachments, { maxInlineBytes: mediaMaxBytes, maxInlineTotalBytes: mediaMaxBytes, }); + const attachmentPlaceholder = attachmentPresentation.placeholder; const rawBody = text || attachmentPlaceholder; const quoteInfo = extractMSTeamsQuoteInfo(attachments); let quoteSenderId: string | undefined; @@ -621,6 +622,16 @@ export function createMSTeamsMessageHandler(deps: MSTeamsMessageHandlerDeps) { }); const mediaPayload = buildMSTeamsMediaPayload(mediaList); + const materializedMediaPlaceholder = resolveMSTeamsInboundAttachmentPresentation( + mediaList.map((media) => ({ contentType: media.contentType, name: media.path })), + ).placeholder; + const agentBody = resolveMSTeamsInboundMediaBody({ + body: rawBody, + mediaPlaceholder: attachmentPlaceholder, + materializedMediaPlaceholder, + expectedMediaCount: attachmentPresentation.expectedMediaCount, + mediaCount: mediaList.length, + }); // Fetch thread history when the message is a reply inside a Teams channel thread. // This is a best-effort enhancement; errors are logged and do not block the reply. @@ -717,7 +728,7 @@ export function createMSTeamsMessageHandler(deps: MSTeamsMessageHandlerDeps) { timestamp, previousTimestamp, envelope: envelopeOptions, - body: rawBody, + body: agentBody, }); let combinedBody = body; const isRoomish = !isDirectMessage; @@ -760,8 +771,8 @@ export function createMSTeamsMessageHandler(deps: MSTeamsMessageHandlerDeps) { : true; // Prepend thread history to the agent body so the agent has full thread context. const bodyForAgent = threadContext - ? `[Thread history]\n${threadContext}\n[/Thread history]\n\n${rawBody}` - : rawBody; + ? `[Thread history]\n${threadContext}\n[/Thread history]\n\n${agentBody}` + : agentBody; // For Teams *channel* messages (not group chats / DMs), preserve the // `teamId/channelId` pair on NativeChannelId so downstream action handlers diff --git a/extensions/signal/src/monitor/event-handler.inbound-context.test.ts b/extensions/signal/src/monitor/event-handler.inbound-context.test.ts index dfbaa6d1cc1..b34282ef9ac 100644 --- a/extensions/signal/src/monitor/event-handler.inbound-context.test.ts +++ b/extensions/signal/src/monitor/event-handler.inbound-context.test.ts @@ -1785,6 +1785,114 @@ describe("signal createSignalEventHandler inbound context", () => { expect(context.MediaTypes).toEqual(["image/jpeg", "application/octet-stream"]); }); + it("marks failed attachment downloads unavailable without a phantom media placeholder", async () => { + const handler = createSignalEventHandler( + createBaseSignalEventHandlerDeps({ + cfg: { + messages: { inbound: { debounceMs: 0 } }, + channels: { signal: { dmPolicy: "open", allowFrom: ["*"] } }, + }, + ignoreAttachments: false, + fetchAttachment: async () => { + throw new Error("expired attachment"); + }, + historyLimit: 0, + }), + ); + + await handler( + createSignalReceiveEvent({ + dataMessage: { + message: "please inspect this", + attachments: [{ id: "a1", contentType: "image/jpeg" }], + }, + }), + ); + + const context = requireCapturedContext(); + expect(context.BodyForAgent).toContain( + "please inspect this\n\n[signal attachment unavailable]", + ); + expect(context.RawBody).toBe("please inspect this"); + expect(context.CommandBody).toBe("please inspect this"); + expect(context.BodyForAgent).not.toContain(""); + expect(context.MediaPath).toBeUndefined(); + }); + + it("combines raw and command text across failed-media debounce batches", async () => { + vi.useFakeTimers(); + try { + const handler = createSignalEventHandler( + createBaseSignalEventHandlerDeps({ + cfg: { + messages: { inbound: { debounceMs: 10 } }, + channels: { signal: { dmPolicy: "open", allowFrom: ["*"] } }, + }, + ignoreAttachments: false, + fetchAttachment: async () => { + throw new Error("expired attachment"); + }, + historyLimit: 0, + }), + ); + + await handler( + createSignalReceiveEvent({ + dataMessage: { + message: "first request", + attachments: [{ id: "a1", contentType: "image/jpeg" }], + }, + }), + ); + await handler( + createSignalReceiveEvent({ + dataMessage: { + message: "second request", + attachments: [], + }, + }), + ); + await vi.advanceTimersByTimeAsync(10); + + const context = requireCapturedContext(); + expect(context.BodyForAgent).toContain("[signal attachment unavailable]"); + expect(context.RawBody).toBe("first request\\nsecond request"); + expect(context.CommandBody).toBe("first request\\nsecond request"); + } finally { + vi.useRealTimers(); + } + }); + + it("dispatches failed-media commands without text debounce", async () => { + const handler = createSignalEventHandler( + createBaseSignalEventHandlerDeps({ + cfg: { + messages: { inbound: { debounceMs: 60_000 } }, + channels: { signal: { dmPolicy: "open", allowFrom: ["*"] } }, + }, + ignoreAttachments: false, + fetchAttachment: async () => { + throw new Error("expired attachment"); + }, + historyLimit: 0, + }), + ); + + await handler( + createSignalReceiveEvent({ + dataMessage: { + message: "/stop", + attachments: [{ id: "a1", contentType: "image/jpeg" }], + }, + }), + ); + + const context = requireCapturedContext(); + expect(context.CommandBody).toBe("/stop"); + expect(context.RawBody).toBe("/stop"); + expect(context.BodyForAgent).toBe("/stop\n\n[signal attachment unavailable]"); + }); + it("threads resolved audio contentType for Signal voice attachments", async () => { const handler = createSignalEventHandler( createBaseSignalEventHandlerDeps({ diff --git a/extensions/signal/src/monitor/event-handler.ts b/extensions/signal/src/monitor/event-handler.ts index e7d27995f30..0d96c8e0511 100644 --- a/extensions/signal/src/monitor/event-handler.ts +++ b/extensions/signal/src/monitor/event-handler.ts @@ -15,6 +15,7 @@ import { buildMentionRegexes, buildChannelInboundEventContext, createChannelInboundDebouncer, + formatInboundMediaUnavailableText, formatInboundEnvelope, formatInboundFromLabel, matchesMentionPatterns, @@ -334,7 +335,7 @@ export function createSignalEventHandler(deps: SignalEventHandlerDeps) { body: combinedBody, bodyForAgent: entry.bodyText, inboundHistory, - rawBody: entry.bodyText, + rawBody: entry.commandBody, commandBody: entry.commandBody, }, access: { @@ -514,7 +515,7 @@ export function createSignalEventHandler(deps: SignalEventHandlerDeps) { ingest: () => ({ id: entry.messageId ?? `${entry.timestamp ?? Date.now()}`, timestamp: entry.timestamp, - rawText: entry.bodyText, + rawText: entry.commandBody, raw: entry, }), resolveTurn: () => ({ @@ -644,7 +645,7 @@ export function createSignalEventHandler(deps: SignalEventHandlerDeps) { }, shouldDebounce: (entry) => { return shouldDebounceTextInbound({ - text: entry.bodyText, + text: entry.commandBody, cfg: deps.cfg, hasMedia: Boolean(entry.mediaPath || entry.mediaType || entry.mediaPaths?.length), }); @@ -662,12 +663,17 @@ export function createSignalEventHandler(deps: SignalEventHandlerDeps) { .map((entry) => entry.bodyText) .filter(Boolean) .join("\\n"); + const combinedCommandBody = entries + .map((entry) => entry.commandBody) + .filter(Boolean) + .join("\\n"); if (!combinedText.trim()) { return; } await handleSignalInboundMessage({ ...last, bodyText: combinedText, + commandBody: combinedCommandBody, mediaPath: undefined, mediaType: undefined, mediaPaths: undefined, @@ -1064,9 +1070,11 @@ export function createSignalEventHandler(deps: SignalEventHandlerDeps) { const mediaTypes: string[] = []; let placeholder = ""; const attachments = dataMessage.attachments ?? []; + let unavailableAttachmentCount = deps.ignoreAttachments ? attachments.length : 0; if (!deps.ignoreAttachments) { for (const attachment of attachments) { if (!attachment?.id) { + unavailableAttachmentCount += 1; continue; } try { @@ -1087,8 +1095,11 @@ export function createSignalEventHandler(deps: SignalEventHandlerDeps) { mediaPath = fetched.path; mediaType = fetched.contentType ?? attachment.contentType ?? undefined; } + } else { + unavailableAttachmentCount += 1; } } catch (err) { + unavailableAttachmentCount += 1; deps.runtime.error?.(danger(`attachment fetch failed: ${String(err)}`)); } } @@ -1100,12 +1111,19 @@ export function createSignalEventHandler(deps: SignalEventHandlerDeps) { const kind = kindFromMime(mediaType ?? undefined); if (kind) { placeholder = ``; - } else if (attachments.length) { + } else if (mediaPaths.length > 0) { placeholder = ""; } } - const bodyText = messageText || placeholder || visibleQuoteText || ""; + let bodyText = messageText || placeholder || visibleQuoteText || ""; + if (unavailableAttachmentCount > 0) { + const attachmentLabel = unavailableAttachmentCount === 1 ? "attachment" : "attachments"; + bodyText = formatInboundMediaUnavailableText({ + body: bodyText, + notice: `[signal ${unavailableAttachmentCount > 1 ? `${unavailableAttachmentCount} ` : ""}${attachmentLabel} unavailable]`, + }); + } if (!bodyText) { return; } diff --git a/extensions/whatsapp/src/auto-reply.web-auto-reply.connection-and-logging.e2e.test.ts b/extensions/whatsapp/src/auto-reply.web-auto-reply.connection-and-logging.e2e.test.ts index 1ed5ee18fea..4951586371e 100644 --- a/extensions/whatsapp/src/auto-reply.web-auto-reply.connection-and-logging.e2e.test.ts +++ b/extensions/whatsapp/src/auto-reply.web-auto-reply.connection-and-logging.e2e.test.ts @@ -998,6 +998,17 @@ describe("web auto-reply connection", () => { }); expect(capture.getLastOptions()?.shouldDebounce?.(msg)).toBe(true); + expect( + capture.getLastOptions()?.shouldDebounce?.( + createTestWebInboundMessage({ + payload: { + body: "/stop\n\n[whatsapp attachment unavailable]", + commandBody: "/stop", + }, + platform: { sendComposing, reply, sendMedia }, + }), + ), + ).toBe(false); await onMessage(msg); expect(reply).toHaveBeenCalledWith("ok", undefined); diff --git a/extensions/whatsapp/src/auto-reply/monitor.ts b/extensions/whatsapp/src/auto-reply/monitor.ts index 2aaf79a4add..865427eab09 100644 --- a/extensions/whatsapp/src/auto-reply/monitor.ts +++ b/extensions/whatsapp/src/auto-reply/monitor.ts @@ -267,7 +267,10 @@ export async function monitorWebChannel( if (normalized.quote?.id || normalized.quote?.body) { return false; } - return !isControlCommandMessage(normalized.payload.body, cfg); + return !isControlCommandMessage( + normalized.payload.commandBody ?? normalized.payload.body, + cfg, + ); }; let connection; diff --git a/extensions/whatsapp/src/auto-reply/monitor/group-gating.ts b/extensions/whatsapp/src/auto-reply/monitor/group-gating.ts index ee81b724e60..1ee7b4b7784 100644 --- a/extensions/whatsapp/src/auto-reply/monitor/group-gating.ts +++ b/extensions/whatsapp/src/auto-reply/monitor/group-gating.ts @@ -187,7 +187,13 @@ export async function applyGroupGating(params: ApplyGroupGatingParams) { const mentionMsg: AdmittedWebInboundMessage = params.mentionText !== undefined ? { ...params.msg, payload: { ...params.msg.payload, body: params.mentionText } } - : params.msg; + : { + ...params.msg, + payload: { + ...params.msg.payload, + body: params.msg.payload.commandBody ?? params.msg.payload.body, + }, + }; const commandBody = stripMentionsForCommand( mentionMsg.payload.body, mentionConfig.mentionRegexes, diff --git a/extensions/whatsapp/src/auto-reply/monitor/process-message.test.ts b/extensions/whatsapp/src/auto-reply/monitor/process-message.test.ts index edffa465955..8f463364dc8 100644 --- a/extensions/whatsapp/src/auto-reply/monitor/process-message.test.ts +++ b/extensions/whatsapp/src/auto-reply/monitor/process-message.test.ts @@ -173,7 +173,7 @@ function makePolicy(account: ReturnType) { const GROUP_JID = "123@g.us"; -function makeBaseMsg(overrides: { body?: string } = {}) { +function makeBaseMsg(overrides: { body?: string; commandBody?: string } = {}) { const body = overrides.body ?? "hi"; return createTestWebInboundMessage({ event: { @@ -182,6 +182,7 @@ function makeBaseMsg(overrides: { body?: string } = {}) { }, payload: { body, + commandBody: overrides.commandBody, }, platform: { chatJid: GROUP_JID, @@ -328,6 +329,27 @@ describe("processMessage group system prompt wiring", () => { }); }); + it("keeps generated media notices out of command input", async () => { + resolvePolicyMock.mockReturnValue(makePolicy(makeAccount())); + isControlCommandMessageMock.mockReturnValue(true); + shouldComputeCommandAuthorizedMock.mockReturnValue(true); + + await callProcessMessage({ + msg: makeBaseMsg({ + body: "/reset\n\n[whatsapp attachment unavailable]", + commandBody: "/reset", + }), + }); + + expect(shouldComputeCommandAuthorizedMock).toHaveBeenCalledWith("/reset", {}); + expect(isControlCommandMessageMock).toHaveBeenCalledWith("/reset", {}); + expect(buildContextMock.mock.calls[0][0]).toMatchObject({ + bodyForAgent: "/reset\n\n[whatsapp attachment unavailable]", + commandBody: "/reset", + rawBody: "/reset", + }); + }); + it("checks auth for inline command tokens without marking them as command-source turns", async () => { resolvePolicyMock.mockReturnValue(makePolicy(makeAccount())); isControlCommandMessageMock.mockReturnValue(false); diff --git a/extensions/whatsapp/src/auto-reply/monitor/process-message.ts b/extensions/whatsapp/src/auto-reply/monitor/process-message.ts index 572f92ba6ed..0c9c8685482 100644 --- a/extensions/whatsapp/src/auto-reply/monitor/process-message.ts +++ b/extensions/whatsapp/src/auto-reply/monitor/process-message.ts @@ -422,16 +422,14 @@ export async function processMessage(params: { } const sender = getSenderIdentity(params.msg); + const commandBody = params.msg.payload.commandBody ?? params.msg.payload.body; const dmRouteTarget = resolveWhatsAppDmRouteTarget({ msg: params.msg, senderE164: sender.e164 ?? undefined, normalizeE164, }); - const shouldCheckCommandAuth = shouldComputeCommandAuthorized( - params.msg.payload.body, - params.cfg, - ); - const isTextCommand = isControlCommandMessage(params.msg.payload.body, params.cfg); + const shouldCheckCommandAuth = shouldComputeCommandAuthorized(commandBody, params.cfg); + const isTextCommand = isControlCommandMessage(commandBody, params.cfg); const commandAuthorized = shouldCheckCommandAuth ? await resolveWhatsAppCommandAuthorized({ cfg: params.cfg, @@ -445,13 +443,13 @@ export async function processMessage(params: { kind: "text-slash", source: "text", authorized: Boolean(commandAuthorized), - body: params.msg.payload.body, + body: commandBody, } : { kind: "normal", source: "message", authorized: false, - body: params.msg.payload.body, + body: commandBody, }; const { onModelSelected, ...replyPipeline } = createChannelMessageReplyPipeline({ cfg: params.cfg, @@ -485,14 +483,14 @@ export async function processMessage(params: { const ctxPayload = await buildWhatsAppInboundContext({ bodyForAgent: msgForAgent.payload.body, combinedBody, - commandBody: params.msg.payload.body, + commandBody, commandAuthorized, commandTurn, groupHistory: visibleGroupHistory, groupMemberRoster: params.groupMemberNames.get(params.groupHistoryKey), groupSystemPrompt: conversationSystemPrompt, msg: params.msg, - rawBody: params.msg.payload.body, + rawBody: commandBody, route: params.route, sender: { id: getPrimaryIdentityId(sender) ?? undefined, diff --git a/extensions/whatsapp/src/inbound.media.test.ts b/extensions/whatsapp/src/inbound.media.test.ts index 5214455a84f..92b5a73d6a0 100644 --- a/extensions/whatsapp/src/inbound.media.test.ts +++ b/extensions/whatsapp/src/inbound.media.test.ts @@ -67,6 +67,7 @@ function createInMemoryKeyedStore() { const readAllowFromStoreMock = vi.fn().mockResolvedValue([]); const upsertPairingRequestMock = vi.fn().mockResolvedValue({ code: "PAIRCODE", created: true }); const saveMediaStreamSpy = vi.fn(); +const downloadMediaMessageMock = vi.hoisted(() => vi.fn()); let currentMockSocket: | { ev: import("node:events").EventEmitter; @@ -185,7 +186,9 @@ vi.mock("baileys", async () => { return { ...actual, DisconnectReason: actual.DisconnectReason ?? { loggedOut: 401 }, - downloadMediaMessage: vi.fn().mockImplementation(() => Readable.from([jpegBuffer])), + downloadMediaMessage: downloadMediaMessageMock.mockImplementation(() => + Readable.from([jpegBuffer]), + ), extractMessageContent: vi.fn((message: MockMessageInput) => mockExtractMessageContent(message)), getContentType: vi.fn((message: MockMessageInput) => mockGetContentType(message)), isJidGroup: vi.fn((jid: string | undefined | null) => mockIsJidGroup(jid)), @@ -257,6 +260,7 @@ describe("web inbound media saves with extension", () => { beforeEach(() => { vi.useRealTimers(); currentMockSocket = undefined; + downloadMediaMessageMock.mockClear(); saveMediaStreamSpy.mockClear(); resetWebInboundDedupe(); }); @@ -436,4 +440,38 @@ describe("web inbound media saves with extension", () => { await listener.close(); }); + + it("replaces a failed image placeholder with an unavailable notice", async () => { + downloadMediaMessageMock.mockRejectedValueOnce(new Error("expired media reference")); + const onMessage = vi.fn(); + const listener = await monitorWebInbox({ + cfg: { + channels: { whatsapp: { allowFrom: ["*"] } }, + messages: { messagePrefix: undefined, responsePrefix: undefined }, + } as never, + verbose: false, + onMessage, + accountId: "default", + authDir: path.join(HOME, "wa-auth"), + }); + const realSock = await getMockSocket(); + + realSock.ev.emit("messages.upsert", { + type: "notify", + messages: [ + { + key: { id: "img-failed", fromMe: false, remoteJid: "111@s.whatsapp.net" }, + message: { imageMessage: { mimetype: "image/jpeg" } }, + messageTimestamp: 1_700_000_006, + }, + ], + }); + + const inbound = await waitForMessage(onMessage); + expect(inbound.payload.body).toBe("[whatsapp attachment unavailable]"); + expect(inbound.payload.commandBody).toBe(""); + expect(inbound.payload.media).toBeUndefined(); + + await listener.close(); + }); }); diff --git a/extensions/whatsapp/src/inbound/media.node.test.ts b/extensions/whatsapp/src/inbound/media.node.test.ts index f1a62aff21c..1d1cd3a8e29 100644 --- a/extensions/whatsapp/src/inbound/media.node.test.ts +++ b/extensions/whatsapp/src/inbound/media.node.test.ts @@ -136,4 +136,15 @@ describe("downloadInboundMedia", () => { ).rejects.toThrow(/Media exceeds/i); expect(downloadMediaMessage.mock.calls[0]?.[1]).toBe("stream"); }); + + it("propagates transport download failures to the message owner", async () => { + downloadMediaMessage.mockRejectedValueOnce(new Error("expired media reference")); + + await expect( + downloadInboundMedia( + { message: { imageMessage: { mimetype: "image/jpeg" } } } as never, + mockSock as never, + ), + ).rejects.toThrow("expired media reference"); + }); }); diff --git a/extensions/whatsapp/src/inbound/media.ts b/extensions/whatsapp/src/inbound/media.ts index 2b8dc6ec381..44c14a4ec50 100644 --- a/extensions/whatsapp/src/inbound/media.ts +++ b/extensions/whatsapp/src/inbound/media.ts @@ -1,7 +1,6 @@ // Whatsapp plugin module implements media behavior. import type { proto, WAMessage } from "baileys"; import { saveMediaStream, type SavedMedia } from "openclaw/plugin-sdk/media-store"; -import { logVerbose } from "openclaw/plugin-sdk/runtime-env"; import type { createWaSocket } from "../session.js"; import { extractContextInfo } from "./extract.js"; import { resolveInboundMediaMimetype } from "./media-mimetype.js"; @@ -39,36 +38,28 @@ export async function downloadInboundMedia( ) { return undefined; } - try { - const stream = await downloadMediaMessage( - msg as WAMessage, - "stream", - {}, - { - reuploadRequest: sock.updateMediaMessage, - logger: sock.logger, - }, - ); - const saved = await saveMediaStream( - stream as AsyncIterable, - mimetype, - "inbound", - maxBytes, - fileName, - ).catch((err: unknown) => { - if (err instanceof Error && /Media exceeds/i.test(err.message)) { - throw new WhatsAppInboundMediaLimitExceededError(maxBytes); - } - throw err; - }); - return { saved, mimetype, fileName }; - } catch (err) { - if (err instanceof WhatsAppInboundMediaLimitExceededError) { - throw err; + const stream = await downloadMediaMessage( + msg as WAMessage, + "stream", + {}, + { + reuploadRequest: sock.updateMediaMessage, + logger: sock.logger, + }, + ); + const saved = await saveMediaStream( + stream as AsyncIterable, + mimetype, + "inbound", + maxBytes, + fileName, + ).catch((err: unknown) => { + if (err instanceof Error && /Media exceeds/i.test(err.message)) { + throw new WhatsAppInboundMediaLimitExceededError(maxBytes); } - logVerbose(`downloadMediaMessage failed: ${String(err)}`); - return undefined; - } + throw err; + }); + return { saved, mimetype, fileName }; } export async function downloadQuotedInboundMedia( diff --git a/extensions/whatsapp/src/inbound/monitor.ts b/extensions/whatsapp/src/inbound/monitor.ts index 8bf0e5b1e4f..f9ceae416f5 100644 --- a/extensions/whatsapp/src/inbound/monitor.ts +++ b/extensions/whatsapp/src/inbound/monitor.ts @@ -9,7 +9,10 @@ import type { WASocket, } from "baileys"; import { recordChannelActivity } from "openclaw/plugin-sdk/channel-activity-runtime"; -import { formatLocationText } from "openclaw/plugin-sdk/channel-inbound"; +import { + formatInboundMediaUnavailableText, + formatLocationText, +} from "openclaw/plugin-sdk/channel-inbound"; import { createInboundDebouncer } from "openclaw/plugin-sdk/channel-inbound-debounce"; import { getChildLogger } from "openclaw/plugin-sdk/logging-core"; import { @@ -510,6 +513,10 @@ export async function attachWebInboxToSocket( .map((entry) => entry.payload.body) .filter(Boolean) .join("\n"); + const combinedCommandBody = orderedEntries + .map((entry) => entry.payload.commandBody ?? entry.payload.body) + .filter(Boolean) + .join("\n"); const combinedMentions = mentioned.size > 0 ? { @@ -529,6 +536,7 @@ export async function attachWebInboxToSocket( payload: { ...last.payload, body: combinedBody, + commandBody: combinedCommandBody, }, group: combinedGroup, event: { @@ -1101,6 +1109,7 @@ export async function attachWebInboxToSocket( type EnrichedInboundMessage = { body: string; + commandBody: string; location?: ReturnType; contactContext?: ReturnType; externalAdReplyContext?: ReturnType; @@ -1115,16 +1124,18 @@ export async function attachWebInboxToSocket( const locationText = location ? formatLocationText(location) : undefined; const contactContext = extractContactContext(msg.message ?? undefined); const externalAdReplyContext = extractExternalAdReplyContext(msg.message ?? undefined); + const mediaPlaceholder = extractMediaPlaceholder(msg.message ?? undefined); let body = extractText(msg.message ?? undefined); if (locationText) { body = [body, locationText].filter(Boolean).join("\n").trim(); } if (!body) { - body = extractMediaPlaceholder(msg.message ?? undefined); + body = mediaPlaceholder; if (!body) { return null; } } + const commandBody = body; const replyContext = describeReplyContext(msg.message as proto.IMessage | undefined); let mediaPath: string | undefined; @@ -1146,17 +1157,31 @@ export async function attachWebInboxToSocket( try { const inboundMedia = await downloadInboundMedia(msg as proto.IWebMessageInfo, sock, maxBytes); await saveInboundMedia(inboundMedia); - if (!mediaPath && replyContext) { + } catch (err) { + logWhatsAppVerbose(options.verbose, `Inbound media download failed: ${String(err)}`); + body = formatInboundMediaUnavailableText({ + body, + mediaPlaceholder, + notice: "[whatsapp attachment unavailable]", + }); + } + if (!mediaPath && replyContext) { + try { await saveInboundMedia( await downloadQuotedInboundMedia(msg as proto.IWebMessageInfo, sock, maxBytes), ); + } catch (err) { + logWhatsAppVerbose(options.verbose, `Quoted media download failed: ${String(err)}`); + body = formatInboundMediaUnavailableText({ + body, + notice: "[whatsapp quoted attachment unavailable]", + }); } - } catch (err) { - logWhatsAppVerbose(options.verbose, `Inbound media download failed: ${String(err)}`); } return { body, + commandBody, location: location ?? undefined, contactContext, externalAdReplyContext, @@ -1272,6 +1297,7 @@ export async function attachWebInboxToSocket( }, payload: { body: enriched.body, + commandBody: enriched.commandBody, location: enriched.location ?? undefined, untrustedStructuredContext: untrustedStructuredContext.length > 0 ? untrustedStructuredContext : undefined, diff --git a/extensions/whatsapp/src/inbound/types.ts b/extensions/whatsapp/src/inbound/types.ts index 617ccac97d6..56447bcb66e 100644 --- a/extensions/whatsapp/src/inbound/types.ts +++ b/extensions/whatsapp/src/inbound/types.ts @@ -86,6 +86,7 @@ export type WhatsAppInboundGroupContext = { export type WhatsAppInboundPayload = { body: string; + commandBody?: string; media?: { path?: string; type?: string; diff --git a/extensions/zalo/src/monitor.image.polling.test.ts b/extensions/zalo/src/monitor.image.polling.test.ts index d1c2b3ad082..9fed7b8130a 100644 --- a/extensions/zalo/src/monitor.image.polling.test.ts +++ b/extensions/zalo/src/monitor.image.polling.test.ts @@ -1,6 +1,6 @@ // Zalo tests cover monitor.image.polling plugin behavior. import { createRuntimeEnv } from "openclaw/plugin-sdk/plugin-test-runtime"; -import { afterAll, beforeEach, describe, expect, it } from "vitest"; +import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; import { createImageLifecycleCore, createImageUpdate, @@ -114,4 +114,42 @@ describe("Zalo polling image handling", () => { abort.abort(); await run; }); + + it("dispatches an unavailable notice when the inbound image download fails", async () => { + saveRemoteMediaMock.mockRejectedValueOnce(new Error("expired image URL")); + getUpdatesMock + .mockResolvedValueOnce({ + ok: true, + result: createImageUpdate({ caption: "/reset" }), + }) + .mockImplementation(() => new Promise(() => {})); + + const { monitorZaloProvider } = await loadCachedLifecycleMonitorModule("zalo-image-polling"); + const abort = new AbortController(); + const runtime = createRuntimeEnv(); + const { account, config } = createLifecycleMonitorSetup({ + accountId: "default", + dmPolicy: "open", + }); + const run = monitorZaloProvider({ + token: "zalo-token", // pragma: allowlist secret + account, + config, + runtime, + abortSignal: abort.signal, + }); + + await vi.waitFor(() => expect(finalizeInboundContextMock).toHaveBeenCalledTimes(1)); + expect(finalizeInboundContextMock).toHaveBeenCalledWith( + expect.objectContaining({ + RawBody: "/reset", + CommandBody: "/reset", + BodyForAgent: "/reset\n\n[zalo image attachment unavailable]", + MediaPath: undefined, + }), + ); + + abort.abort(); + await run; + }); }); diff --git a/extensions/zalo/src/monitor.ts b/extensions/zalo/src/monitor.ts index 27995da8cd5..1d8b6ea4603 100644 --- a/extensions/zalo/src/monitor.ts +++ b/extensions/zalo/src/monitor.ts @@ -1,6 +1,7 @@ // Zalo plugin module implements monitor behavior. import type { IncomingMessage, ServerResponse } from "node:http"; import { logTypingFailure } from "openclaw/plugin-sdk/channel-feedback"; +import { formatInboundMediaUnavailableText } from "openclaw/plugin-sdk/channel-inbound"; import { resolveStableChannelMessageIngress } from "openclaw/plugin-sdk/channel-ingress-runtime"; import { createChannelPairingController } from "openclaw/plugin-sdk/channel-pairing"; import type { MarkdownTableMode, OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; @@ -154,6 +155,7 @@ function registerSharedHostedMediaRoute(params: { type ZaloMessagePipelineParams = ZaloProcessingContext & { message: ZaloMessage; text?: string; + agentBody?: string; mediaPath?: string; mediaType?: string; authorization?: ZaloMessageAuthorizationResult; @@ -363,7 +365,7 @@ async function handleImageMessage(params: ZaloImageMessageParams): Promise ...params, text: caption, // Use a sentinel so auth sees this as an inbound image before the download happens. - mediaPath: photo_url ? "__pending_media__" : undefined, + mediaPath: "__pending_media__", mediaType: undefined, }); if (!authorization) { @@ -384,9 +386,18 @@ async function handleImageMessage(params: ZaloImageMessageParams): Promise } } + const agentBody = mediaPath + ? authorization.rawBody + : formatInboundMediaUnavailableText({ + body: caption, + mediaPlaceholder: "", + notice: "[zalo image attachment unavailable]", + }); + await processMessageWithPipeline({ ...params, authorization, + agentBody, text: caption, mediaPath, mediaType, @@ -531,6 +542,7 @@ async function processMessageWithPipeline(params: ZaloMessagePipelineParams): Pr mediaType, statusSink, fetcher, + agentBody: agentBodyOverride, authorization: authorizationOverride, } = params; const { message_id, date } = message; @@ -545,6 +557,7 @@ async function processMessageWithPipeline(params: ZaloMessagePipelineParams): Pr return; } const { isGroup, chatId, senderId, senderName, rawBody, commandAuthorized } = authorization; + const agentBody = agentBodyOverride ?? rawBody; const { route, buildEnvelope } = resolveInboundRouteEnvelopeBuilderWithRuntime({ cfg: config, @@ -573,7 +586,7 @@ async function processMessageWithPipeline(params: ZaloMessagePipelineParams): Pr channel: "Zalo", from: fromLabel, timestamp, - body: rawBody, + body: agentBody, }); const ctxPayload = core.channel.inbound.buildContext({ @@ -601,7 +614,7 @@ async function processMessageWithPipeline(params: ZaloMessagePipelineParams): Pr }, message: { body, - bodyForAgent: rawBody, + bodyForAgent: agentBody, rawBody, commandBody: rawBody, }, diff --git a/extensions/zalo/src/test-support/lifecycle-test-support.ts b/extensions/zalo/src/test-support/lifecycle-test-support.ts index 2c390094f00..6488b7a4881 100644 --- a/extensions/zalo/src/test-support/lifecycle-test-support.ts +++ b/extensions/zalo/src/test-support/lifecycle-test-support.ts @@ -102,6 +102,7 @@ export function createImageUpdate(params?: { displayName?: string; chatId?: string; photoUrl?: string; + caption?: string; date?: number; }) { return { @@ -109,7 +110,7 @@ export function createImageUpdate(params?: { message: { date: params?.date ?? 1774086023728, chat: { chat_type: "PRIVATE" as const, id: params?.chatId ?? "chat-123" }, - caption: "", + caption: params?.caption ?? "", message_id: params?.messageId ?? "msg-123", message_type: "CHAT_PHOTO", from: { diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs index 16a0894d367..19df18ec577 100644 --- a/scripts/plugin-sdk-surface-report.mjs +++ b/scripts/plugin-sdk-surface-report.mjs @@ -202,8 +202,8 @@ let publicDeprecatedExportsByEntrypointBudget; try { budgets = { publicEntrypoints: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_ENTRYPOINTS", 324), - publicExports: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_EXPORTS", 10428), - publicFunctionExports: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_FUNCTION_EXPORTS", 5239), + publicExports: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_EXPORTS", 10429), + publicFunctionExports: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_FUNCTION_EXPORTS", 5240), publicDeprecatedExports: readBudgetEnv( "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_DEPRECATED_EXPORTS", 3261, diff --git a/src/channels/inbound-event/media.test.ts b/src/channels/inbound-event/media.test.ts index 256dcb54129..6a280225004 100644 --- a/src/channels/inbound-event/media.test.ts +++ b/src/channels/inbound-event/media.test.ts @@ -3,11 +3,29 @@ import { describe, expect, it } from "vitest"; import { normalizeAttachments } from "../../media-understanding/attachments.normalize.js"; import { buildChannelInboundMediaPayload, + formatInboundMediaUnavailableText, toHistoryMediaEntries, toInboundMediaFacts, } from "./media.js"; describe("channel inbound media facts", () => { + it("replaces optimistic media placeholders and preserves real captions", () => { + expect( + formatInboundMediaUnavailableText({ + body: "", + mediaPlaceholder: "", + notice: "[test image attachment unavailable]", + }), + ).toBe("[test image attachment unavailable]"); + expect( + formatInboundMediaUnavailableText({ + body: "please inspect this", + mediaPlaceholder: "", + notice: "[test image attachment unavailable]", + }), + ).toBe("please inspect this\n\n[test image attachment unavailable]"); + }); + it("normalizes provider media into inbound media facts", () => { expect( toInboundMediaFacts( diff --git a/src/channels/inbound-event/media.ts b/src/channels/inbound-event/media.ts index 65e6e997ccf..977bed8e69f 100644 --- a/src/channels/inbound-event/media.ts +++ b/src/channels/inbound-event/media.ts @@ -32,6 +32,24 @@ export type ChannelInboundMediaPayload = { MediaTranscribedIndexes?: number[]; }; +/** + * Replaces an optimistic media placeholder, or appends to real caption text, + * when transport media could not be materialized for the agent turn. + */ +export function formatInboundMediaUnavailableText(params: { + body?: string | null; + mediaPlaceholder?: string | null; + notice: string; +}): string { + const body = params.body?.trim() ?? ""; + const placeholder = params.mediaPlaceholder?.trim() ?? ""; + const notice = params.notice.trim(); + if (!body || (placeholder && body === placeholder)) { + return notice; + } + return `${body}\n\n${notice}`; +} + function alignedStrings(values: Array): string[] | undefined { if (!values.some(Boolean)) { return undefined; diff --git a/src/plugin-sdk/channel-inbound.ts b/src/plugin-sdk/channel-inbound.ts index f587b7775e2..a6df7234e3e 100644 --- a/src/plugin-sdk/channel-inbound.ts +++ b/src/plugin-sdk/channel-inbound.ts @@ -183,6 +183,7 @@ export { toHistoryMediaEntries, toInboundMediaFacts, buildChannelInboundMediaPayload, + formatInboundMediaUnavailableText, // @deprecated Prefer `buildChannelInboundMediaPayload`. buildChannelInboundMediaPayload as buildChannelTurnMediaPayload, } from "../channels/inbound-event/media.js";