mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-10 00:11:19 +00:00
fix(channels): expose inbound media download failures [AI-assisted] (#100119)
* fix(channels): expose inbound media download failures * fix(msteams): correlate attachment references
This commit is contained in:
parent
161c4581a7
commit
73fc0f5c5f
46 changed files with 1269 additions and 164 deletions
|
|
@ -33,6 +33,7 @@ Docs: https://docs.openclaw.ai
|
|||
|
||||
- **ClawRouter auth profiles:** resolve credential-scoped catalog models during agent runs when the proxy key is stored in an auth profile, and document plugin and model allowlists.
|
||||
- **Telegram durability:** recover stalled ingress claims, retry restart-dropped media, survive transient polling errors, dead-letter poison updates, preserve forwarded rich text, route plugin callbacks correctly, keep progress updates in one stable multi-line window, map self-hosted Bot API container paths through trusted host roots, and fall back safely when Telegram rejects rich final replies. (#97118, #98102, #98735, #98775, #98776, #97174, #98907, #91984, #98786) Thanks @vincentkoc, @luoyanglang, @DaveArcher18, @obviyus, @goldmar, @Marvinthebored, @Dizesales, and @shakkernerd.
|
||||
- **Cross-channel inbound media:** preserve captions and expose unavailable-attachment notices when WhatsApp, LINE, Signal, iMessage, Microsoft Teams, Feishu, Mattermost, or Zalo cannot materialize inbound media, instead of dispatching phantom placeholders or dropping media-only turns. (#100092)
|
||||
- **Agent and context reliability:** preserve runtime overrides, steered subagent tasks, fallback tool-call hints, and legacy reseed attachments; soft-resume CLI sessions across prompt-only drift; improve harness-aware context estimation and compaction prechecks; time out silent local streams; recover mid-stream failures; and cap Gateway run-cache growth. (#92237, #77539, #99851, #99839, #99822, #97928, #97861, #98525, #95430, #77973) Thanks @sercada, @amittell, @obviyus, @liuhao1024, @yetval, @osolmaz, @lzyyzznl, @vincentkoc, @alexelgier, and @fede-kamel.
|
||||
- **Provider and network safety:** bound oversized or malformed responses across Moonshot, MiniMax, Anthropic OAuth, Discord, Matrix, SMS, browser, update, embeddings, Tlön, and Inworld paths. (#96502, #96322, #96644, #97693, #97662, #97999, #98455, #98508, #98554, #98496, #98660) Thanks @hugenshen, @cursoragent, @lsr911, @solodmd, @Alix-007, @wings1029, @lzyyzznl, @sunlit-deng, @vincentkoc, and @Pandah97.
|
||||
- **Channel delivery and routing:** keep Slack replies in the active thread, preserve account-bound delivery routes, apply response prefixes, suppress internal traces and unwanted fallback replies, and retain WeChat session routing for opaque account ids. (#97168, #98240, #89949, #93639, #97989, #80928, #93686) Thanks @LiuwqGit, @gorkem2020, @yetval, @wangwllu, @ZengWen-DT, @alexuser, @UnClouded77, @zhangguiping-xydt, @htkillermax-gif, and @vincentkoc.
|
||||
|
|
|
|||
|
|
@ -141,17 +141,8 @@ export function parseMessageContent(content: string, messageType: string): strin
|
|||
if (messageType === "text") {
|
||||
return parsed.text || "";
|
||||
}
|
||||
if (["image", "file", "audio", "video", "media", "sticker"].includes(messageType)) {
|
||||
if (messageType === "audio") {
|
||||
const speechToText =
|
||||
typeof parsed.speech_to_text === "string" ? parsed.speech_to_text.trim() : "";
|
||||
if (speechToText) {
|
||||
return speechToText;
|
||||
}
|
||||
}
|
||||
const placeholder = inferPlaceholder(messageType);
|
||||
const fileName = typeof parsed.file_name === "string" ? parsed.file_name.trim() : "";
|
||||
return fileName ? `${placeholder} (${fileName})` : placeholder;
|
||||
if (FEISHU_MEDIA_MESSAGE_TYPES.has(messageType)) {
|
||||
return formatFeishuMediaContent(parsed, messageType).body;
|
||||
}
|
||||
if (messageType === "share_chat") {
|
||||
if (parsed && typeof parsed === "object") {
|
||||
|
|
@ -177,6 +168,60 @@ export function parseMessageContent(content: string, messageType: string): strin
|
|||
}
|
||||
}
|
||||
|
||||
const FEISHU_MEDIA_MESSAGE_TYPES = new Set(["image", "file", "audio", "video", "media", "sticker"]);
|
||||
|
||||
function formatFeishuMediaContent(
|
||||
parsed: Record<string, unknown>,
|
||||
messageType: string,
|
||||
): { body: string; mediaPlaceholder?: string; unavailableBody?: string } {
|
||||
const speechToText =
|
||||
messageType === "audio" && typeof parsed.speech_to_text === "string"
|
||||
? parsed.speech_to_text.trim()
|
||||
: "";
|
||||
if (speechToText) {
|
||||
return { body: speechToText };
|
||||
}
|
||||
|
||||
const placeholder = inferPlaceholder(messageType);
|
||||
const fileName = typeof parsed.file_name === "string" ? parsed.file_name.trim() : "";
|
||||
const body = fileName ? `${placeholder} (${fileName})` : placeholder;
|
||||
return {
|
||||
body,
|
||||
mediaPlaceholder: placeholder,
|
||||
unavailableBody: fileName || undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveFeishuMediaFailurePresentation(
|
||||
content: string,
|
||||
messageType: string,
|
||||
): { mediaPlaceholder?: string; unavailableBody?: string } {
|
||||
if (messageType === "post") {
|
||||
return {
|
||||
unavailableBody: parsePostContent(content, {
|
||||
renderMediaPlaceholders: false,
|
||||
emptyTextFallback: "",
|
||||
}).textContent,
|
||||
};
|
||||
}
|
||||
if (!FEISHU_MEDIA_MESSAGE_TYPES.has(messageType)) {
|
||||
return {};
|
||||
}
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(content);
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
||||
return {};
|
||||
}
|
||||
const presentation = formatFeishuMediaContent(parsed as Record<string, unknown>, messageType);
|
||||
return {
|
||||
mediaPlaceholder: presentation.mediaPlaceholder,
|
||||
unavailableBody: presentation.unavailableBody,
|
||||
};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function formatSubMessageContent(content: string, contentType: string): string {
|
||||
try {
|
||||
const parsed = JSON.parse(content);
|
||||
|
|
@ -380,18 +425,19 @@ export async function resolveFeishuMediaList(params: {
|
|||
maxBytes: number;
|
||||
log?: (msg: string) => void;
|
||||
accountId?: string;
|
||||
}): Promise<FeishuMediaInfo[]> {
|
||||
}): Promise<{ media: FeishuMediaInfo[]; unavailableCount: number }> {
|
||||
const { cfg, messageId, messageType, content, maxBytes, log, accountId } = params;
|
||||
const mediaTypes = ["image", "file", "audio", "video", "media", "sticker", "post"];
|
||||
if (!mediaTypes.includes(messageType)) {
|
||||
return [];
|
||||
return { media: [], unavailableCount: 0 };
|
||||
}
|
||||
|
||||
const out: FeishuMediaInfo[] = [];
|
||||
let unavailableCount = 0;
|
||||
if (messageType === "post") {
|
||||
const { imageKeys, mediaKeys } = parsePostContent(content);
|
||||
if (imageKeys.length === 0 && mediaKeys.length === 0) {
|
||||
return [];
|
||||
return { media: [], unavailableCount: 0 };
|
||||
}
|
||||
if (imageKeys.length > 0) {
|
||||
log?.(`feishu: post message contains ${imageKeys.length} embedded image(s)`);
|
||||
|
|
@ -418,6 +464,7 @@ export async function resolveFeishuMediaList(params: {
|
|||
});
|
||||
log?.(`feishu: downloaded embedded image ${imageKey}, saved to ${saved.path}`);
|
||||
} catch (err) {
|
||||
unavailableCount += 1;
|
||||
log?.(`feishu: failed to download embedded image ${imageKey}: ${String(err)}`);
|
||||
}
|
||||
}
|
||||
|
|
@ -445,21 +492,22 @@ export async function resolveFeishuMediaList(params: {
|
|||
});
|
||||
log?.(`feishu: downloaded embedded media ${media.fileKey}, saved to ${saved.path}`);
|
||||
} catch (err) {
|
||||
unavailableCount += 1;
|
||||
log?.(`feishu: failed to download embedded media ${media.fileKey}: ${String(err)}`);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
return { media: out, unavailableCount };
|
||||
}
|
||||
|
||||
const mediaKeys = parseMediaKeys(content, messageType);
|
||||
if (!mediaKeys.imageKey && !mediaKeys.fileKey) {
|
||||
return [];
|
||||
return { media: [], unavailableCount: 1 };
|
||||
}
|
||||
|
||||
try {
|
||||
const fileKey = mediaKeys.fileKey || mediaKeys.imageKey;
|
||||
if (!fileKey) {
|
||||
return [];
|
||||
return { media: [], unavailableCount: 1 };
|
||||
}
|
||||
const result = await saveMessageResourceFeishu({
|
||||
cfg,
|
||||
|
|
@ -482,7 +530,8 @@ export async function resolveFeishuMediaList(params: {
|
|||
});
|
||||
log?.(`feishu: downloaded ${messageType} media, saved to ${saved.path}`);
|
||||
} catch (err) {
|
||||
unavailableCount += 1;
|
||||
log?.(`feishu: failed to download ${messageType} media: ${String(err)}`);
|
||||
}
|
||||
return out;
|
||||
return { media: out, unavailableCount };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
// Feishu tests cover bot.helpers plugin behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { ClawdbotConfig } from "../runtime-api.js";
|
||||
import { parseMessageContent } from "./bot-content.js";
|
||||
import { parseMessageContent, resolveFeishuMediaFailurePresentation } from "./bot-content.js";
|
||||
import {
|
||||
buildBroadcastSessionKey,
|
||||
buildFeishuAgentBody,
|
||||
|
|
@ -97,12 +97,24 @@ describe("parseMessageContent media placeholders", () => {
|
|||
"audio",
|
||||
),
|
||||
).toBe("spoken words");
|
||||
expect(
|
||||
resolveFeishuMediaFailurePresentation(
|
||||
JSON.stringify({ file_key: "file_audio", speech_to_text: " spoken words " }),
|
||||
"audio",
|
||||
),
|
||||
).toEqual({ mediaPlaceholder: undefined, unavailableBody: undefined });
|
||||
});
|
||||
|
||||
it("keeps media filenames as placeholder context without raw payload fields", () => {
|
||||
expect(
|
||||
parseMessageContent(JSON.stringify({ file_key: "file_doc", file_name: "q1.pdf" }), "file"),
|
||||
).toBe("<media:document> (q1.pdf)");
|
||||
expect(
|
||||
resolveFeishuMediaFailurePresentation(
|
||||
JSON.stringify({ file_key: "file_doc", file_name: "q1.pdf" }),
|
||||
"file",
|
||||
),
|
||||
).toEqual({ mediaPlaceholder: "<media:document>", unavailableBody: "q1.pdf" });
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -2265,6 +2265,107 @@ describe("handleFeishuMessage command authorization", () => {
|
|||
expect(mockDispatchReplyFromConfig).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("replaces a failed image download placeholder with an unavailable notice", async () => {
|
||||
mockShouldComputeCommandAuthorized.mockReturnValue(false);
|
||||
mockDownloadMessageResourceFeishu.mockRejectedValueOnce(new Error("expired image key"));
|
||||
|
||||
await dispatchMessage({
|
||||
cfg: {
|
||||
channels: { feishu: { dmPolicy: "open" } },
|
||||
} as ClawdbotConfig,
|
||||
event: {
|
||||
sender: { sender_id: { open_id: "ou-sender" } },
|
||||
message: {
|
||||
message_id: "msg-image-failed",
|
||||
chat_id: "oc-dm",
|
||||
chat_type: "p2p",
|
||||
message_type: "image",
|
||||
content: JSON.stringify({ image_key: "expired-image" }),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const context = mockCallArg<{
|
||||
BodyForAgent?: string;
|
||||
CommandBody?: string;
|
||||
MediaPath?: string;
|
||||
RawBody?: string;
|
||||
}>(mockFinalizeInboundContext, 0, 0);
|
||||
expect(context.RawBody).toBe("<media:image>");
|
||||
expect(context.CommandBody).toBe("<media:image>");
|
||||
expect(context.BodyForAgent).toContain("[feishu attachment unavailable]");
|
||||
expect(context.BodyForAgent).not.toContain("<media:image>");
|
||||
expect(context.MediaPath).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves an audio transcript when the media download fails", async () => {
|
||||
mockShouldComputeCommandAuthorized.mockReturnValue(false);
|
||||
mockDownloadMessageResourceFeishu.mockRejectedValueOnce(new Error("expired audio key"));
|
||||
|
||||
await dispatchMessage({
|
||||
cfg: {
|
||||
channels: { feishu: { dmPolicy: "open" } },
|
||||
} as ClawdbotConfig,
|
||||
event: {
|
||||
sender: { sender_id: { open_id: "ou-sender" } },
|
||||
message: {
|
||||
message_id: "msg-audio-failed",
|
||||
chat_id: "oc-dm",
|
||||
chat_type: "p2p",
|
||||
message_type: "audio",
|
||||
content: JSON.stringify({
|
||||
file_key: "expired-audio",
|
||||
speech_to_text: "spoken words",
|
||||
}),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const context = mockCallArg<{
|
||||
BodyForAgent?: string;
|
||||
CommandBody?: string;
|
||||
MediaPath?: string;
|
||||
RawBody?: string;
|
||||
}>(mockFinalizeInboundContext, 0, 0);
|
||||
expect(context.RawBody).toBe("spoken words");
|
||||
expect(context.CommandBody).toBe("spoken words");
|
||||
expect(context.BodyForAgent).toContain("spoken words\n\n[feishu attachment unavailable]");
|
||||
expect(context.MediaPath).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves a filename without a phantom placeholder when a file download fails", async () => {
|
||||
mockShouldComputeCommandAuthorized.mockReturnValue(false);
|
||||
mockDownloadMessageResourceFeishu.mockRejectedValueOnce(new Error("expired file key"));
|
||||
|
||||
await dispatchMessage({
|
||||
cfg: {
|
||||
channels: { feishu: { dmPolicy: "open" } },
|
||||
} as ClawdbotConfig,
|
||||
event: {
|
||||
sender: { sender_id: { open_id: "ou-sender" } },
|
||||
message: {
|
||||
message_id: "msg-file-failed",
|
||||
chat_id: "oc-dm",
|
||||
chat_type: "p2p",
|
||||
message_type: "file",
|
||||
content: JSON.stringify({ file_key: "expired-file", file_name: "q1.pdf" }),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const context = mockCallArg<{
|
||||
BodyForAgent?: string;
|
||||
CommandBody?: string;
|
||||
MediaPath?: string;
|
||||
RawBody?: string;
|
||||
}>(mockFinalizeInboundContext, 0, 0);
|
||||
expect(context.RawBody).toBe("<media:document> (q1.pdf)");
|
||||
expect(context.CommandBody).toBe("<media:document> (q1.pdf)");
|
||||
expect(context.BodyForAgent).toContain("q1.pdf\n\n[feishu attachment unavailable]");
|
||||
expect(context.BodyForAgent).not.toContain("<media:document>");
|
||||
expect(context.MediaPath).toBeUndefined();
|
||||
});
|
||||
|
||||
it("drops group image message when groupPolicy is open but requireMention is explicitly true", async () => {
|
||||
mockShouldComputeCommandAuthorized.mockReturnValue(false);
|
||||
|
||||
|
|
@ -2750,6 +2851,48 @@ describe("handleFeishuMessage command authorization", () => {
|
|||
expect(typeof mockCallArg(mockSaveMediaBuffer, 0, 3)).toBe("number");
|
||||
});
|
||||
|
||||
it("removes failed rich-post media markers while preserving post text", async () => {
|
||||
mockShouldComputeCommandAuthorized.mockReturnValue(false);
|
||||
mockDownloadMessageResourceFeishu.mockRejectedValueOnce(new Error("expired image key"));
|
||||
|
||||
await dispatchMessage({
|
||||
cfg: {
|
||||
channels: { feishu: { dmPolicy: "open" } },
|
||||
} as ClawdbotConfig,
|
||||
event: {
|
||||
sender: { sender_id: { open_id: "ou-sender" } },
|
||||
message: {
|
||||
message_id: "msg-post-image-failed",
|
||||
chat_id: "oc-dm",
|
||||
chat_type: "p2p",
|
||||
message_type: "post",
|
||||
content: JSON.stringify({
|
||||
title: "Rich text",
|
||||
content: [
|
||||
[
|
||||
{ tag: "text", text: "Before " },
|
||||
{ tag: "img", image_key: "expired-image" },
|
||||
{ tag: "text", text: " after" },
|
||||
],
|
||||
],
|
||||
}),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const context = mockCallArg<{
|
||||
BodyForAgent?: string;
|
||||
MediaPath?: string;
|
||||
RawBody?: string;
|
||||
}>(mockFinalizeInboundContext, 0, 0);
|
||||
expect(context.RawBody).toBe("Rich text\n\nBefore ![image] after");
|
||||
expect(context.BodyForAgent).toContain(
|
||||
"Rich text\n\nBefore after\n\n[feishu attachment unavailable]",
|
||||
);
|
||||
expect(context.BodyForAgent).not.toContain("![image]");
|
||||
expect(context.MediaPath).toBeUndefined();
|
||||
});
|
||||
|
||||
it("includes message_id in BodyForAgent on its own line", async () => {
|
||||
mockShouldComputeCommandAuthorized.mockReturnValue(false);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
// Feishu plugin module implements bot behavior.
|
||||
import {
|
||||
buildChannelInboundEventContext,
|
||||
formatInboundMediaUnavailableText,
|
||||
toInboundMediaFacts,
|
||||
} from "openclaw/plugin-sdk/channel-inbound";
|
||||
import { resolveAgentOutboundIdentity } from "openclaw/plugin-sdk/channel-outbound";
|
||||
|
|
@ -38,6 +39,7 @@ import {
|
|||
parseMessageContent,
|
||||
resolveFeishuGroupSession,
|
||||
resolveFeishuMediaList,
|
||||
resolveFeishuMediaFailurePresentation,
|
||||
} from "./bot-content.js";
|
||||
import {
|
||||
evaluateSupplementalContextVisibility,
|
||||
|
|
@ -1018,7 +1020,7 @@ export async function handleFeishuMessage(params: {
|
|||
|
||||
// Resolve media from message
|
||||
const mediaMaxBytes = (feishuCfg?.mediaMaxMb ?? 30) * 1024 * 1024; // 30MB default
|
||||
const mediaList = await resolveFeishuMediaList({
|
||||
const mediaResolution = await resolveFeishuMediaList({
|
||||
cfg,
|
||||
messageId: ctx.messageId,
|
||||
messageType: event.message.message_type,
|
||||
|
|
@ -1027,6 +1029,19 @@ export async function handleFeishuMessage(params: {
|
|||
log,
|
||||
accountId: account.accountId,
|
||||
});
|
||||
const mediaList = mediaResolution.media;
|
||||
const mediaFailurePresentation = resolveFeishuMediaFailurePresentation(
|
||||
event.message.content,
|
||||
event.message.message_type,
|
||||
);
|
||||
const mediaFailureContent =
|
||||
mediaResolution.unavailableCount > 0
|
||||
? formatInboundMediaUnavailableText({
|
||||
body: mediaFailurePresentation.unavailableBody ?? ctx.content,
|
||||
mediaPlaceholder: mediaFailurePresentation.mediaPlaceholder,
|
||||
notice: `[feishu ${mediaResolution.unavailableCount > 1 ? `${mediaResolution.unavailableCount} attachments` : "attachment"} unavailable]`,
|
||||
})
|
||||
: ctx.content;
|
||||
// Fetch quoted/replied message content before the empty-message guard
|
||||
// so a reply with only @bot (no text, no media) is not dropped when
|
||||
// the quoted message carries meaningful content.
|
||||
|
|
@ -1075,7 +1090,7 @@ export async function handleFeishuMessage(params: {
|
|||
// be empty" errors. Logging the skip avoids silent loss without polluting
|
||||
// the agent session. Quoted content is checked too so a reply-only @bot
|
||||
// with quoted context is not dropped.
|
||||
if (!ctx.content.trim() && mediaList.length === 0 && !quotedContent?.trim()) {
|
||||
if (!mediaFailureContent.trim() && mediaList.length === 0 && !quotedContent?.trim()) {
|
||||
log(
|
||||
`feishu[${account.accountId}]: skipping empty message (no text, no media, no quoted) from ${ctx.senderOpenId}`,
|
||||
);
|
||||
|
|
@ -1096,13 +1111,14 @@ export async function handleFeishuMessage(params: {
|
|||
const inboundMedia = toInboundMediaFacts(mediaList, {
|
||||
transcribed: (_media, index) => index === preflightAudioIndex,
|
||||
});
|
||||
const agentFacingContent = audioTranscript ?? ctx.content;
|
||||
const agentFacingContent = audioTranscript ?? mediaFailureContent;
|
||||
const commandFacingContent = audioTranscript ?? ctx.content;
|
||||
const agentFacingCtx =
|
||||
audioTranscript === undefined
|
||||
agentFacingContent === ctx.content
|
||||
? ctx
|
||||
: {
|
||||
...ctx,
|
||||
content: audioTranscript,
|
||||
content: agentFacingContent,
|
||||
};
|
||||
const effectiveCommandProbeBody =
|
||||
audioTranscript === undefined
|
||||
|
|
@ -1434,8 +1450,8 @@ export async function handleFeishuMessage(params: {
|
|||
body: combinedBody,
|
||||
bodyForAgent: messageBody,
|
||||
inboundHistory,
|
||||
rawBody: agentFacingContent,
|
||||
commandBody: agentFacingContent,
|
||||
rawBody: commandFacingContent,
|
||||
commandBody: commandFacingContent,
|
||||
},
|
||||
access: {
|
||||
mentions: {
|
||||
|
|
|
|||
|
|
@ -72,6 +72,10 @@ describe("parsePostContent", () => {
|
|||
expect(result.textContent).toBe("Before ![image] after\n![image]");
|
||||
expect(result.imageKeys).toEqual(["img_1", "img_2"]);
|
||||
expect(result.mentionedOpenIds).toStrictEqual([]);
|
||||
expect(
|
||||
parsePostContent(content, { renderMediaPlaceholders: false, emptyTextFallback: "" })
|
||||
.textContent,
|
||||
).toBe("Before after");
|
||||
});
|
||||
|
||||
it("supports locale wrappers", () => {
|
||||
|
|
|
|||
|
|
@ -130,6 +130,7 @@ function renderElement(
|
|||
imageKeys: string[],
|
||||
mediaKeys: Array<{ fileKey: string; fileName?: string }>,
|
||||
mentionedOpenIds: string[],
|
||||
renderMediaPlaceholders: boolean,
|
||||
): string {
|
||||
if (!isRecord(element)) {
|
||||
return escapeMarkdownText(toStringOrEmpty(element));
|
||||
|
|
@ -155,7 +156,7 @@ function renderElement(
|
|||
if (imageKey) {
|
||||
imageKeys.push(imageKey);
|
||||
}
|
||||
return "![image]";
|
||||
return renderMediaPlaceholders ? "![image]" : "";
|
||||
}
|
||||
case "media": {
|
||||
const fileKey = normalizeFeishuExternalKey(toStringOrEmpty(element.file_key));
|
||||
|
|
@ -163,7 +164,7 @@ function renderElement(
|
|||
const fileName = toStringOrEmpty(element.file_name) || undefined;
|
||||
mediaKeys.push({ fileKey, fileName });
|
||||
}
|
||||
return "[media]";
|
||||
return renderMediaPlaceholders ? "[media]" : "";
|
||||
}
|
||||
case "emotion":
|
||||
return renderEmotionElement(element);
|
||||
|
|
@ -231,7 +232,10 @@ function resolvePostPayload(parsed: unknown): PostPayload | null {
|
|||
return resolveLocalePayload(parsed);
|
||||
}
|
||||
|
||||
export function parsePostContent(content: string): PostParseResult {
|
||||
export function parsePostContent(
|
||||
content: string,
|
||||
options: { renderMediaPlaceholders?: boolean; emptyTextFallback?: string } = {},
|
||||
): PostParseResult {
|
||||
try {
|
||||
const parsed = JSON.parse(content);
|
||||
const payload = resolvePostPayload(parsed);
|
||||
|
|
@ -255,7 +259,13 @@ export function parsePostContent(content: string): PostParseResult {
|
|||
}
|
||||
let renderedParagraph = "";
|
||||
for (const element of paragraph) {
|
||||
renderedParagraph += renderElement(element, imageKeys, mediaKeys, mentionedOpenIds);
|
||||
renderedParagraph += renderElement(
|
||||
element,
|
||||
imageKeys,
|
||||
mediaKeys,
|
||||
mentionedOpenIds,
|
||||
options.renderMediaPlaceholders !== false,
|
||||
);
|
||||
}
|
||||
paragraphs.push(renderedParagraph);
|
||||
}
|
||||
|
|
@ -265,7 +275,7 @@ export function parsePostContent(content: string): PostParseResult {
|
|||
const textContent = [title, body].filter(Boolean).join("\n\n").trim();
|
||||
|
||||
return {
|
||||
textContent: textContent || FALLBACK_POST_TEXT,
|
||||
textContent: textContent || (options.emptyTextFallback ?? FALLBACK_POST_TEXT),
|
||||
imageKeys,
|
||||
mediaKeys,
|
||||
mentionedOpenIds,
|
||||
|
|
|
|||
|
|
@ -1,9 +1,13 @@
|
|||
// Imessage tests cover monitor.media policy plugin behavior.
|
||||
import type { waitForTransportReady } from "openclaw/plugin-sdk/transport-ready-runtime";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { createIMessageRpcClient } from "./client.js";
|
||||
import { monitorIMessageProvider } from "./monitor.js";
|
||||
import type { stageIMessageAttachments } from "./monitor/media-staging.js";
|
||||
import {
|
||||
formatIMessageInboundMediaBody,
|
||||
resolveIMessageInboundMediaInput,
|
||||
} from "./monitor/monitor-provider.js";
|
||||
|
||||
const waitForTransportReadyMock = vi.hoisted(() =>
|
||||
vi.fn<typeof waitForTransportReady>(async () => {}),
|
||||
|
|
@ -52,8 +56,14 @@ vi.mock("./monitor/media-staging.js", () => ({
|
|||
}));
|
||||
|
||||
describe("iMessage monitor attachment policy", () => {
|
||||
beforeEach(() => {
|
||||
createIMessageRpcClientMock.mockReset();
|
||||
stageIMessageAttachmentsMock.mockReset();
|
||||
readChannelAllowFromStoreMock.mockReset().mockResolvedValue([]);
|
||||
});
|
||||
|
||||
it("does not stage local attachments for messages dropped by inbound policy", async () => {
|
||||
stageIMessageAttachmentsMock.mockResolvedValue([]);
|
||||
stageIMessageAttachmentsMock.mockResolvedValue({ attachments: [], unavailableCount: 0 });
|
||||
readChannelAllowFromStoreMock.mockResolvedValue([]);
|
||||
|
||||
const attachmentPath = "/Users/openclaw/Library/Messages/Attachments/AA/BB/photo.heic";
|
||||
|
|
@ -115,4 +125,60 @@ describe("iMessage monitor attachment policy", () => {
|
|||
await vi.waitFor(() => expect(readChannelAllowFromStoreMock).toHaveBeenCalled());
|
||||
expect(stageIMessageAttachmentsMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("admits attachment-only messages that are marked missing", async () => {
|
||||
const attachment = {
|
||||
original_path: "/Users/openclaw/Library/Messages/Attachments/missing.heic",
|
||||
mime_type: "image/heic",
|
||||
missing: true,
|
||||
};
|
||||
expect(
|
||||
resolveIMessageInboundMediaInput({
|
||||
messageText: "",
|
||||
attachments: [attachment],
|
||||
effectiveAttachmentRoots: [],
|
||||
}),
|
||||
).toEqual({
|
||||
bodyText: "<media:image>",
|
||||
mediaPlaceholder: "<media:image>",
|
||||
mediaCandidates: [attachment],
|
||||
rawMediaAttachments: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the first materialized attachment type when earlier media is unavailable", () => {
|
||||
const missingImage = {
|
||||
original_path: "/Users/openclaw/Library/Messages/Attachments/missing.heic",
|
||||
mime_type: "image/heic",
|
||||
missing: true,
|
||||
};
|
||||
const availableDocument = {
|
||||
original_path: "/Users/openclaw/Library/Messages/Attachments/report.pdf",
|
||||
mime_type: "application/pdf",
|
||||
missing: false,
|
||||
};
|
||||
|
||||
expect(
|
||||
resolveIMessageInboundMediaInput({
|
||||
messageText: "",
|
||||
attachments: [missingImage, availableDocument],
|
||||
effectiveAttachmentRoots: ["/Users/openclaw/Library/Messages/Attachments"],
|
||||
}),
|
||||
).toMatchObject({
|
||||
bodyText: "<media:document>",
|
||||
mediaPlaceholder: "<media:document>",
|
||||
mediaCandidates: [missingImage, availableDocument],
|
||||
rawMediaAttachments: [
|
||||
{ path: availableDocument.original_path, contentType: "application/pdf" },
|
||||
],
|
||||
});
|
||||
expect(
|
||||
formatIMessageInboundMediaBody({
|
||||
messageText: "",
|
||||
optimisticPlaceholder: "<media:image>",
|
||||
mediaAttachments: [{ contentType: "application/pdf" }],
|
||||
unavailableCount: 1,
|
||||
}),
|
||||
).toBe("<media:document>\n\n[imessage attachment unavailable]");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -788,6 +788,61 @@ describe("buildIMessageInboundContext", () => {
|
|||
expect(ctxPayload.MessageSidFull).toBe("p:0/GUID-current");
|
||||
});
|
||||
|
||||
it("keeps generated media notices out of command input", async () => {
|
||||
const decision = await resolveIMessageInboundDecision({
|
||||
cfg: {} as OpenClawConfig,
|
||||
accountId: "default",
|
||||
message: {
|
||||
id: 12347,
|
||||
guid: "p:0/GUID-media-failure",
|
||||
sender: "+15555550123",
|
||||
text: "/reset",
|
||||
is_from_me: false,
|
||||
is_group: false,
|
||||
},
|
||||
opts: undefined,
|
||||
messageText: "/reset",
|
||||
bodyText: "/reset",
|
||||
allowFrom: ["*"],
|
||||
groupAllowFrom: [],
|
||||
groupPolicy: "open",
|
||||
dmPolicy: "open",
|
||||
storeAllowFrom: [],
|
||||
historyLimit: 0,
|
||||
groupHistories: new Map(),
|
||||
echoCache: undefined,
|
||||
selfChatCache: undefined,
|
||||
logVerbose: undefined,
|
||||
});
|
||||
expect(decision.kind).toBe("dispatch");
|
||||
if (decision.kind !== "dispatch") {
|
||||
return;
|
||||
}
|
||||
|
||||
const { ctxPayload } = await buildIMessageInboundContext({
|
||||
cfg: {} as OpenClawConfig,
|
||||
decision: {
|
||||
...decision,
|
||||
agentBodyText: "/reset\n\n[imessage attachment unavailable]",
|
||||
},
|
||||
message: {
|
||||
id: 12347,
|
||||
guid: "p:0/GUID-media-failure",
|
||||
sender: "+15555550123",
|
||||
text: "/reset",
|
||||
is_from_me: false,
|
||||
is_group: false,
|
||||
},
|
||||
historyLimit: 0,
|
||||
groupHistories: new Map(),
|
||||
});
|
||||
|
||||
expect(ctxPayload.RawBody).toBe("/reset");
|
||||
expect(ctxPayload.CommandBody).toBe("/reset");
|
||||
expect(ctxPayload.BodyForAgent).toBe("/reset\n\n[imessage attachment unavailable]");
|
||||
expect(ctxPayload.Body).toContain("/reset\n\n[imessage attachment unavailable]");
|
||||
});
|
||||
|
||||
it("prepends direct-message history when supplied", async () => {
|
||||
const decision = await resolveIMessageInboundDecision({
|
||||
cfg: {} as OpenClawConfig,
|
||||
|
|
|
|||
|
|
@ -351,6 +351,7 @@ type IMessageInboundDispatchDecision = {
|
|||
senderNormalized: string;
|
||||
route: ReturnType<typeof resolveAgentRoute>;
|
||||
bodyText: string;
|
||||
agentBodyText?: string;
|
||||
createdAt?: number;
|
||||
replyContext: IMessageReplyContext | null;
|
||||
effectiveWasMentioned: boolean;
|
||||
|
|
@ -930,7 +931,7 @@ export async function buildIMessageInboundContext(params: {
|
|||
channel: "iMessage",
|
||||
from: fromLabel,
|
||||
timestamp: decision.createdAt,
|
||||
body: `${decision.bodyText}${replySuffix}`,
|
||||
body: `${decision.agentBodyText ?? decision.bodyText}${replySuffix}`,
|
||||
chatType: decision.isGroup ? "group" : "direct",
|
||||
sender: { name: decision.senderNormalized, id: decision.sender },
|
||||
previousTimestamp: params.previousTimestamp,
|
||||
|
|
@ -1028,7 +1029,7 @@ export async function buildIMessageInboundContext(params: {
|
|||
},
|
||||
message: {
|
||||
body: combinedBody,
|
||||
bodyForAgent: decision.bodyText,
|
||||
bodyForAgent: decision.agentBodyText ?? decision.bodyText,
|
||||
inboundHistory,
|
||||
rawBody: decision.bodyText,
|
||||
commandBody: decision.bodyText,
|
||||
|
|
|
|||
|
|
@ -36,7 +36,10 @@ describe("stageIMessageAttachments", () => {
|
|||
[{ original_path: sourcePath, mime_type: "image/png", missing: false }],
|
||||
{ maxBytes: 1024, allowedRoots: [tempDir], deps: { saveMediaBuffer } },
|
||||
),
|
||||
).resolves.toEqual([{ path: "/state/media/inbound/saved.png", contentType: "image/png" }]);
|
||||
).resolves.toEqual({
|
||||
attachments: [{ path: "/state/media/inbound/saved.png", contentType: "image/png" }],
|
||||
unavailableCount: 0,
|
||||
});
|
||||
|
||||
expect(saveMediaBuffer).toHaveBeenCalledWith(
|
||||
Buffer.from("png-bytes"),
|
||||
|
|
@ -70,7 +73,7 @@ describe("stageIMessageAttachments", () => {
|
|||
],
|
||||
{ maxBytes: 1024, allowedRoots: [allowedRoot], deps: { saveMediaBuffer, logVerbose } },
|
||||
),
|
||||
).resolves.toEqual([]);
|
||||
).resolves.toEqual({ attachments: [], unavailableCount: 1 });
|
||||
|
||||
expect(saveMediaBuffer).not.toHaveBeenCalled();
|
||||
expect(logVerbose).toHaveBeenCalledWith(
|
||||
|
|
@ -113,9 +116,18 @@ describe("stageIMessageAttachments", () => {
|
|||
[{ original_path: sourcePath, mime_type: "image/png", missing: false }],
|
||||
{ maxBytes: 4, deps: { saveMediaBuffer, logVerbose } },
|
||||
),
|
||||
).resolves.toEqual([]);
|
||||
).resolves.toEqual({ attachments: [], unavailableCount: 1 });
|
||||
|
||||
expect(saveMediaBuffer).not.toHaveBeenCalled();
|
||||
expect(logVerbose).toHaveBeenCalledWith(expect.stringContaining("failed to stage"));
|
||||
});
|
||||
|
||||
it("counts attachments already marked missing", async () => {
|
||||
await expect(
|
||||
stageIMessageAttachments(
|
||||
[{ original_path: "/tmp/missing.heic", mime_type: "image/heic", missing: true }],
|
||||
{ maxBytes: 1024, deps: { saveMediaBuffer: vi.fn() } },
|
||||
),
|
||||
).resolves.toEqual({ attachments: [], unavailableCount: 1 });
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -11,6 +11,11 @@ export type StagedIMessageAttachment = {
|
|||
contentType?: string;
|
||||
};
|
||||
|
||||
export type StagedIMessageAttachments = {
|
||||
attachments: StagedIMessageAttachment[];
|
||||
unavailableCount: number;
|
||||
};
|
||||
|
||||
type SaveMediaBufferImpl = typeof saveMediaBuffer;
|
||||
|
||||
type StageIMessageAttachmentsDeps = {
|
||||
|
|
@ -135,14 +140,16 @@ export async function stageIMessageAttachments(
|
|||
allowedRoots?: readonly string[];
|
||||
deps?: StageIMessageAttachmentsDeps;
|
||||
},
|
||||
): Promise<StagedIMessageAttachment[]> {
|
||||
): Promise<StagedIMessageAttachments> {
|
||||
const deps = params.deps ?? {};
|
||||
const save = deps.saveMediaBuffer ?? saveMediaBuffer;
|
||||
const staged: StagedIMessageAttachment[] = [];
|
||||
let unavailableCount = 0;
|
||||
|
||||
for (const attachment of attachments) {
|
||||
const attachmentPath = attachment.original_path?.trim();
|
||||
if (!attachmentPath || attachment.missing) {
|
||||
unavailableCount += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -163,9 +170,10 @@ export async function stageIMessageAttachments(
|
|||
);
|
||||
staged.push({ path: saved.path, contentType: saved.contentType });
|
||||
} catch (err) {
|
||||
unavailableCount += 1;
|
||||
deps.logVerbose?.(`imessage: failed to stage inbound attachment: ${String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
return staged;
|
||||
return { attachments: staged, unavailableCount };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import { CHANNEL_APPROVAL_NATIVE_RUNTIME_CONTEXT_CAPABILITY } from "openclaw/plu
|
|||
import { logTypingFailure } from "openclaw/plugin-sdk/channel-feedback";
|
||||
import {
|
||||
createChannelInboundDebouncer,
|
||||
formatInboundMediaUnavailableText,
|
||||
resolveEnvelopeFormatOptions,
|
||||
runChannelInboundEvent,
|
||||
shouldDebounceTextInbound,
|
||||
|
|
@ -109,7 +110,7 @@ import { enqueueIMessageReactionSystemEvent } from "./reaction-system-event.js";
|
|||
import { advanceIMessageRecoveryCursor, loadIMessageRecoveryCursor } from "./recovery-cursor.js";
|
||||
import { normalizeAllowList, resolveRuntime } from "./runtime.js";
|
||||
import { createSelfChatCache } from "./self-chat-cache.js";
|
||||
import type { IMessagePayload, MonitorIMessageOpts } from "./types.js";
|
||||
import type { IMessageAttachment, IMessagePayload, MonitorIMessageOpts } from "./types.js";
|
||||
import { sanitizeIMessageWatchErrorPayload } from "./watch-error-log.js";
|
||||
|
||||
const WATCH_SUBSCRIBE_MAX_ATTEMPTS = 3;
|
||||
|
|
@ -158,6 +159,69 @@ function isIMessagePluginPayloadAttachment(attachment: {
|
|||
);
|
||||
}
|
||||
|
||||
export function resolveIMessageInboundMediaInput(params: {
|
||||
messageText: string;
|
||||
attachments: IMessageAttachment[];
|
||||
effectiveAttachmentRoots: readonly string[];
|
||||
logVerbose?: (message: string) => void;
|
||||
}) {
|
||||
// Apple rich-link previews are opaque plugin payloads; the useful URL stays
|
||||
// in message text. Treating them as media creates phantom attachments and
|
||||
// keeps split-send URL previews out of the text debounce path.
|
||||
const mediaCandidates = params.attachments.filter(
|
||||
(entry) => !isIMessagePluginPayloadAttachment(entry),
|
||||
);
|
||||
const rawMediaAttachments = mediaCandidates.flatMap((attachment) => {
|
||||
const attachmentPath = attachment.original_path?.trim();
|
||||
if (!attachmentPath || attachment.missing) {
|
||||
return [];
|
||||
}
|
||||
if (
|
||||
!isInboundPathAllowed({ filePath: attachmentPath, roots: params.effectiveAttachmentRoots })
|
||||
) {
|
||||
params.logVerbose?.(
|
||||
`imessage: dropping inbound attachment outside allowed roots: ${attachmentPath}`,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
return [{ path: attachmentPath, contentType: attachment.mime_type ?? undefined }];
|
||||
});
|
||||
const kind = kindFromMime(
|
||||
rawMediaAttachments[0]?.contentType ?? mediaCandidates[0]?.mime_type ?? undefined,
|
||||
);
|
||||
const mediaPlaceholder = kind
|
||||
? `<media:${kind}>`
|
||||
: mediaCandidates.length
|
||||
? "<media:attachment>"
|
||||
: "";
|
||||
return {
|
||||
bodyText: params.messageText || mediaPlaceholder,
|
||||
mediaPlaceholder,
|
||||
mediaCandidates,
|
||||
rawMediaAttachments,
|
||||
};
|
||||
}
|
||||
|
||||
export function formatIMessageInboundMediaBody(params: {
|
||||
messageText: string;
|
||||
optimisticPlaceholder: string;
|
||||
mediaAttachments: Array<{ contentType?: string }>;
|
||||
unavailableCount: number;
|
||||
}): string {
|
||||
const materializedKind = kindFromMime(params.mediaAttachments[0]?.contentType);
|
||||
const materializedPlaceholder = materializedKind
|
||||
? `<media:${materializedKind}>`
|
||||
: params.mediaAttachments.length > 0
|
||||
? "<media:attachment>"
|
||||
: "";
|
||||
return formatInboundMediaUnavailableText({
|
||||
body: params.messageText || materializedPlaceholder || params.optimisticPlaceholder,
|
||||
mediaPlaceholder:
|
||||
params.mediaAttachments.length === 0 ? params.optimisticPlaceholder : undefined,
|
||||
notice: `[imessage ${params.unavailableCount > 1 ? `${params.unavailableCount} attachments` : "attachment"} unavailable]`,
|
||||
});
|
||||
}
|
||||
|
||||
async function detectRemoteHostFromCliPath(cliPath: string): Promise<string | undefined> {
|
||||
try {
|
||||
const expanded = cliPath.startsWith("~")
|
||||
|
|
@ -845,42 +909,15 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P
|
|||
const messageText = (pollBody ?? message.text ?? "").trim();
|
||||
const attachments = includeAttachments ? (message.attachments ?? []) : [];
|
||||
const effectiveAttachmentRoots = remoteHost ? remoteAttachmentRoots : attachmentRoots;
|
||||
const validAttachments = attachments.filter((entry) => {
|
||||
if (isIMessagePluginPayloadAttachment(entry)) {
|
||||
// Apple rich-link previews arrive as opaque .pluginPayloadAttachment
|
||||
// files. The useful URL remains in message.text/attributedBody; treating
|
||||
// the preview blob as media creates noisy phantom attachments and can
|
||||
// keep split-send URL previews out of the text debounce path.
|
||||
return false;
|
||||
}
|
||||
const attachmentPath = entry?.original_path?.trim();
|
||||
if (!attachmentPath || entry?.missing) {
|
||||
return false;
|
||||
}
|
||||
if (isInboundPathAllowed({ filePath: attachmentPath, roots: effectiveAttachmentRoots })) {
|
||||
return true;
|
||||
}
|
||||
logVerbose(`imessage: dropping inbound attachment outside allowed roots: ${attachmentPath}`);
|
||||
return false;
|
||||
const mediaInput = resolveIMessageInboundMediaInput({
|
||||
messageText,
|
||||
attachments,
|
||||
effectiveAttachmentRoots,
|
||||
logVerbose,
|
||||
});
|
||||
const rawMediaAttachments = validAttachments.flatMap((a) => {
|
||||
const attachmentPath = a.original_path?.trim();
|
||||
return attachmentPath
|
||||
? [{ path: attachmentPath, contentType: a.mime_type ?? undefined }]
|
||||
: [];
|
||||
});
|
||||
const placeholderMediaType = rawMediaAttachments[0]?.contentType;
|
||||
const kind = kindFromMime(placeholderMediaType ?? undefined);
|
||||
const placeholder = kind
|
||||
? `<media:${kind}>`
|
||||
: validAttachments.length
|
||||
? "<media:attachment>"
|
||||
: "";
|
||||
return {
|
||||
messageText,
|
||||
bodyText: messageText || placeholder,
|
||||
validAttachments,
|
||||
rawMediaAttachments,
|
||||
...mediaInput,
|
||||
effectiveAttachmentRoots,
|
||||
};
|
||||
}
|
||||
|
|
@ -913,7 +950,8 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P
|
|||
const {
|
||||
messageText,
|
||||
bodyText,
|
||||
validAttachments,
|
||||
mediaPlaceholder,
|
||||
mediaCandidates,
|
||||
rawMediaAttachments,
|
||||
effectiveAttachmentRoots,
|
||||
} = resolveIMessageInboundBodyText(message);
|
||||
|
|
@ -1150,20 +1188,36 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P
|
|||
});
|
||||
};
|
||||
}
|
||||
const stagedAttachments = remoteHost
|
||||
? []
|
||||
: await stageIMessageAttachments(validAttachments, {
|
||||
const staged = remoteHost
|
||||
? {
|
||||
attachments: rawMediaAttachments,
|
||||
unavailableCount: mediaCandidates.length - rawMediaAttachments.length,
|
||||
}
|
||||
: await stageIMessageAttachments(mediaCandidates, {
|
||||
maxBytes: mediaMaxBytes,
|
||||
allowedRoots: effectiveAttachmentRoots,
|
||||
deps: { logVerbose },
|
||||
});
|
||||
const mediaAttachments = remoteHost ? rawMediaAttachments : stagedAttachments;
|
||||
const mediaAttachments = staged.attachments;
|
||||
const firstAttachment = mediaAttachments[0];
|
||||
const mediaPath = firstAttachment?.path ?? undefined;
|
||||
const mediaType = firstAttachment?.contentType ?? undefined;
|
||||
// Build arrays for all attachments (for multi-image support)
|
||||
const mediaPaths = mediaAttachments.map((a) => a.path).filter(Boolean);
|
||||
const mediaTypes = mediaAttachments.map((a) => a.contentType ?? undefined);
|
||||
const unavailableCount = staged.unavailableCount;
|
||||
const contextDecision =
|
||||
unavailableCount > 0
|
||||
? {
|
||||
...decision,
|
||||
agentBodyText: formatIMessageInboundMediaBody({
|
||||
messageText,
|
||||
optimisticPlaceholder: mediaPlaceholder,
|
||||
mediaAttachments,
|
||||
unavailableCount,
|
||||
}),
|
||||
}
|
||||
: decision;
|
||||
const previousTimestamp = readSessionUpdatedAt({
|
||||
storePath,
|
||||
sessionKey: decision.route.sessionKey,
|
||||
|
|
@ -1188,7 +1242,7 @@ export async function monitorIMessageProvider(opts: MonitorIMessageOpts = {}): P
|
|||
: undefined;
|
||||
const { ctxPayload, chatTarget, imessageTo } = await buildIMessageInboundContext({
|
||||
cfg,
|
||||
decision,
|
||||
decision: contextDecision,
|
||||
message,
|
||||
previousTimestamp,
|
||||
remoteHost,
|
||||
|
|
|
|||
|
|
@ -1051,6 +1051,31 @@ describe("handleLineWebhookEvents", () => {
|
|||
expect(processMessage).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("reports failed media materialization to the message-context owner", async () => {
|
||||
downloadLineMediaMock.mockRejectedValueOnce(new Error("expired content"));
|
||||
const processMessage = vi.fn();
|
||||
const event = createTestMessageEvent({
|
||||
message: {
|
||||
id: "image-failed-1",
|
||||
type: "image",
|
||||
contentProvider: { type: "line" },
|
||||
quoteToken: "q-image-failed",
|
||||
},
|
||||
source: { type: "user", userId: "user-image-failed" },
|
||||
webhookEventId: "evt-image-failed",
|
||||
});
|
||||
|
||||
await handleLineWebhookEvents(
|
||||
[event],
|
||||
createLineWebhookTestContext({ processMessage, dmPolicy: "open" }),
|
||||
);
|
||||
|
||||
expect(buildLineMessageContextMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ allMedia: [], mediaUnavailable: true }),
|
||||
);
|
||||
expect(processMessage).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("allows non-text group messages through when requireMention is set (cannot detect mention)", async () => {
|
||||
// Image message -- LINE only carries mention metadata on text messages.
|
||||
const event = createTestMessageEvent({
|
||||
|
|
|
|||
|
|
@ -467,6 +467,7 @@ async function handleMessageEvent(event: MessageEvent, context: LineHandlerConte
|
|||
}
|
||||
|
||||
const allMedia: MediaRef[] = [];
|
||||
let mediaUnavailable = false;
|
||||
|
||||
if (isDownloadableLineMessageType(message.type)) {
|
||||
try {
|
||||
|
|
@ -480,6 +481,7 @@ async function handleMessageEvent(event: MessageEvent, context: LineHandlerConte
|
|||
contentType: media.contentType,
|
||||
});
|
||||
} catch (err) {
|
||||
mediaUnavailable = true;
|
||||
const errMsg = String(err);
|
||||
if (errMsg.includes("exceeds") && errMsg.includes("limit")) {
|
||||
logVerbose(`line: media exceeds size limit for message ${message.id}`);
|
||||
|
|
@ -492,6 +494,7 @@ async function handleMessageEvent(event: MessageEvent, context: LineHandlerConte
|
|||
const messageContext = await buildLineMessageContext({
|
||||
event,
|
||||
allMedia,
|
||||
mediaUnavailable,
|
||||
cfg,
|
||||
account,
|
||||
commandAuthorized: decision.commandAccess.authorized,
|
||||
|
|
|
|||
|
|
@ -113,6 +113,30 @@ describe("buildLineMessageContext", () => {
|
|||
expect(context?.ctxPayload.To).toBe("line:group:group-1");
|
||||
});
|
||||
|
||||
it("replaces a failed media placeholder with an unavailable notice", async () => {
|
||||
const event = createMessageEvent({ type: "user", userId: "user-image" }, {
|
||||
message: {
|
||||
id: "image-1",
|
||||
type: "image",
|
||||
contentProvider: { type: "line" },
|
||||
},
|
||||
} as Partial<MessageEvent>);
|
||||
|
||||
const context = await buildLineMessageContext({
|
||||
event,
|
||||
allMedia: [],
|
||||
mediaUnavailable: true,
|
||||
cfg,
|
||||
account,
|
||||
commandAuthorized: true,
|
||||
});
|
||||
|
||||
expect(context?.ctxPayload.RawBody).toBe("<media:image>");
|
||||
expect(context?.ctxPayload.CommandBody).toBe("<media:image>");
|
||||
expect(context?.ctxPayload.BodyForAgent).toBe("[line attachment unavailable]");
|
||||
expect(context?.ctxPayload.MediaPath).toBeUndefined();
|
||||
});
|
||||
|
||||
it("routes group postback replies to the group id", async () => {
|
||||
const event = createPostbackEvent({ type: "group", groupId: "group-2", userId: "user-2" });
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
import type { webhook } from "@line/bot-sdk";
|
||||
import { recordChannelActivity } from "openclaw/plugin-sdk/channel-activity-runtime";
|
||||
import {
|
||||
formatInboundMediaUnavailableText,
|
||||
formatInboundEnvelope,
|
||||
formatLocationText,
|
||||
resolveInboundSessionEnvelopeContext,
|
||||
|
|
@ -36,6 +37,7 @@ interface MediaRef {
|
|||
interface BuildLineMessageContextParams {
|
||||
event: MessageEvent;
|
||||
allMedia: MediaRef[];
|
||||
mediaUnavailable?: boolean;
|
||||
cfg: OpenClawConfig;
|
||||
account: ResolvedLineAccount;
|
||||
commandAuthorized: boolean;
|
||||
|
|
@ -282,6 +284,7 @@ async function finalizeLineInboundContext(params: {
|
|||
route: LineRouteInfo;
|
||||
source: LineSourceInfoWithPeerId;
|
||||
rawBody: string;
|
||||
agentBody?: string;
|
||||
timestamp: number;
|
||||
messageSid: string;
|
||||
commandAuthorized: boolean;
|
||||
|
|
@ -318,11 +321,12 @@ async function finalizeLineInboundContext(params: {
|
|||
sessionKey: params.route.sessionKey,
|
||||
});
|
||||
|
||||
const agentBody = params.agentBody ?? params.rawBody;
|
||||
const body = formatInboundEnvelope({
|
||||
channel: "LINE",
|
||||
from: conversationLabel,
|
||||
timestamp: params.timestamp,
|
||||
body: params.rawBody,
|
||||
body: agentBody,
|
||||
chatType: params.source.isGroup ? "group" : "direct",
|
||||
sender: {
|
||||
id: senderId,
|
||||
|
|
@ -333,7 +337,7 @@ async function finalizeLineInboundContext(params: {
|
|||
|
||||
const ctxPayload = finalizeInboundContext({
|
||||
Body: body,
|
||||
BodyForAgent: params.rawBody,
|
||||
BodyForAgent: agentBody,
|
||||
RawBody: params.rawBody,
|
||||
CommandBody: params.rawBody,
|
||||
From: fromAddress,
|
||||
|
|
@ -437,7 +441,16 @@ async function finalizeLineInboundContext(params: {
|
|||
}
|
||||
|
||||
export async function buildLineMessageContext(params: BuildLineMessageContextParams) {
|
||||
const { event, allMedia, cfg, account, commandAuthorized, groupHistories, historyLimit } = params;
|
||||
const {
|
||||
event,
|
||||
allMedia,
|
||||
mediaUnavailable,
|
||||
cfg,
|
||||
account,
|
||||
commandAuthorized,
|
||||
groupHistories,
|
||||
historyLimit,
|
||||
} = params;
|
||||
|
||||
const source = event.source;
|
||||
const { userId, groupId, roomId, isGroup, peerId, route } = await resolveLineInboundRoute({
|
||||
|
|
@ -457,8 +470,15 @@ export async function buildLineMessageContext(params: BuildLineMessageContextPar
|
|||
if (!rawBody && allMedia.length > 0) {
|
||||
rawBody = `<media:image>${allMedia.length > 1 ? ` (${allMedia.length} images)` : ""}`;
|
||||
}
|
||||
const agentBody = mediaUnavailable
|
||||
? formatInboundMediaUnavailableText({
|
||||
body: rawBody,
|
||||
mediaPlaceholder: placeholder,
|
||||
notice: "[line attachment unavailable]",
|
||||
})
|
||||
: rawBody;
|
||||
|
||||
if (!rawBody && allMedia.length === 0) {
|
||||
if (!agentBody && allMedia.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
@ -489,6 +509,7 @@ export async function buildLineMessageContext(params: BuildLineMessageContextPar
|
|||
route,
|
||||
source: { userId, groupId, roomId, isGroup, peerId },
|
||||
rawBody,
|
||||
agentBody,
|
||||
timestamp,
|
||||
messageSid: messageId,
|
||||
commandAuthorized,
|
||||
|
|
|
|||
|
|
@ -20,9 +20,33 @@ vi.mock("./interactions.js", () => ({
|
|||
|
||||
describe("mattermost monitor resources", () => {
|
||||
let createMattermostMonitorResources: typeof import("./monitor-resources.js").createMattermostMonitorResources;
|
||||
let formatMattermostInboundMediaText: typeof import("./monitor-resources.js").formatMattermostInboundMediaText;
|
||||
|
||||
beforeAll(async () => {
|
||||
({ createMattermostMonitorResources } = await import("./monitor-resources.js"));
|
||||
({ createMattermostMonitorResources, formatMattermostInboundMediaText } =
|
||||
await import("./monitor-resources.js"));
|
||||
});
|
||||
|
||||
it("keeps media-only download failures visible to the agent", () => {
|
||||
expect(
|
||||
formatMattermostInboundMediaText({
|
||||
body: "",
|
||||
mediaPlaceholder: "",
|
||||
expectedCount: 1,
|
||||
mediaCount: 0,
|
||||
}),
|
||||
).toBe("[mattermost attachment unavailable]");
|
||||
});
|
||||
|
||||
it("preserves successful media placeholders on partial failures", () => {
|
||||
expect(
|
||||
formatMattermostInboundMediaText({
|
||||
body: "<media:document> (2 files)",
|
||||
mediaPlaceholder: "<media:document> (2 files)",
|
||||
expectedCount: 2,
|
||||
mediaCount: 1,
|
||||
}),
|
||||
).toBe("<media:document> (2 files)\n\n[mattermost attachment unavailable]");
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
// Mattermost plugin module implements monitor resources behavior.
|
||||
import { formatInboundMediaUnavailableText } from "openclaw/plugin-sdk/channel-inbound";
|
||||
import {
|
||||
asDateTimestampMs,
|
||||
resolveExpiresAtMsFromDurationMs,
|
||||
|
|
@ -23,6 +24,23 @@ export type MattermostMediaInfo = {
|
|||
kind: MattermostMediaKind;
|
||||
};
|
||||
|
||||
export function formatMattermostInboundMediaText(params: {
|
||||
body: string;
|
||||
mediaPlaceholder: string;
|
||||
expectedCount: number;
|
||||
mediaCount: number;
|
||||
}): string {
|
||||
const unavailableCount = Math.max(0, params.expectedCount - params.mediaCount);
|
||||
if (unavailableCount === 0) {
|
||||
return params.body;
|
||||
}
|
||||
return formatInboundMediaUnavailableText({
|
||||
body: params.body,
|
||||
mediaPlaceholder: params.mediaCount === 0 ? params.mediaPlaceholder : undefined,
|
||||
notice: `[mattermost ${unavailableCount > 1 ? `${unavailableCount} attachments` : "attachment"} unavailable]`,
|
||||
});
|
||||
}
|
||||
|
||||
const CHANNEL_CACHE_TTL_MS = 5 * 60_000;
|
||||
const USER_CACHE_TTL_MS = 10 * 60_000;
|
||||
|
||||
|
|
|
|||
|
|
@ -73,7 +73,11 @@ import {
|
|||
shouldDropEmptyMattermostBody,
|
||||
} from "./monitor-helpers.js";
|
||||
import { resolveOncharPrefixes, stripOncharPrefix } from "./monitor-onchar.js";
|
||||
import { createMattermostMonitorResources, type MattermostMediaInfo } from "./monitor-resources.js";
|
||||
import {
|
||||
createMattermostMonitorResources,
|
||||
formatMattermostInboundMediaText,
|
||||
type MattermostMediaInfo,
|
||||
} from "./monitor-resources.js";
|
||||
import { registerMattermostMonitorSlashCommands } from "./monitor-slash.js";
|
||||
import {
|
||||
createMattermostConnectOnce,
|
||||
|
|
@ -1538,10 +1542,17 @@ export async function monitorMattermostProvider(opts: MonitorMattermostOpts = {}
|
|||
recordPendingHistory();
|
||||
return;
|
||||
}
|
||||
const mediaList = await resolveMattermostMedia(post.file_ids);
|
||||
const fileIds = uniqueStrings(normalizeTrimmedStringList(post.file_ids ?? []));
|
||||
const mediaList = await resolveMattermostMedia(fileIds);
|
||||
const mediaPlaceholder = buildMattermostAttachmentPlaceholder(mediaList);
|
||||
const bodySource = oncharTriggered ? oncharResult.stripped : rawText;
|
||||
const baseText = [bodySource, mediaPlaceholder].filter(Boolean).join("\n").trim();
|
||||
const downloadedText = [bodySource, mediaPlaceholder].filter(Boolean).join("\n").trim();
|
||||
const baseText = formatMattermostInboundMediaText({
|
||||
body: downloadedText,
|
||||
mediaPlaceholder,
|
||||
expectedCount: fileIds.length,
|
||||
mediaCount: mediaList.length,
|
||||
});
|
||||
const bodyText = normalizeMention(baseText, botUsername);
|
||||
if (shouldDropEmptyMattermostBody({ bodyText, rawText: rawPostText, botUsername })) {
|
||||
logVerboseMessage(
|
||||
|
|
@ -1612,7 +1623,7 @@ export async function monitorMattermostProvider(opts: MonitorMattermostOpts = {}
|
|||
Body: combinedBody,
|
||||
BodyForAgent: bodyForAgent,
|
||||
InboundHistory: inboundHistory,
|
||||
RawBody: bodyText,
|
||||
RawBody: commandBody,
|
||||
CommandBody: commandBody,
|
||||
BodyForCommands: commandBody,
|
||||
From:
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import {
|
|||
buildMSTeamsAttachmentPlaceholder,
|
||||
buildMSTeamsGraphMessageUrls,
|
||||
buildMSTeamsMediaPayload,
|
||||
resolveMSTeamsInboundAttachmentPresentation,
|
||||
} from "./attachments.js";
|
||||
import { setMSTeamsRuntime } from "./runtime.js";
|
||||
|
||||
|
|
@ -199,6 +200,65 @@ describe("msteams attachment helpers", () => {
|
|||
}),
|
||||
).toBe("<media:document>");
|
||||
});
|
||||
|
||||
it("counts advertised files without URLs and ignores mention-only HTML", () => {
|
||||
expect(
|
||||
resolveMSTeamsInboundAttachmentPresentation([
|
||||
{ contentType: "application/pdf", name: "report.pdf" },
|
||||
]),
|
||||
).toEqual({ placeholder: "<media:document>", expectedMediaCount: 1 });
|
||||
expect(
|
||||
resolveMSTeamsInboundAttachmentPresentation([
|
||||
{ contentType: "text/html", content: "<div><at>Bot</at> hello</div>" },
|
||||
]),
|
||||
).toEqual({ placeholder: "", expectedMediaCount: 0 });
|
||||
});
|
||||
|
||||
it("does not count HTML references separately from files or cards", () => {
|
||||
expect(
|
||||
resolveMSTeamsInboundAttachmentPresentation([
|
||||
createHtmlAttachment('<attachment id="file-1"></attachment>'),
|
||||
{
|
||||
id: "file-1",
|
||||
contentType: CONTENT_TYPE_APPLICATION_PDF,
|
||||
contentUrl: TEST_URL_PDF,
|
||||
},
|
||||
]),
|
||||
).toEqual({ placeholder: "<media:document>", expectedMediaCount: 1 });
|
||||
|
||||
expect(
|
||||
resolveMSTeamsInboundAttachmentPresentation([
|
||||
createHtmlAttachment('<attachment id="card-1"></attachment>'),
|
||||
{
|
||||
id: "card-1",
|
||||
contentType: "application/vnd.microsoft.card.adaptive",
|
||||
content: { type: "AdaptiveCard" },
|
||||
},
|
||||
]),
|
||||
).toEqual({ placeholder: "", expectedMediaCount: 0 });
|
||||
});
|
||||
|
||||
it("counts repeated inline URLs once while keeping data images per occurrence", () => {
|
||||
const repeatedUrl = "https://example.com/repeated.png";
|
||||
expect(
|
||||
resolveMSTeamsInboundAttachmentPresentation([
|
||||
{
|
||||
contentType: "text/html",
|
||||
content: `<img src="${repeatedUrl}"><img src="${repeatedUrl}">`,
|
||||
},
|
||||
]),
|
||||
).toEqual({ placeholder: "<media:image>", expectedMediaCount: 1 });
|
||||
|
||||
const dataUrl = "data:image/png;base64,AQ==";
|
||||
expect(
|
||||
resolveMSTeamsInboundAttachmentPresentation([
|
||||
{
|
||||
contentType: "text/html",
|
||||
content: `<img src="${dataUrl}"><img src="${dataUrl}">`,
|
||||
},
|
||||
]),
|
||||
).toEqual({ placeholder: "<media:image> (2 images)", expectedMediaCount: 2 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildMSTeamsGraphMessageUrls", () => {
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ export { buildMSTeamsGraphMessageUrls, downloadMSTeamsGraphMedia } from "./attac
|
|||
export {
|
||||
buildMSTeamsAttachmentPlaceholder,
|
||||
extractMSTeamsHtmlAttachmentIds,
|
||||
resolveMSTeamsInboundAttachmentPresentation,
|
||||
summarizeMSTeamsHtmlAttachments,
|
||||
} from "./attachments/html.js";
|
||||
export { buildMSTeamsMediaPayload } from "./attachments/payload.js";
|
||||
|
|
|
|||
|
|
@ -4,7 +4,9 @@ import {
|
|||
extractHtmlFromAttachment,
|
||||
extractInlineImageCandidates,
|
||||
IMG_SRC_RE,
|
||||
isDownloadableAttachment,
|
||||
isLikelyImageAttachment,
|
||||
normalizeContentType,
|
||||
safeHostForUrl,
|
||||
} from "./shared.js";
|
||||
import type { MSTeamsAttachmentLike, MSTeamsHtmlAttachmentSummary } from "./types.js";
|
||||
|
|
@ -108,16 +110,111 @@ export function buildMSTeamsAttachmentPlaceholder(
|
|||
attachments: MSTeamsAttachmentLike[] | undefined,
|
||||
limits?: { maxInlineBytes?: number; maxInlineTotalBytes?: number },
|
||||
): string {
|
||||
return resolveMSTeamsInboundAttachmentPresentation(attachments, limits).placeholder;
|
||||
}
|
||||
|
||||
function isAdvertisedFileAttachment(attachment: MSTeamsAttachmentLike): boolean {
|
||||
const contentType = normalizeContentType(attachment.contentType) ?? "";
|
||||
if (
|
||||
contentType.startsWith("text/html") ||
|
||||
contentType.startsWith("application/vnd.microsoft.card.") ||
|
||||
contentType.startsWith("application/vnd.microsoft.teams.card.")
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return Boolean(
|
||||
isDownloadableAttachment(attachment) ||
|
||||
isLikelyImageAttachment(attachment) ||
|
||||
attachment.name?.trim() ||
|
||||
contentType,
|
||||
);
|
||||
}
|
||||
|
||||
function countDistinctInlineImages(attachments: MSTeamsAttachmentLike[]): number {
|
||||
const seenReferences = new Set<string>();
|
||||
let count = 0;
|
||||
for (const attachment of attachments) {
|
||||
const html = extractHtmlFromAttachment(attachment);
|
||||
if (!html) {
|
||||
continue;
|
||||
}
|
||||
IMG_SRC_RE.lastIndex = 0;
|
||||
let match: RegExpExecArray | null = IMG_SRC_RE.exec(html);
|
||||
while (match) {
|
||||
const src = match[1]?.trim();
|
||||
if (src?.startsWith("data:")) {
|
||||
count += 1;
|
||||
} else if (src && !seenReferences.has(src)) {
|
||||
seenReferences.add(src);
|
||||
count += 1;
|
||||
}
|
||||
match = IMG_SRC_RE.exec(html);
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
function countDistinctInlineCandidates(
|
||||
attachments: MSTeamsAttachmentLike[],
|
||||
limits?: { maxInlineBytes?: number; maxInlineTotalBytes?: number },
|
||||
): number {
|
||||
const seenUrls = new Set<string>();
|
||||
let dataCount = 0;
|
||||
for (const candidate of extractInlineImageCandidates(attachments, limits)) {
|
||||
if (candidate.kind === "data") {
|
||||
dataCount += 1;
|
||||
} else {
|
||||
seenUrls.add(candidate.url);
|
||||
}
|
||||
}
|
||||
return dataCount + seenUrls.size;
|
||||
}
|
||||
|
||||
function countUnrepresentedHtmlAttachmentIds(attachments: MSTeamsAttachmentLike[]): number {
|
||||
const representedIds = new Set<string>();
|
||||
for (const attachment of attachments) {
|
||||
const contentType = normalizeContentType(attachment.contentType) ?? "";
|
||||
if (contentType.startsWith("text/html")) {
|
||||
continue;
|
||||
}
|
||||
const id = attachment.id?.trim();
|
||||
if (id) {
|
||||
representedIds.add(id);
|
||||
}
|
||||
}
|
||||
return extractMSTeamsHtmlAttachmentIds(attachments).filter((id) => !representedIds.has(id))
|
||||
.length;
|
||||
}
|
||||
|
||||
export function resolveMSTeamsInboundAttachmentPresentation(
|
||||
attachments: MSTeamsAttachmentLike[] | undefined,
|
||||
limits?: { maxInlineBytes?: number; maxInlineTotalBytes?: number },
|
||||
): { placeholder: string; expectedMediaCount: number } {
|
||||
const list = Array.isArray(attachments) ? attachments : [];
|
||||
if (list.length === 0) {
|
||||
return "";
|
||||
return { placeholder: "", expectedMediaCount: 0 };
|
||||
}
|
||||
const imageCount = list.filter(isLikelyImageAttachment).length;
|
||||
const inlineCount = extractInlineImageCandidates(list, limits).length;
|
||||
const totalImages = imageCount + inlineCount;
|
||||
const fileAttachments = list.filter(isAdvertisedFileAttachment);
|
||||
const inlinePlaceholderCount = countDistinctInlineCandidates(list, limits);
|
||||
const inlineExpectedCount = countDistinctInlineImages(list);
|
||||
// Teams HTML uses <attachment> tags as references. A matching attachment
|
||||
// entry is the same resource (and may be a card), so count only unmatched
|
||||
// IDs that need the Graph/Bot Framework hosted-content fallback.
|
||||
const htmlAttachmentCount = countUnrepresentedHtmlAttachmentIds(list);
|
||||
const expectedMediaCount = fileAttachments.length + inlineExpectedCount + htmlAttachmentCount;
|
||||
if (expectedMediaCount === 0) {
|
||||
return { placeholder: "", expectedMediaCount: 0 };
|
||||
}
|
||||
const totalImages =
|
||||
fileAttachments.filter(isLikelyImageAttachment).length + inlinePlaceholderCount;
|
||||
if (totalImages > 0) {
|
||||
return `<media:image>${totalImages > 1 ? ` (${totalImages} images)` : ""}`;
|
||||
return {
|
||||
placeholder: `<media:image>${totalImages > 1 ? ` (${totalImages} images)` : ""}`,
|
||||
expectedMediaCount,
|
||||
};
|
||||
}
|
||||
const count = list.length;
|
||||
return `<media:document>${count > 1 ? ` (${count} files)` : ""}`;
|
||||
return {
|
||||
placeholder: `<media:document>${expectedMediaCount > 1 ? ` (${expectedMediaCount} files)` : ""}`,
|
||||
expectedMediaCount,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
// Msteams type declarations define plugin contracts.
|
||||
export type MSTeamsAttachmentLike = {
|
||||
id?: string | null;
|
||||
contentType?: string | null;
|
||||
contentUrl?: string | null;
|
||||
name?: string | null;
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ import {
|
|||
downloadMSTeamsGraphMedia,
|
||||
extractMSTeamsHtmlAttachmentIds,
|
||||
} from "../attachments.js";
|
||||
import { resolveMSTeamsInboundMedia } from "./inbound-media.js";
|
||||
import { resolveMSTeamsInboundMedia, resolveMSTeamsInboundMediaBody } from "./inbound-media.js";
|
||||
|
||||
const baseParams = {
|
||||
maxBytes: 1024 * 1024,
|
||||
|
|
@ -52,6 +52,39 @@ function firstBotFrameworkAttachmentCall() {
|
|||
}
|
||||
|
||||
describe("resolveMSTeamsInboundMedia graph fallback trigger", () => {
|
||||
it("replaces a failed attachment placeholder without marking mention-only HTML", () => {
|
||||
expect(
|
||||
resolveMSTeamsInboundMediaBody({
|
||||
body: "<media:document>",
|
||||
mediaPlaceholder: "<media:document>",
|
||||
materializedMediaPlaceholder: "",
|
||||
expectedMediaCount: 1,
|
||||
mediaCount: 0,
|
||||
}),
|
||||
).toBe("[msteams attachment unavailable]");
|
||||
expect(
|
||||
resolveMSTeamsInboundMediaBody({
|
||||
body: "hello",
|
||||
mediaPlaceholder: "<media:document>",
|
||||
materializedMediaPlaceholder: "",
|
||||
expectedMediaCount: 0,
|
||||
mediaCount: 0,
|
||||
}),
|
||||
).toBe("hello");
|
||||
});
|
||||
|
||||
it("preserves successful media while exposing partial download failures", () => {
|
||||
expect(
|
||||
resolveMSTeamsInboundMediaBody({
|
||||
body: "<media:document> (2 files)",
|
||||
mediaPlaceholder: "<media:document> (2 files)",
|
||||
materializedMediaPlaceholder: "<media:document>",
|
||||
expectedMediaCount: 2,
|
||||
mediaCount: 1,
|
||||
}),
|
||||
).toBe("<media:document>\n\n[msteams attachment unavailable]");
|
||||
});
|
||||
|
||||
it("triggers Graph fallback when HTML contains <attachment> tags", async () => {
|
||||
vi.mocked(downloadMSTeamsAttachments).mockResolvedValue([]);
|
||||
vi.mocked(extractMSTeamsHtmlAttachmentIds).mockReturnValueOnce(["att-0"]);
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
// Msteams plugin module implements inbound media behavior.
|
||||
import { formatInboundMediaUnavailableText } from "openclaw/plugin-sdk/channel-inbound";
|
||||
import {
|
||||
buildMSTeamsGraphMessageUrls,
|
||||
downloadMSTeamsAttachments,
|
||||
|
|
@ -19,6 +20,28 @@ type MSTeamsLogger = {
|
|||
error?: (message: string, meta?: Record<string, unknown>) => void;
|
||||
};
|
||||
|
||||
export function resolveMSTeamsInboundMediaBody(params: {
|
||||
body: string;
|
||||
mediaPlaceholder: string;
|
||||
materializedMediaPlaceholder: string;
|
||||
expectedMediaCount: number;
|
||||
mediaCount: number;
|
||||
}): string {
|
||||
const unavailableCount = Math.max(0, params.expectedMediaCount - params.mediaCount);
|
||||
if (unavailableCount === 0) {
|
||||
return params.body;
|
||||
}
|
||||
const body =
|
||||
params.mediaCount > 0 && params.body === params.mediaPlaceholder
|
||||
? params.materializedMediaPlaceholder
|
||||
: params.body;
|
||||
return formatInboundMediaUnavailableText({
|
||||
body,
|
||||
mediaPlaceholder: params.mediaCount === 0 ? params.mediaPlaceholder : undefined,
|
||||
notice: `[msteams ${unavailableCount > 1 ? `${unavailableCount} attachments` : "attachment"} unavailable]`,
|
||||
});
|
||||
}
|
||||
|
||||
export async function resolveMSTeamsInboundMedia(params: {
|
||||
attachments: MSTeamsAttachmentLike[];
|
||||
htmlSummary?: MSTeamsHtmlAttachmentSummary;
|
||||
|
|
|
|||
|
|
@ -23,10 +23,10 @@ import {
|
|||
import { sliceUtf16Safe, truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
|
||||
import { serializeMSTeamsAdaptiveCardActionValue } from "../adaptive-card-submit.js";
|
||||
import {
|
||||
buildMSTeamsAttachmentPlaceholder,
|
||||
buildMSTeamsMediaPayload,
|
||||
type MSTeamsAttachmentLike,
|
||||
resolveMSTeamsInboundAttachmentPresentation,
|
||||
summarizeMSTeamsHtmlAttachments,
|
||||
type MSTeamsAttachmentLike,
|
||||
} from "../attachments.js";
|
||||
import { isRecord } from "../attachments/shared.js";
|
||||
import { tryNormalizeBotFrameworkServiceUrl } from "../bot-framework-service-url.js";
|
||||
|
|
@ -101,7 +101,7 @@ import {
|
|||
wasMSTeamsMessageSentWithPersistence,
|
||||
} from "../sent-message-cache.js";
|
||||
import { resolveMSTeamsSenderAccess } from "./access.js";
|
||||
import { resolveMSTeamsInboundMedia } from "./inbound-media.js";
|
||||
import { resolveMSTeamsInboundMedia, resolveMSTeamsInboundMediaBody } from "./inbound-media.js";
|
||||
import { resolveMSTeamsRouteSessionKey } from "./thread-session.js";
|
||||
|
||||
function formatMSTeamsSenderReason(params: {
|
||||
|
|
@ -231,10 +231,11 @@ export function createMSTeamsMessageHandler(deps: MSTeamsMessageHandlerDeps) {
|
|||
const rawText = params.rawText;
|
||||
const text = params.text;
|
||||
const attachments = params.attachments;
|
||||
const attachmentPlaceholder = buildMSTeamsAttachmentPlaceholder(attachments, {
|
||||
const attachmentPresentation = resolveMSTeamsInboundAttachmentPresentation(attachments, {
|
||||
maxInlineBytes: mediaMaxBytes,
|
||||
maxInlineTotalBytes: mediaMaxBytes,
|
||||
});
|
||||
const attachmentPlaceholder = attachmentPresentation.placeholder;
|
||||
const rawBody = text || attachmentPlaceholder;
|
||||
const quoteInfo = extractMSTeamsQuoteInfo(attachments);
|
||||
let quoteSenderId: string | undefined;
|
||||
|
|
@ -621,6 +622,16 @@ export function createMSTeamsMessageHandler(deps: MSTeamsMessageHandlerDeps) {
|
|||
});
|
||||
|
||||
const mediaPayload = buildMSTeamsMediaPayload(mediaList);
|
||||
const materializedMediaPlaceholder = resolveMSTeamsInboundAttachmentPresentation(
|
||||
mediaList.map((media) => ({ contentType: media.contentType, name: media.path })),
|
||||
).placeholder;
|
||||
const agentBody = resolveMSTeamsInboundMediaBody({
|
||||
body: rawBody,
|
||||
mediaPlaceholder: attachmentPlaceholder,
|
||||
materializedMediaPlaceholder,
|
||||
expectedMediaCount: attachmentPresentation.expectedMediaCount,
|
||||
mediaCount: mediaList.length,
|
||||
});
|
||||
|
||||
// Fetch thread history when the message is a reply inside a Teams channel thread.
|
||||
// This is a best-effort enhancement; errors are logged and do not block the reply.
|
||||
|
|
@ -717,7 +728,7 @@ export function createMSTeamsMessageHandler(deps: MSTeamsMessageHandlerDeps) {
|
|||
timestamp,
|
||||
previousTimestamp,
|
||||
envelope: envelopeOptions,
|
||||
body: rawBody,
|
||||
body: agentBody,
|
||||
});
|
||||
let combinedBody = body;
|
||||
const isRoomish = !isDirectMessage;
|
||||
|
|
@ -760,8 +771,8 @@ export function createMSTeamsMessageHandler(deps: MSTeamsMessageHandlerDeps) {
|
|||
: true;
|
||||
// Prepend thread history to the agent body so the agent has full thread context.
|
||||
const bodyForAgent = threadContext
|
||||
? `[Thread history]\n${threadContext}\n[/Thread history]\n\n${rawBody}`
|
||||
: rawBody;
|
||||
? `[Thread history]\n${threadContext}\n[/Thread history]\n\n${agentBody}`
|
||||
: agentBody;
|
||||
|
||||
// For Teams *channel* messages (not group chats / DMs), preserve the
|
||||
// `teamId/channelId` pair on NativeChannelId so downstream action handlers
|
||||
|
|
|
|||
|
|
@ -1785,6 +1785,114 @@ describe("signal createSignalEventHandler inbound context", () => {
|
|||
expect(context.MediaTypes).toEqual(["image/jpeg", "application/octet-stream"]);
|
||||
});
|
||||
|
||||
it("marks failed attachment downloads unavailable without a phantom media placeholder", async () => {
|
||||
const handler = createSignalEventHandler(
|
||||
createBaseSignalEventHandlerDeps({
|
||||
cfg: {
|
||||
messages: { inbound: { debounceMs: 0 } },
|
||||
channels: { signal: { dmPolicy: "open", allowFrom: ["*"] } },
|
||||
},
|
||||
ignoreAttachments: false,
|
||||
fetchAttachment: async () => {
|
||||
throw new Error("expired attachment");
|
||||
},
|
||||
historyLimit: 0,
|
||||
}),
|
||||
);
|
||||
|
||||
await handler(
|
||||
createSignalReceiveEvent({
|
||||
dataMessage: {
|
||||
message: "please inspect this",
|
||||
attachments: [{ id: "a1", contentType: "image/jpeg" }],
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const context = requireCapturedContext();
|
||||
expect(context.BodyForAgent).toContain(
|
||||
"please inspect this\n\n[signal attachment unavailable]",
|
||||
);
|
||||
expect(context.RawBody).toBe("please inspect this");
|
||||
expect(context.CommandBody).toBe("please inspect this");
|
||||
expect(context.BodyForAgent).not.toContain("<media:image>");
|
||||
expect(context.MediaPath).toBeUndefined();
|
||||
});
|
||||
|
||||
it("combines raw and command text across failed-media debounce batches", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const handler = createSignalEventHandler(
|
||||
createBaseSignalEventHandlerDeps({
|
||||
cfg: {
|
||||
messages: { inbound: { debounceMs: 10 } },
|
||||
channels: { signal: { dmPolicy: "open", allowFrom: ["*"] } },
|
||||
},
|
||||
ignoreAttachments: false,
|
||||
fetchAttachment: async () => {
|
||||
throw new Error("expired attachment");
|
||||
},
|
||||
historyLimit: 0,
|
||||
}),
|
||||
);
|
||||
|
||||
await handler(
|
||||
createSignalReceiveEvent({
|
||||
dataMessage: {
|
||||
message: "first request",
|
||||
attachments: [{ id: "a1", contentType: "image/jpeg" }],
|
||||
},
|
||||
}),
|
||||
);
|
||||
await handler(
|
||||
createSignalReceiveEvent({
|
||||
dataMessage: {
|
||||
message: "second request",
|
||||
attachments: [],
|
||||
},
|
||||
}),
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
|
||||
const context = requireCapturedContext();
|
||||
expect(context.BodyForAgent).toContain("[signal attachment unavailable]");
|
||||
expect(context.RawBody).toBe("first request\\nsecond request");
|
||||
expect(context.CommandBody).toBe("first request\\nsecond request");
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("dispatches failed-media commands without text debounce", async () => {
|
||||
const handler = createSignalEventHandler(
|
||||
createBaseSignalEventHandlerDeps({
|
||||
cfg: {
|
||||
messages: { inbound: { debounceMs: 60_000 } },
|
||||
channels: { signal: { dmPolicy: "open", allowFrom: ["*"] } },
|
||||
},
|
||||
ignoreAttachments: false,
|
||||
fetchAttachment: async () => {
|
||||
throw new Error("expired attachment");
|
||||
},
|
||||
historyLimit: 0,
|
||||
}),
|
||||
);
|
||||
|
||||
await handler(
|
||||
createSignalReceiveEvent({
|
||||
dataMessage: {
|
||||
message: "/stop",
|
||||
attachments: [{ id: "a1", contentType: "image/jpeg" }],
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const context = requireCapturedContext();
|
||||
expect(context.CommandBody).toBe("/stop");
|
||||
expect(context.RawBody).toBe("/stop");
|
||||
expect(context.BodyForAgent).toBe("/stop\n\n[signal attachment unavailable]");
|
||||
});
|
||||
|
||||
it("threads resolved audio contentType for Signal voice attachments", async () => {
|
||||
const handler = createSignalEventHandler(
|
||||
createBaseSignalEventHandlerDeps({
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import {
|
|||
buildMentionRegexes,
|
||||
buildChannelInboundEventContext,
|
||||
createChannelInboundDebouncer,
|
||||
formatInboundMediaUnavailableText,
|
||||
formatInboundEnvelope,
|
||||
formatInboundFromLabel,
|
||||
matchesMentionPatterns,
|
||||
|
|
@ -334,7 +335,7 @@ export function createSignalEventHandler(deps: SignalEventHandlerDeps) {
|
|||
body: combinedBody,
|
||||
bodyForAgent: entry.bodyText,
|
||||
inboundHistory,
|
||||
rawBody: entry.bodyText,
|
||||
rawBody: entry.commandBody,
|
||||
commandBody: entry.commandBody,
|
||||
},
|
||||
access: {
|
||||
|
|
@ -514,7 +515,7 @@ export function createSignalEventHandler(deps: SignalEventHandlerDeps) {
|
|||
ingest: () => ({
|
||||
id: entry.messageId ?? `${entry.timestamp ?? Date.now()}`,
|
||||
timestamp: entry.timestamp,
|
||||
rawText: entry.bodyText,
|
||||
rawText: entry.commandBody,
|
||||
raw: entry,
|
||||
}),
|
||||
resolveTurn: () => ({
|
||||
|
|
@ -644,7 +645,7 @@ export function createSignalEventHandler(deps: SignalEventHandlerDeps) {
|
|||
},
|
||||
shouldDebounce: (entry) => {
|
||||
return shouldDebounceTextInbound({
|
||||
text: entry.bodyText,
|
||||
text: entry.commandBody,
|
||||
cfg: deps.cfg,
|
||||
hasMedia: Boolean(entry.mediaPath || entry.mediaType || entry.mediaPaths?.length),
|
||||
});
|
||||
|
|
@ -662,12 +663,17 @@ export function createSignalEventHandler(deps: SignalEventHandlerDeps) {
|
|||
.map((entry) => entry.bodyText)
|
||||
.filter(Boolean)
|
||||
.join("\\n");
|
||||
const combinedCommandBody = entries
|
||||
.map((entry) => entry.commandBody)
|
||||
.filter(Boolean)
|
||||
.join("\\n");
|
||||
if (!combinedText.trim()) {
|
||||
return;
|
||||
}
|
||||
await handleSignalInboundMessage({
|
||||
...last,
|
||||
bodyText: combinedText,
|
||||
commandBody: combinedCommandBody,
|
||||
mediaPath: undefined,
|
||||
mediaType: undefined,
|
||||
mediaPaths: undefined,
|
||||
|
|
@ -1064,9 +1070,11 @@ export function createSignalEventHandler(deps: SignalEventHandlerDeps) {
|
|||
const mediaTypes: string[] = [];
|
||||
let placeholder = "";
|
||||
const attachments = dataMessage.attachments ?? [];
|
||||
let unavailableAttachmentCount = deps.ignoreAttachments ? attachments.length : 0;
|
||||
if (!deps.ignoreAttachments) {
|
||||
for (const attachment of attachments) {
|
||||
if (!attachment?.id) {
|
||||
unavailableAttachmentCount += 1;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
|
|
@ -1087,8 +1095,11 @@ export function createSignalEventHandler(deps: SignalEventHandlerDeps) {
|
|||
mediaPath = fetched.path;
|
||||
mediaType = fetched.contentType ?? attachment.contentType ?? undefined;
|
||||
}
|
||||
} else {
|
||||
unavailableAttachmentCount += 1;
|
||||
}
|
||||
} catch (err) {
|
||||
unavailableAttachmentCount += 1;
|
||||
deps.runtime.error?.(danger(`attachment fetch failed: ${String(err)}`));
|
||||
}
|
||||
}
|
||||
|
|
@ -1100,12 +1111,19 @@ export function createSignalEventHandler(deps: SignalEventHandlerDeps) {
|
|||
const kind = kindFromMime(mediaType ?? undefined);
|
||||
if (kind) {
|
||||
placeholder = `<media:${kind}>`;
|
||||
} else if (attachments.length) {
|
||||
} else if (mediaPaths.length > 0) {
|
||||
placeholder = "<media:attachment>";
|
||||
}
|
||||
}
|
||||
|
||||
const bodyText = messageText || placeholder || visibleQuoteText || "";
|
||||
let bodyText = messageText || placeholder || visibleQuoteText || "";
|
||||
if (unavailableAttachmentCount > 0) {
|
||||
const attachmentLabel = unavailableAttachmentCount === 1 ? "attachment" : "attachments";
|
||||
bodyText = formatInboundMediaUnavailableText({
|
||||
body: bodyText,
|
||||
notice: `[signal ${unavailableAttachmentCount > 1 ? `${unavailableAttachmentCount} ` : ""}${attachmentLabel} unavailable]`,
|
||||
});
|
||||
}
|
||||
if (!bodyText) {
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -998,6 +998,17 @@ describe("web auto-reply connection", () => {
|
|||
});
|
||||
|
||||
expect(capture.getLastOptions()?.shouldDebounce?.(msg)).toBe(true);
|
||||
expect(
|
||||
capture.getLastOptions()?.shouldDebounce?.(
|
||||
createTestWebInboundMessage({
|
||||
payload: {
|
||||
body: "/stop\n\n[whatsapp attachment unavailable]",
|
||||
commandBody: "/stop",
|
||||
},
|
||||
platform: { sendComposing, reply, sendMedia },
|
||||
}),
|
||||
),
|
||||
).toBe(false);
|
||||
await onMessage(msg);
|
||||
|
||||
expect(reply).toHaveBeenCalledWith("ok", undefined);
|
||||
|
|
|
|||
|
|
@ -267,7 +267,10 @@ export async function monitorWebChannel(
|
|||
if (normalized.quote?.id || normalized.quote?.body) {
|
||||
return false;
|
||||
}
|
||||
return !isControlCommandMessage(normalized.payload.body, cfg);
|
||||
return !isControlCommandMessage(
|
||||
normalized.payload.commandBody ?? normalized.payload.body,
|
||||
cfg,
|
||||
);
|
||||
};
|
||||
|
||||
let connection;
|
||||
|
|
|
|||
|
|
@ -187,7 +187,13 @@ export async function applyGroupGating(params: ApplyGroupGatingParams) {
|
|||
const mentionMsg: AdmittedWebInboundMessage =
|
||||
params.mentionText !== undefined
|
||||
? { ...params.msg, payload: { ...params.msg.payload, body: params.mentionText } }
|
||||
: params.msg;
|
||||
: {
|
||||
...params.msg,
|
||||
payload: {
|
||||
...params.msg.payload,
|
||||
body: params.msg.payload.commandBody ?? params.msg.payload.body,
|
||||
},
|
||||
};
|
||||
const commandBody = stripMentionsForCommand(
|
||||
mentionMsg.payload.body,
|
||||
mentionConfig.mentionRegexes,
|
||||
|
|
|
|||
|
|
@ -173,7 +173,7 @@ function makePolicy(account: ReturnType<typeof makeAccount>) {
|
|||
|
||||
const GROUP_JID = "123@g.us";
|
||||
|
||||
function makeBaseMsg(overrides: { body?: string } = {}) {
|
||||
function makeBaseMsg(overrides: { body?: string; commandBody?: string } = {}) {
|
||||
const body = overrides.body ?? "hi";
|
||||
return createTestWebInboundMessage({
|
||||
event: {
|
||||
|
|
@ -182,6 +182,7 @@ function makeBaseMsg(overrides: { body?: string } = {}) {
|
|||
},
|
||||
payload: {
|
||||
body,
|
||||
commandBody: overrides.commandBody,
|
||||
},
|
||||
platform: {
|
||||
chatJid: GROUP_JID,
|
||||
|
|
@ -328,6 +329,27 @@ describe("processMessage group system prompt wiring", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("keeps generated media notices out of command input", async () => {
|
||||
resolvePolicyMock.mockReturnValue(makePolicy(makeAccount()));
|
||||
isControlCommandMessageMock.mockReturnValue(true);
|
||||
shouldComputeCommandAuthorizedMock.mockReturnValue(true);
|
||||
|
||||
await callProcessMessage({
|
||||
msg: makeBaseMsg({
|
||||
body: "/reset\n\n[whatsapp attachment unavailable]",
|
||||
commandBody: "/reset",
|
||||
}),
|
||||
});
|
||||
|
||||
expect(shouldComputeCommandAuthorizedMock).toHaveBeenCalledWith("/reset", {});
|
||||
expect(isControlCommandMessageMock).toHaveBeenCalledWith("/reset", {});
|
||||
expect(buildContextMock.mock.calls[0][0]).toMatchObject({
|
||||
bodyForAgent: "/reset\n\n[whatsapp attachment unavailable]",
|
||||
commandBody: "/reset",
|
||||
rawBody: "/reset",
|
||||
});
|
||||
});
|
||||
|
||||
it("checks auth for inline command tokens without marking them as command-source turns", async () => {
|
||||
resolvePolicyMock.mockReturnValue(makePolicy(makeAccount()));
|
||||
isControlCommandMessageMock.mockReturnValue(false);
|
||||
|
|
|
|||
|
|
@ -422,16 +422,14 @@ export async function processMessage(params: {
|
|||
}
|
||||
|
||||
const sender = getSenderIdentity(params.msg);
|
||||
const commandBody = params.msg.payload.commandBody ?? params.msg.payload.body;
|
||||
const dmRouteTarget = resolveWhatsAppDmRouteTarget({
|
||||
msg: params.msg,
|
||||
senderE164: sender.e164 ?? undefined,
|
||||
normalizeE164,
|
||||
});
|
||||
const shouldCheckCommandAuth = shouldComputeCommandAuthorized(
|
||||
params.msg.payload.body,
|
||||
params.cfg,
|
||||
);
|
||||
const isTextCommand = isControlCommandMessage(params.msg.payload.body, params.cfg);
|
||||
const shouldCheckCommandAuth = shouldComputeCommandAuthorized(commandBody, params.cfg);
|
||||
const isTextCommand = isControlCommandMessage(commandBody, params.cfg);
|
||||
const commandAuthorized = shouldCheckCommandAuth
|
||||
? await resolveWhatsAppCommandAuthorized({
|
||||
cfg: params.cfg,
|
||||
|
|
@ -445,13 +443,13 @@ export async function processMessage(params: {
|
|||
kind: "text-slash",
|
||||
source: "text",
|
||||
authorized: Boolean(commandAuthorized),
|
||||
body: params.msg.payload.body,
|
||||
body: commandBody,
|
||||
}
|
||||
: {
|
||||
kind: "normal",
|
||||
source: "message",
|
||||
authorized: false,
|
||||
body: params.msg.payload.body,
|
||||
body: commandBody,
|
||||
};
|
||||
const { onModelSelected, ...replyPipeline } = createChannelMessageReplyPipeline({
|
||||
cfg: params.cfg,
|
||||
|
|
@ -485,14 +483,14 @@ export async function processMessage(params: {
|
|||
const ctxPayload = await buildWhatsAppInboundContext({
|
||||
bodyForAgent: msgForAgent.payload.body,
|
||||
combinedBody,
|
||||
commandBody: params.msg.payload.body,
|
||||
commandBody,
|
||||
commandAuthorized,
|
||||
commandTurn,
|
||||
groupHistory: visibleGroupHistory,
|
||||
groupMemberRoster: params.groupMemberNames.get(params.groupHistoryKey),
|
||||
groupSystemPrompt: conversationSystemPrompt,
|
||||
msg: params.msg,
|
||||
rawBody: params.msg.payload.body,
|
||||
rawBody: commandBody,
|
||||
route: params.route,
|
||||
sender: {
|
||||
id: getPrimaryIdentityId(sender) ?? undefined,
|
||||
|
|
|
|||
|
|
@ -67,6 +67,7 @@ function createInMemoryKeyedStore<T>() {
|
|||
const readAllowFromStoreMock = vi.fn().mockResolvedValue([]);
|
||||
const upsertPairingRequestMock = vi.fn().mockResolvedValue({ code: "PAIRCODE", created: true });
|
||||
const saveMediaStreamSpy = vi.fn();
|
||||
const downloadMediaMessageMock = vi.hoisted(() => vi.fn());
|
||||
let currentMockSocket:
|
||||
| {
|
||||
ev: import("node:events").EventEmitter;
|
||||
|
|
@ -185,7 +186,9 @@ vi.mock("baileys", async () => {
|
|||
return {
|
||||
...actual,
|
||||
DisconnectReason: actual.DisconnectReason ?? { loggedOut: 401 },
|
||||
downloadMediaMessage: vi.fn().mockImplementation(() => Readable.from([jpegBuffer])),
|
||||
downloadMediaMessage: downloadMediaMessageMock.mockImplementation(() =>
|
||||
Readable.from([jpegBuffer]),
|
||||
),
|
||||
extractMessageContent: vi.fn((message: MockMessageInput) => mockExtractMessageContent(message)),
|
||||
getContentType: vi.fn((message: MockMessageInput) => mockGetContentType(message)),
|
||||
isJidGroup: vi.fn((jid: string | undefined | null) => mockIsJidGroup(jid)),
|
||||
|
|
@ -257,6 +260,7 @@ describe("web inbound media saves with extension", () => {
|
|||
beforeEach(() => {
|
||||
vi.useRealTimers();
|
||||
currentMockSocket = undefined;
|
||||
downloadMediaMessageMock.mockClear();
|
||||
saveMediaStreamSpy.mockClear();
|
||||
resetWebInboundDedupe();
|
||||
});
|
||||
|
|
@ -436,4 +440,38 @@ describe("web inbound media saves with extension", () => {
|
|||
|
||||
await listener.close();
|
||||
});
|
||||
|
||||
it("replaces a failed image placeholder with an unavailable notice", async () => {
|
||||
downloadMediaMessageMock.mockRejectedValueOnce(new Error("expired media reference"));
|
||||
const onMessage = vi.fn();
|
||||
const listener = await monitorWebInbox({
|
||||
cfg: {
|
||||
channels: { whatsapp: { allowFrom: ["*"] } },
|
||||
messages: { messagePrefix: undefined, responsePrefix: undefined },
|
||||
} as never,
|
||||
verbose: false,
|
||||
onMessage,
|
||||
accountId: "default",
|
||||
authDir: path.join(HOME, "wa-auth"),
|
||||
});
|
||||
const realSock = await getMockSocket();
|
||||
|
||||
realSock.ev.emit("messages.upsert", {
|
||||
type: "notify",
|
||||
messages: [
|
||||
{
|
||||
key: { id: "img-failed", fromMe: false, remoteJid: "111@s.whatsapp.net" },
|
||||
message: { imageMessage: { mimetype: "image/jpeg" } },
|
||||
messageTimestamp: 1_700_000_006,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const inbound = await waitForMessage(onMessage);
|
||||
expect(inbound.payload.body).toBe("[whatsapp attachment unavailable]");
|
||||
expect(inbound.payload.commandBody).toBe("<media:image>");
|
||||
expect(inbound.payload.media).toBeUndefined();
|
||||
|
||||
await listener.close();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -136,4 +136,15 @@ describe("downloadInboundMedia", () => {
|
|||
).rejects.toThrow(/Media exceeds/i);
|
||||
expect(downloadMediaMessage.mock.calls[0]?.[1]).toBe("stream");
|
||||
});
|
||||
|
||||
it("propagates transport download failures to the message owner", async () => {
|
||||
downloadMediaMessage.mockRejectedValueOnce(new Error("expired media reference"));
|
||||
|
||||
await expect(
|
||||
downloadInboundMedia(
|
||||
{ message: { imageMessage: { mimetype: "image/jpeg" } } } as never,
|
||||
mockSock as never,
|
||||
),
|
||||
).rejects.toThrow("expired media reference");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
// Whatsapp plugin module implements media behavior.
|
||||
import type { proto, WAMessage } from "baileys";
|
||||
import { saveMediaStream, type SavedMedia } from "openclaw/plugin-sdk/media-store";
|
||||
import { logVerbose } from "openclaw/plugin-sdk/runtime-env";
|
||||
import type { createWaSocket } from "../session.js";
|
||||
import { extractContextInfo } from "./extract.js";
|
||||
import { resolveInboundMediaMimetype } from "./media-mimetype.js";
|
||||
|
|
@ -39,36 +38,28 @@ export async function downloadInboundMedia(
|
|||
) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
const stream = await downloadMediaMessage(
|
||||
msg as WAMessage,
|
||||
"stream",
|
||||
{},
|
||||
{
|
||||
reuploadRequest: sock.updateMediaMessage,
|
||||
logger: sock.logger,
|
||||
},
|
||||
);
|
||||
const saved = await saveMediaStream(
|
||||
stream as AsyncIterable<unknown>,
|
||||
mimetype,
|
||||
"inbound",
|
||||
maxBytes,
|
||||
fileName,
|
||||
).catch((err: unknown) => {
|
||||
if (err instanceof Error && /Media exceeds/i.test(err.message)) {
|
||||
throw new WhatsAppInboundMediaLimitExceededError(maxBytes);
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
return { saved, mimetype, fileName };
|
||||
} catch (err) {
|
||||
if (err instanceof WhatsAppInboundMediaLimitExceededError) {
|
||||
throw err;
|
||||
const stream = await downloadMediaMessage(
|
||||
msg as WAMessage,
|
||||
"stream",
|
||||
{},
|
||||
{
|
||||
reuploadRequest: sock.updateMediaMessage,
|
||||
logger: sock.logger,
|
||||
},
|
||||
);
|
||||
const saved = await saveMediaStream(
|
||||
stream as AsyncIterable<unknown>,
|
||||
mimetype,
|
||||
"inbound",
|
||||
maxBytes,
|
||||
fileName,
|
||||
).catch((err: unknown) => {
|
||||
if (err instanceof Error && /Media exceeds/i.test(err.message)) {
|
||||
throw new WhatsAppInboundMediaLimitExceededError(maxBytes);
|
||||
}
|
||||
logVerbose(`downloadMediaMessage failed: ${String(err)}`);
|
||||
return undefined;
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
return { saved, mimetype, fileName };
|
||||
}
|
||||
|
||||
export async function downloadQuotedInboundMedia(
|
||||
|
|
|
|||
|
|
@ -9,7 +9,10 @@ import type {
|
|||
WASocket,
|
||||
} from "baileys";
|
||||
import { recordChannelActivity } from "openclaw/plugin-sdk/channel-activity-runtime";
|
||||
import { formatLocationText } from "openclaw/plugin-sdk/channel-inbound";
|
||||
import {
|
||||
formatInboundMediaUnavailableText,
|
||||
formatLocationText,
|
||||
} from "openclaw/plugin-sdk/channel-inbound";
|
||||
import { createInboundDebouncer } from "openclaw/plugin-sdk/channel-inbound-debounce";
|
||||
import { getChildLogger } from "openclaw/plugin-sdk/logging-core";
|
||||
import {
|
||||
|
|
@ -510,6 +513,10 @@ export async function attachWebInboxToSocket(
|
|||
.map((entry) => entry.payload.body)
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
const combinedCommandBody = orderedEntries
|
||||
.map((entry) => entry.payload.commandBody ?? entry.payload.body)
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
const combinedMentions =
|
||||
mentioned.size > 0
|
||||
? {
|
||||
|
|
@ -529,6 +536,7 @@ export async function attachWebInboxToSocket(
|
|||
payload: {
|
||||
...last.payload,
|
||||
body: combinedBody,
|
||||
commandBody: combinedCommandBody,
|
||||
},
|
||||
group: combinedGroup,
|
||||
event: {
|
||||
|
|
@ -1101,6 +1109,7 @@ export async function attachWebInboxToSocket(
|
|||
|
||||
type EnrichedInboundMessage = {
|
||||
body: string;
|
||||
commandBody: string;
|
||||
location?: ReturnType<typeof extractLocationData>;
|
||||
contactContext?: ReturnType<typeof extractContactContext>;
|
||||
externalAdReplyContext?: ReturnType<typeof extractExternalAdReplyContext>;
|
||||
|
|
@ -1115,16 +1124,18 @@ export async function attachWebInboxToSocket(
|
|||
const locationText = location ? formatLocationText(location) : undefined;
|
||||
const contactContext = extractContactContext(msg.message ?? undefined);
|
||||
const externalAdReplyContext = extractExternalAdReplyContext(msg.message ?? undefined);
|
||||
const mediaPlaceholder = extractMediaPlaceholder(msg.message ?? undefined);
|
||||
let body = extractText(msg.message ?? undefined);
|
||||
if (locationText) {
|
||||
body = [body, locationText].filter(Boolean).join("\n").trim();
|
||||
}
|
||||
if (!body) {
|
||||
body = extractMediaPlaceholder(msg.message ?? undefined);
|
||||
body = mediaPlaceholder;
|
||||
if (!body) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
const commandBody = body;
|
||||
const replyContext = describeReplyContext(msg.message as proto.IMessage | undefined);
|
||||
|
||||
let mediaPath: string | undefined;
|
||||
|
|
@ -1146,17 +1157,31 @@ export async function attachWebInboxToSocket(
|
|||
try {
|
||||
const inboundMedia = await downloadInboundMedia(msg as proto.IWebMessageInfo, sock, maxBytes);
|
||||
await saveInboundMedia(inboundMedia);
|
||||
if (!mediaPath && replyContext) {
|
||||
} catch (err) {
|
||||
logWhatsAppVerbose(options.verbose, `Inbound media download failed: ${String(err)}`);
|
||||
body = formatInboundMediaUnavailableText({
|
||||
body,
|
||||
mediaPlaceholder,
|
||||
notice: "[whatsapp attachment unavailable]",
|
||||
});
|
||||
}
|
||||
if (!mediaPath && replyContext) {
|
||||
try {
|
||||
await saveInboundMedia(
|
||||
await downloadQuotedInboundMedia(msg as proto.IWebMessageInfo, sock, maxBytes),
|
||||
);
|
||||
} catch (err) {
|
||||
logWhatsAppVerbose(options.verbose, `Quoted media download failed: ${String(err)}`);
|
||||
body = formatInboundMediaUnavailableText({
|
||||
body,
|
||||
notice: "[whatsapp quoted attachment unavailable]",
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
logWhatsAppVerbose(options.verbose, `Inbound media download failed: ${String(err)}`);
|
||||
}
|
||||
|
||||
return {
|
||||
body,
|
||||
commandBody,
|
||||
location: location ?? undefined,
|
||||
contactContext,
|
||||
externalAdReplyContext,
|
||||
|
|
@ -1272,6 +1297,7 @@ export async function attachWebInboxToSocket(
|
|||
},
|
||||
payload: {
|
||||
body: enriched.body,
|
||||
commandBody: enriched.commandBody,
|
||||
location: enriched.location ?? undefined,
|
||||
untrustedStructuredContext:
|
||||
untrustedStructuredContext.length > 0 ? untrustedStructuredContext : undefined,
|
||||
|
|
|
|||
|
|
@ -86,6 +86,7 @@ export type WhatsAppInboundGroupContext = {
|
|||
|
||||
export type WhatsAppInboundPayload = {
|
||||
body: string;
|
||||
commandBody?: string;
|
||||
media?: {
|
||||
path?: string;
|
||||
type?: string;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
// Zalo tests cover monitor.image.polling plugin behavior.
|
||||
import { createRuntimeEnv } from "openclaw/plugin-sdk/plugin-test-runtime";
|
||||
import { afterAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { afterAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
createImageLifecycleCore,
|
||||
createImageUpdate,
|
||||
|
|
@ -114,4 +114,42 @@ describe("Zalo polling image handling", () => {
|
|||
abort.abort();
|
||||
await run;
|
||||
});
|
||||
|
||||
it("dispatches an unavailable notice when the inbound image download fails", async () => {
|
||||
saveRemoteMediaMock.mockRejectedValueOnce(new Error("expired image URL"));
|
||||
getUpdatesMock
|
||||
.mockResolvedValueOnce({
|
||||
ok: true,
|
||||
result: createImageUpdate({ caption: "/reset" }),
|
||||
})
|
||||
.mockImplementation(() => new Promise(() => {}));
|
||||
|
||||
const { monitorZaloProvider } = await loadCachedLifecycleMonitorModule("zalo-image-polling");
|
||||
const abort = new AbortController();
|
||||
const runtime = createRuntimeEnv();
|
||||
const { account, config } = createLifecycleMonitorSetup({
|
||||
accountId: "default",
|
||||
dmPolicy: "open",
|
||||
});
|
||||
const run = monitorZaloProvider({
|
||||
token: "zalo-token", // pragma: allowlist secret
|
||||
account,
|
||||
config,
|
||||
runtime,
|
||||
abortSignal: abort.signal,
|
||||
});
|
||||
|
||||
await vi.waitFor(() => expect(finalizeInboundContextMock).toHaveBeenCalledTimes(1));
|
||||
expect(finalizeInboundContextMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
RawBody: "/reset",
|
||||
CommandBody: "/reset",
|
||||
BodyForAgent: "/reset\n\n[zalo image attachment unavailable]",
|
||||
MediaPath: undefined,
|
||||
}),
|
||||
);
|
||||
|
||||
abort.abort();
|
||||
await run;
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
// Zalo plugin module implements monitor behavior.
|
||||
import type { IncomingMessage, ServerResponse } from "node:http";
|
||||
import { logTypingFailure } from "openclaw/plugin-sdk/channel-feedback";
|
||||
import { formatInboundMediaUnavailableText } from "openclaw/plugin-sdk/channel-inbound";
|
||||
import { resolveStableChannelMessageIngress } from "openclaw/plugin-sdk/channel-ingress-runtime";
|
||||
import { createChannelPairingController } from "openclaw/plugin-sdk/channel-pairing";
|
||||
import type { MarkdownTableMode, OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
|
|
@ -154,6 +155,7 @@ function registerSharedHostedMediaRoute(params: {
|
|||
type ZaloMessagePipelineParams = ZaloProcessingContext & {
|
||||
message: ZaloMessage;
|
||||
text?: string;
|
||||
agentBody?: string;
|
||||
mediaPath?: string;
|
||||
mediaType?: string;
|
||||
authorization?: ZaloMessageAuthorizationResult;
|
||||
|
|
@ -363,7 +365,7 @@ async function handleImageMessage(params: ZaloImageMessageParams): Promise<void>
|
|||
...params,
|
||||
text: caption,
|
||||
// Use a sentinel so auth sees this as an inbound image before the download happens.
|
||||
mediaPath: photo_url ? "__pending_media__" : undefined,
|
||||
mediaPath: "__pending_media__",
|
||||
mediaType: undefined,
|
||||
});
|
||||
if (!authorization) {
|
||||
|
|
@ -384,9 +386,18 @@ async function handleImageMessage(params: ZaloImageMessageParams): Promise<void>
|
|||
}
|
||||
}
|
||||
|
||||
const agentBody = mediaPath
|
||||
? authorization.rawBody
|
||||
: formatInboundMediaUnavailableText({
|
||||
body: caption,
|
||||
mediaPlaceholder: "<media:image>",
|
||||
notice: "[zalo image attachment unavailable]",
|
||||
});
|
||||
|
||||
await processMessageWithPipeline({
|
||||
...params,
|
||||
authorization,
|
||||
agentBody,
|
||||
text: caption,
|
||||
mediaPath,
|
||||
mediaType,
|
||||
|
|
@ -531,6 +542,7 @@ async function processMessageWithPipeline(params: ZaloMessagePipelineParams): Pr
|
|||
mediaType,
|
||||
statusSink,
|
||||
fetcher,
|
||||
agentBody: agentBodyOverride,
|
||||
authorization: authorizationOverride,
|
||||
} = params;
|
||||
const { message_id, date } = message;
|
||||
|
|
@ -545,6 +557,7 @@ async function processMessageWithPipeline(params: ZaloMessagePipelineParams): Pr
|
|||
return;
|
||||
}
|
||||
const { isGroup, chatId, senderId, senderName, rawBody, commandAuthorized } = authorization;
|
||||
const agentBody = agentBodyOverride ?? rawBody;
|
||||
|
||||
const { route, buildEnvelope } = resolveInboundRouteEnvelopeBuilderWithRuntime({
|
||||
cfg: config,
|
||||
|
|
@ -573,7 +586,7 @@ async function processMessageWithPipeline(params: ZaloMessagePipelineParams): Pr
|
|||
channel: "Zalo",
|
||||
from: fromLabel,
|
||||
timestamp,
|
||||
body: rawBody,
|
||||
body: agentBody,
|
||||
});
|
||||
|
||||
const ctxPayload = core.channel.inbound.buildContext({
|
||||
|
|
@ -601,7 +614,7 @@ async function processMessageWithPipeline(params: ZaloMessagePipelineParams): Pr
|
|||
},
|
||||
message: {
|
||||
body,
|
||||
bodyForAgent: rawBody,
|
||||
bodyForAgent: agentBody,
|
||||
rawBody,
|
||||
commandBody: rawBody,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -102,6 +102,7 @@ export function createImageUpdate(params?: {
|
|||
displayName?: string;
|
||||
chatId?: string;
|
||||
photoUrl?: string;
|
||||
caption?: string;
|
||||
date?: number;
|
||||
}) {
|
||||
return {
|
||||
|
|
@ -109,7 +110,7 @@ export function createImageUpdate(params?: {
|
|||
message: {
|
||||
date: params?.date ?? 1774086023728,
|
||||
chat: { chat_type: "PRIVATE" as const, id: params?.chatId ?? "chat-123" },
|
||||
caption: "",
|
||||
caption: params?.caption ?? "",
|
||||
message_id: params?.messageId ?? "msg-123",
|
||||
message_type: "CHAT_PHOTO",
|
||||
from: {
|
||||
|
|
|
|||
|
|
@ -202,8 +202,8 @@ let publicDeprecatedExportsByEntrypointBudget;
|
|||
try {
|
||||
budgets = {
|
||||
publicEntrypoints: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_ENTRYPOINTS", 324),
|
||||
publicExports: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_EXPORTS", 10428),
|
||||
publicFunctionExports: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_FUNCTION_EXPORTS", 5239),
|
||||
publicExports: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_EXPORTS", 10429),
|
||||
publicFunctionExports: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_FUNCTION_EXPORTS", 5240),
|
||||
publicDeprecatedExports: readBudgetEnv(
|
||||
"OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_DEPRECATED_EXPORTS",
|
||||
3261,
|
||||
|
|
|
|||
|
|
@ -3,11 +3,29 @@ import { describe, expect, it } from "vitest";
|
|||
import { normalizeAttachments } from "../../media-understanding/attachments.normalize.js";
|
||||
import {
|
||||
buildChannelInboundMediaPayload,
|
||||
formatInboundMediaUnavailableText,
|
||||
toHistoryMediaEntries,
|
||||
toInboundMediaFacts,
|
||||
} from "./media.js";
|
||||
|
||||
describe("channel inbound media facts", () => {
|
||||
it("replaces optimistic media placeholders and preserves real captions", () => {
|
||||
expect(
|
||||
formatInboundMediaUnavailableText({
|
||||
body: "<media:image>",
|
||||
mediaPlaceholder: "<media:image>",
|
||||
notice: "[test image attachment unavailable]",
|
||||
}),
|
||||
).toBe("[test image attachment unavailable]");
|
||||
expect(
|
||||
formatInboundMediaUnavailableText({
|
||||
body: "please inspect this",
|
||||
mediaPlaceholder: "<media:image>",
|
||||
notice: "[test image attachment unavailable]",
|
||||
}),
|
||||
).toBe("please inspect this\n\n[test image attachment unavailable]");
|
||||
});
|
||||
|
||||
it("normalizes provider media into inbound media facts", () => {
|
||||
expect(
|
||||
toInboundMediaFacts(
|
||||
|
|
|
|||
|
|
@ -32,6 +32,24 @@ export type ChannelInboundMediaPayload = {
|
|||
MediaTranscribedIndexes?: number[];
|
||||
};
|
||||
|
||||
/**
|
||||
* Replaces an optimistic media placeholder, or appends to real caption text,
|
||||
* when transport media could not be materialized for the agent turn.
|
||||
*/
|
||||
export function formatInboundMediaUnavailableText(params: {
|
||||
body?: string | null;
|
||||
mediaPlaceholder?: string | null;
|
||||
notice: string;
|
||||
}): string {
|
||||
const body = params.body?.trim() ?? "";
|
||||
const placeholder = params.mediaPlaceholder?.trim() ?? "";
|
||||
const notice = params.notice.trim();
|
||||
if (!body || (placeholder && body === placeholder)) {
|
||||
return notice;
|
||||
}
|
||||
return `${body}\n\n${notice}`;
|
||||
}
|
||||
|
||||
function alignedStrings(values: Array<string | undefined>): string[] | undefined {
|
||||
if (!values.some(Boolean)) {
|
||||
return undefined;
|
||||
|
|
|
|||
|
|
@ -183,6 +183,7 @@ export {
|
|||
toHistoryMediaEntries,
|
||||
toInboundMediaFacts,
|
||||
buildChannelInboundMediaPayload,
|
||||
formatInboundMediaUnavailableText,
|
||||
// @deprecated Prefer `buildChannelInboundMediaPayload`.
|
||||
buildChannelInboundMediaPayload as buildChannelTurnMediaPayload,
|
||||
} from "../channels/inbound-event/media.js";
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue