diff --git a/.changeset/fix-provider-media-placeholders.md b/.changeset/fix-provider-media-placeholders.md new file mode 100644 index 000000000..e506b28ac --- /dev/null +++ b/.changeset/fix-provider-media-placeholders.md @@ -0,0 +1,6 @@ +--- +"@moonshot-ai/kosong": patch +"@moonshot-ai/kimi-code": patch +--- + +Degrade unsupported audio/video to placeholder text and reattach tool result media instead of silently dropping them. diff --git a/packages/kosong/src/providers/anthropic.ts b/packages/kosong/src/providers/anthropic.ts index 9bdbc15dd..f8a09239d 100644 --- a/packages/kosong/src/providers/anthropic.ts +++ b/packages/kosong/src/providers/anthropic.ts @@ -412,6 +412,15 @@ interface AnthropicImageBlock { cache_control?: { type: 'ephemeral' }; } +// The Messages API has no representation for audio or video input. Instead of +// silently dropping such parts (the model would not even know an attachment +// existed), emit a placeholder text block so it can acknowledge the gap. +// Consecutive parts of the same kind collapse into a single placeholder. +const OMITTED_MEDIA_PLACEHOLDER = { + audio_url: '(audio omitted: not supported by this provider)', + video_url: '(video omitted: not supported by this provider)', +} as const; + const SUPPORTED_B64_MEDIA_TYPES = new Set(['image/png', 'image/jpeg', 'image/gif', 'image/webp']); function imageUrlPartToAnthropic(url: string): AnthropicImageBlock { @@ -458,8 +467,13 @@ function toolResultToBlock(toolCallId: string, content: ContentPart[]): ToolResu } } else if (part.type === 'image_url') { blocks.push(imageUrlPartToAnthropic(part.imageUrl.url)); + } else if (part.type === 'audio_url' || part.type === 'video_url') { + const placeholder = OMITTED_MEDIA_PLACEHOLDER[part.type]; + const last = blocks.at(-1); + if (!(last?.type === 'text' && last.text === placeholder)) { + blocks.push({ type: 'text', text: placeholder }); + } } - // Other types not supported by Anthropic in tool results } return { type: 'tool_result', @@ -522,8 +536,13 @@ function convertMessage(message: Message, model: string): MessageParam { } else if (part.think !== '' && shouldPreserveUnsignedThinking(model)) { blocks.push({ type: 'thinking', thinking: part.think } as unknown as ThinkingBlockParam); } + } else if (part.type === 'audio_url' || part.type === 'video_url') { + const placeholder = OMITTED_MEDIA_PLACEHOLDER[part.type]; + const last = blocks.at(-1); + if (!(last?.type === 'text' && last.text === placeholder)) { + blocks.push({ type: 'text', text: placeholder } satisfies TextBlockParam); + } } - // audio_url, video_url: not supported by Anthropic, skip } // Tool calls -> ToolUseBlockParam diff --git a/packages/kosong/src/providers/openai-common.ts b/packages/kosong/src/providers/openai-common.ts index 10fb42726..8727ef5ed 100644 --- a/packages/kosong/src/providers/openai-common.ts +++ b/packages/kosong/src/providers/openai-common.ts @@ -286,6 +286,18 @@ export function normalizeOpenAIFinishReason(raw: string | null | undefined): { */ export type ToolMessageConversion = 'extract_text' | null; +/** + * Shared wording for tool-result media that cannot live inside the tool + * message itself and is reattached as a follow-up user message instead. + */ +export const TOOL_RESULT_MEDIA_PROMPT = 'Attached media from tool result:'; +export const TOOL_RESULT_MEDIA_PLACEHOLDER = '(see attached media)'; + +/** A content part that is neither plain text nor reasoning. */ +export function isMediaPart(part: ContentPart): boolean { + return part.type !== 'text' && part.type !== 'think'; +} + /** * Convert tool-role message content according to the chosen strategy. */ diff --git a/packages/kosong/src/providers/openai-legacy.ts b/packages/kosong/src/providers/openai-legacy.ts index 5fb313c22..9bdca8bae 100644 --- a/packages/kosong/src/providers/openai-legacy.ts +++ b/packages/kosong/src/providers/openai-legacy.ts @@ -21,6 +21,8 @@ import { isFunctionToolCall, normalizeOpenAIFinishReason, type OpenAIContentPart, + TOOL_RESULT_MEDIA_PLACEHOLDER, + TOOL_RESULT_MEDIA_PROMPT, type ToolMessageConversion, reasoningEffortToThinkingEffort, thinkingEffortToReasoningEffort, @@ -50,8 +52,6 @@ const OPENAI_CHAT_TOOL_CALL_ID_POLICY: ToolCallIdPolicy = { normalize: (id) => sanitizeToolCallId(id, 64), maxLength: 64, }; -const TOOL_RESULT_IMAGE_PROMPT = 'Attached image(s) from tool result:'; -const TOOL_RESULT_IMAGE_PLACEHOLDER = '(see attached image)'; function extractReasoningContent( source: unknown, @@ -220,46 +220,55 @@ function convertMessage( return result; } +// Chat Completions has no url-based audio/video content part (only base64 +// `input_audio`), so unlike images these cannot be reattached as user input. +// Note the omission inline in the tool message text instead. +const OMITTED_AUDIO_PLACEHOLDER = '(audio omitted: not supported by this provider)'; +const OMITTED_VIDEO_PLACEHOLDER = '(video omitted: not supported by this provider)'; + function convertToolMessageContentForChat( message: Message, conversion: ToolMessageConversion, ): string | OpenAIContentPart[] { const content = convertToolMessageContent(message, conversion); - if ( - typeof content === 'string' && - content.length === 0 && - message.content.some((part) => part.type === 'image_url') - ) { - return TOOL_RESULT_IMAGE_PLACEHOLDER; + if (typeof content !== 'string') { + return content; } - return content; + const lines: string[] = content.length > 0 ? [content] : []; + if (message.content.some((part) => part.type === 'audio_url')) { + lines.push(OMITTED_AUDIO_PLACEHOLDER); + } + if (message.content.some((part) => part.type === 'video_url')) { + lines.push(OMITTED_VIDEO_PLACEHOLDER); + } + if (lines.length === 0 && message.content.some((part) => part.type === 'image_url')) { + return TOOL_RESULT_MEDIA_PLACEHOLDER; + } + return lines.join('\n'); } function toolResultImageParts(message: Message): OpenAIContentPart[] { const images: OpenAIContentPart[] = []; for (const part of message.content) { if (part.type !== 'image_url') continue; - images.push({ - type: 'image_url', - image_url: - part.imageUrl.id === undefined - ? { url: part.imageUrl.url } - : { url: part.imageUrl.url, id: part.imageUrl.id }, - }); + const converted = convertContentPart(part); + if (converted !== null) { + images.push(converted); + } } return images; } -function appendToolResultImagesMessage( +function appendToolResultMediaMessage( messages: OpenAIMessage[], - pendingToolResultImages: OpenAIContentPart[], + pendingToolResultMedia: OpenAIContentPart[], ): void { - if (pendingToolResultImages.length === 0) return; + if (pendingToolResultMedia.length === 0) return; messages.push({ role: 'user', - content: [{ type: 'text', text: TOOL_RESULT_IMAGE_PROMPT }, ...pendingToolResultImages], + content: [{ type: 'text', text: TOOL_RESULT_MEDIA_PROMPT }, ...pendingToolResultMedia], }); - pendingToolResultImages.length = 0; + pendingToolResultMedia.length = 0; } function convertHistoryMessages( @@ -268,19 +277,19 @@ function convertHistoryMessages( toolMessageConversion: ToolMessageConversion, ): OpenAIMessage[] { const messages: OpenAIMessage[] = []; - const pendingToolResultImages: OpenAIContentPart[] = []; + const pendingToolResultMedia: OpenAIContentPart[] = []; for (const msg of history) { if (msg.role !== 'tool') { - appendToolResultImagesMessage(messages, pendingToolResultImages); + appendToolResultMediaMessage(messages, pendingToolResultMedia); } messages.push(convertMessage(msg, reasoningKey, toolMessageConversion)); if (msg.role === 'tool') { - pendingToolResultImages.push(...toolResultImageParts(msg)); + pendingToolResultMedia.push(...toolResultImageParts(msg)); } } - appendToolResultImagesMessage(messages, pendingToolResultImages); + appendToolResultMediaMessage(messages, pendingToolResultMedia); return messages; } export class OpenAILegacyStreamedMessage implements StreamedMessage { diff --git a/packages/kosong/src/providers/openai-responses.ts b/packages/kosong/src/providers/openai-responses.ts index 1afa70696..b3804f025 100644 --- a/packages/kosong/src/providers/openai-responses.ts +++ b/packages/kosong/src/providers/openai-responses.ts @@ -25,6 +25,9 @@ import { } from './capability-registry'; import { convertOpenAIError, + isMediaPart, + TOOL_RESULT_MEDIA_PLACEHOLDER, + TOOL_RESULT_MEDIA_PROMPT, type ToolMessageConversion, reasoningEffortToThinkingEffort, thinkingEffortToReasoningEffort, @@ -363,6 +366,12 @@ interface ResponseToolParam { parameters: Record; strict: boolean; } +// The Responses API has no input type for video, and only mp3/wav audio can +// be inlined as input_file data. Degrade such parts to placeholder text so +// the model still learns an attachment existed instead of silently losing it. +const OMITTED_AUDIO_PLACEHOLDER = '(audio omitted: unsupported audio format)'; +const OMITTED_VIDEO_PLACEHOLDER = '(video omitted: not supported by this provider)'; + function contentPartsToInputItems(parts: ContentPart[]): unknown[] { const items: unknown[] = []; for (const part of parts) { @@ -381,14 +390,14 @@ function contentPartsToInputItems(parts: ContentPart[]): unknown[] { break; case 'audio_url': { const mapped = mapAudioUrlToInputItem(part.audioUrl.url); - if (mapped !== null) { - items.push(mapped); - } + items.push(mapped ?? { type: 'input_text', text: OMITTED_AUDIO_PLACEHOLDER }); break; } - case 'think': case 'video_url': - // think: handled separately. video_url: not supported by Responses API. + items.push({ type: 'input_text', text: OMITTED_VIDEO_PLACEHOLDER }); + break; + case 'think': + // Handled separately as reasoning items. break; } } @@ -405,7 +414,7 @@ function contentPartsToOutputItems(parts: ContentPart[]): unknown[] { return items; } -function messageContentToFunctionOutputItems(content: ContentPart[]): string | unknown[] { +function messageContentToFunctionOutputItems(content: ContentPart[]): unknown[] { const items: unknown[] = []; for (const part of content) { switch (part.type) { @@ -424,15 +433,14 @@ function messageContentToFunctionOutputItems(content: ContentPart[]): string | u // branch here, audio returned by a tool would be dropped on the // next turn. const mapped = mapAudioUrlToInputItem(part.audioUrl.url); - if (mapped !== null) { - items.push(mapped); - } + items.push(mapped ?? { type: 'input_text', text: OMITTED_AUDIO_PLACEHOLDER }); break; } - case 'think': case 'video_url': - // think / video_url still intentionally skipped: the Responses - // API has no representation for them inside a function_call_output. + items.push({ type: 'input_text', text: OMITTED_VIDEO_PLACEHOLDER }); + break; + case 'think': + // Handled separately as reasoning items. break; } } @@ -477,10 +485,20 @@ function convertMessage( // tool role -> function_call_output if (role === 'tool') { const callId = message.toolCallId ?? ''; - const output: string | unknown[] = - toolMessageConversion === 'extract_text' - ? extractText(message) - : messageContentToFunctionOutputItems(message.content); + let output: string | unknown[]; + if (toolMessageConversion === 'extract_text') { + // Plain-string output for backends that reject structured + // function_call_output. Media parts are reattached as a user message + // by `convertHistoryMessages`; when the result carries no text at + // all, point the model at that follow-up message. + const text = extractText(message); + output = + text.length === 0 && message.content.some(isMediaPart) + ? TOOL_RESULT_MEDIA_PLACEHOLDER + : text; + } else { + output = messageContentToFunctionOutputItems(message.content); + } return [ { call_id: callId, @@ -571,6 +589,49 @@ function convertTool(tool: Tool): ResponseToolParam { strict: false, }; } + +/** + * Convert the history, buffering tool-result media when `extract_text` + * flattens tool outputs to plain strings. The buffered media items are + * reattached as a single user message after each run of consecutive tool + * messages — mirroring the OpenAI Chat Completions provider. + */ +function convertHistoryMessages( + history: readonly Message[], + modelName: string, + toolMessageConversion: ToolMessageConversion, +): unknown[] { + const input: unknown[] = []; + const pendingToolResultMedia: unknown[] = []; + + const flushPendingMedia = (): void => { + if (pendingToolResultMedia.length === 0) return; + input.push({ + type: 'message', + role: 'user', + content: [ + { type: 'input_text', text: TOOL_RESULT_MEDIA_PROMPT }, + ...pendingToolResultMedia, + ], + }); + pendingToolResultMedia.length = 0; + }; + + for (const msg of history) { + if (msg.role !== 'tool') { + flushPendingMedia(); + } + input.push(...convertMessage(msg, modelName, toolMessageConversion)); + if (msg.role === 'tool' && toolMessageConversion === 'extract_text') { + pendingToolResultMedia.push( + ...messageContentToFunctionOutputItems(msg.content.filter(isMediaPart)), + ); + } + } + + flushPendingMedia(); + return input; +} export class OpenAIResponsesStreamedMessage implements StreamedMessage { private _id: string | null = null; private _usage: TokenUsage | null = null; @@ -991,9 +1052,9 @@ export class OpenAIResponsesChatProvider implements ChatProvider { history, OPENAI_RESPONSES_TOOL_CALL_ID_POLICY, ); - for (const msg of normalizedHistory) { - input.push(...convertMessage(msg, this._model, this._toolMessageConversion)); - } + input.push( + ...convertHistoryMessages(normalizedHistory, this._model, this._toolMessageConversion), + ); const kwargs: Record = { ...this._generationKwargs }; const reasoningEffort = kwargs['reasoning_effort'] as string | undefined; diff --git a/packages/kosong/test/anthropic.test.ts b/packages/kosong/test/anthropic.test.ts index fe6f2e29c..b9a10dc37 100644 --- a/packages/kosong/test/anthropic.test.ts +++ b/packages/kosong/test/anthropic.test.ts @@ -447,6 +447,73 @@ describe('AnthropicChatProvider', () => { }); }); + it('user audio/video parts degrade to placeholder text, consecutive same-kind collapse', async () => { + // The Messages API cannot carry audio or video. Dropping the parts + // silently would leave the model unaware an attachment ever existed, + // so each unsupported part degrades to a placeholder text block. + const provider = createProvider(); + const history: Message[] = [ + { + role: 'user', + content: [ + { type: 'text', text: 'Listen and watch:' }, + { type: 'audio_url', audioUrl: { url: 'https://example.com/a.mp3' } }, + { type: 'audio_url', audioUrl: { url: 'https://example.com/b.mp3' } }, + { type: 'video_url', videoUrl: { url: 'https://example.com/c.mp4' } }, + ] satisfies ContentPart[], + toolCalls: [], + }, + ]; + const body = await captureRequestBody(provider, '', [], history); + const messages = body['messages'] as Record[]; + + expect(messages[0]?.['content']).toEqual([ + { type: 'text', text: 'Listen and watch:' }, + { type: 'text', text: '(audio omitted: not supported by this provider)' }, + { + type: 'text', + text: '(video omitted: not supported by this provider)', + cache_control: { type: 'ephemeral' }, + }, + ]); + }); + + it('tool result audio degrades to placeholder text inside tool_result', async () => { + const provider = createProvider(); + const history: Message[] = [ + { role: 'user', content: [{ type: 'text', text: 'Say hi' }], toolCalls: [] }, + { + role: 'assistant', + content: [], + toolCalls: [{ type: 'function', id: 'call_tts', name: 'tts', arguments: '{}' }], + }, + { + role: 'tool', + content: [ + { type: 'audio_url', audioUrl: { url: 'https://example.com/hi.mp3' } }, + ] satisfies ContentPart[], + toolCallId: 'call_tts', + toolCalls: [], + }, + ]; + const body = await captureRequestBody(provider, '', [], history); + const messages = body['messages'] as Record[]; + + expect(messages[2]).toEqual({ + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'call_tts', + content: [ + { type: 'text', text: '(audio omitted: not supported by this provider)' }, + ], + cache_control: { type: 'ephemeral' }, + }, + ], + }); + }); + it('parallel tool calls and tool results (request body capture)', async () => { const provider = createProvider(); const tcAdd: ToolCall = { diff --git a/packages/kosong/test/openai-legacy.test.ts b/packages/kosong/test/openai-legacy.test.ts index 07aec0f01..8335a8228 100644 --- a/packages/kosong/test/openai-legacy.test.ts +++ b/packages/kosong/test/openai-legacy.test.ts @@ -332,12 +332,77 @@ describe('OpenAILegacyChatProvider', () => { expect(messages[3]).toEqual({ role: 'user', content: [ - { type: 'text', text: 'Attached image(s) from tool result:' }, + { type: 'text', text: 'Attached media from tool result:' }, { type: 'image_url', image_url: { url: 'https://example.com/image.png' } }, ], }); }); + it('tool call with audio result notes the omission inline without reattaching', async () => { + // Chat Completions has no url-based audio/video content part (only + // base64 input_audio), so unlike images these cannot be reattached as + // a user message — a standard OpenAI endpoint would reject the request + // with a 400. The tool message notes the omission inline instead. + const provider = createProvider(); + const history: Message[] = [ + { role: 'user', content: [{ type: 'text', text: 'Say hi' }], toolCalls: [] }, + { + role: 'assistant', + content: [], + toolCalls: [{ type: 'function', id: 'call_tts', name: 'tts', arguments: '{}' }], + }, + { + role: 'tool', + content: [ + { type: 'audio_url', audioUrl: { url: 'https://example.com/hi.mp3' } }, + ] satisfies ContentPart[], + toolCallId: 'call_tts', + toolCalls: [], + }, + ]; + const body = await captureRequestBody(provider, '', [], history); + + const messages = body['messages'] as Record[]; + expect(messages[2]).toEqual({ + role: 'tool', + content: '(audio omitted: not supported by this provider)', + tool_call_id: 'call_tts', + }); + // No follow-up user message: audio_url is not a standard Chat + // Completions content part and must not reach the wire. + expect(messages).toHaveLength(3); + }); + + it('tool call with text and video result appends the omission note to the text', async () => { + const provider = createProvider(); + const history: Message[] = [ + { role: 'user', content: [{ type: 'text', text: 'Record it' }], toolCalls: [] }, + { + role: 'assistant', + content: [], + toolCalls: [{ type: 'function', id: 'call_rec', name: 'record', arguments: '{}' }], + }, + { + role: 'tool', + content: [ + { type: 'text', text: 'recorded 5s clip' }, + { type: 'video_url', videoUrl: { url: 'https://example.com/rec.mp4' } }, + ] satisfies ContentPart[], + toolCallId: 'call_rec', + toolCalls: [], + }, + ]; + const body = await captureRequestBody(provider, '', [], history); + + const messages = body['messages'] as Record[]; + expect(messages[2]).toEqual({ + role: 'tool', + content: 'recorded 5s clip\n(video omitted: not supported by this provider)', + tool_call_id: 'call_rec', + }); + expect(messages).toHaveLength(3); + }); + it('groups consecutive tool result images after all matching tool messages', async () => { const provider = createProvider(); const history: Message[] = [ @@ -388,12 +453,12 @@ describe('OpenAILegacyChatProvider', () => { }, ], }, - { role: 'tool', content: '(see attached image)', tool_call_id: 'call_first' }, + { role: 'tool', content: '(see attached media)', tool_call_id: 'call_first' }, { role: 'tool', content: 'second', tool_call_id: 'call_second' }, { role: 'user', content: [ - { type: 'text', text: 'Attached image(s) from tool result:' }, + { type: 'text', text: 'Attached media from tool result:' }, { type: 'image_url', image_url: { url: 'https://example.com/first.png' } }, { type: 'image_url', image_url: { url: 'https://example.com/second.png' } }, ], diff --git a/packages/kosong/test/openai-responses.test.ts b/packages/kosong/test/openai-responses.test.ts index 3ce2fd414..3e95d9152 100644 --- a/packages/kosong/test/openai-responses.test.ts +++ b/packages/kosong/test/openai-responses.test.ts @@ -292,7 +292,7 @@ describe('OpenAIResponsesChatProvider', () => { ]); }); - it('user message with unsupported audio_url format drops the audio part silently', async () => { + it('user message with unsupported audio_url format degrades to placeholder text', async () => { const provider = createProvider(); const history: Message[] = [ { @@ -307,8 +307,12 @@ describe('OpenAIResponsesChatProvider', () => { const body = await captureRequestBody(provider, '', [], history); const input = body['input'] as Array<{ content: unknown[] }>; - // Only the text part survives; the unsupported ogg audio is dropped. - expect(input[0]!.content).toEqual([{ type: 'input_text', text: 'Bare text' }]); + // The unsupported ogg audio degrades to a placeholder instead of + // silently vanishing, so the model knows an attachment existed. + expect(input[0]!.content).toEqual([ + { type: 'input_text', text: 'Bare text' }, + { type: 'input_text', text: '(audio omitted: unsupported audio format)' }, + ]); }); it('multiple consecutive ThinkParts with the same encrypted value aggregate into one reasoning item with multiple summaries', async () => { @@ -407,6 +411,102 @@ describe('OpenAIResponsesChatProvider', () => { }); }); + it('toolMessageConversion=extract_text reattaches tool result media as a user message', async () => { + // extract_text flattens function_call_output to a plain string for + // backends that reject structured output. Media must not vanish with + // the flattening: the image-only result gets a placeholder string and + // the media items are reattached as a follow-up user message after + // the run of consecutive tool messages. + const provider = new OpenAIResponsesChatProvider({ + model: 'gpt-4.1', + apiKey: 'test-key', + toolMessageConversion: 'extract_text', + }); + const history: Message[] = [ + { role: 'user', content: [{ type: 'text', text: 'Q' }], toolCalls: [] }, + { + role: 'assistant', + content: [], + toolCalls: [ + { type: 'function', id: 'call_shot', name: 'screenshot', arguments: '{}' }, + { type: 'function', id: 'call_read', name: 'read', arguments: '{}' }, + ], + }, + { + role: 'tool', + content: [ + { type: 'image_url', imageUrl: { url: 'https://example.com/shot.png' } }, + ], + toolCallId: 'call_shot', + toolCalls: [], + }, + { + role: 'tool', + content: [{ type: 'text', text: 'file body' }], + toolCallId: 'call_read', + toolCalls: [], + }, + ]; + const body = await captureRequestBody(provider, '', [], history); + + const input = body['input'] as Array>; + expect(input).toEqual([ + { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'Q' }] }, + { + type: 'function_call', + call_id: 'call_shot', + name: 'screenshot', + arguments: '{}', + }, + { type: 'function_call', call_id: 'call_read', name: 'read', arguments: '{}' }, + { + type: 'function_call_output', + call_id: 'call_shot', + output: '(see attached media)', + }, + { type: 'function_call_output', call_id: 'call_read', output: 'file body' }, + { + type: 'message', + role: 'user', + content: [ + { type: 'input_text', text: 'Attached media from tool result:' }, + { type: 'input_image', image_url: 'https://example.com/shot.png' }, + ], + }, + ]); + }); + + it('video_url in tool result degrades to placeholder text in function_call_output', async () => { + const provider = createProvider(); + const history: Message[] = [ + { role: 'user', content: [{ type: 'text', text: 'Record it' }], toolCalls: [] }, + { + role: 'assistant', + content: [], + toolCalls: [{ type: 'function', id: 'call_rec', name: 'record', arguments: '{}' }], + }, + { + role: 'tool', + content: [ + { type: 'video_url', videoUrl: { url: 'https://example.com/rec.mp4' } }, + ], + toolCallId: 'call_rec', + toolCalls: [], + }, + ]; + const body = await captureRequestBody(provider, '', [], history); + + const input = body['input'] as Array>; + const fnOutput = input.find((item) => item['type'] === 'function_call_output'); + expect(fnOutput).toEqual({ + type: 'function_call_output', + call_id: 'call_rec', + output: [ + { type: 'input_text', text: '(video omitted: not supported by this provider)' }, + ], + }); + }); + it('parallel tool calls produce multiple function_call and function_call_output items', async () => { const provider = createProvider(); const history: Message[] = [ @@ -1120,7 +1220,7 @@ describe('OpenAIResponsesChatProvider', () => { expect(body['max_output_tokens']).toBe(512); }); - it('video_url in user content is silently skipped (Responses API has no video representation)', async () => { + it('video_url in user content degrades to placeholder text (no video input type)', async () => { const provider = createProvider(); const history: Message[] = [ { @@ -1134,12 +1234,14 @@ describe('OpenAIResponsesChatProvider', () => { ]; const body = await captureRequestBody(provider, '', [], history); - // Only the text part survives; video is dropped. const input = body['input'] as Array<{ content: unknown[] }>; - expect(input[0]!.content).toEqual([{ type: 'input_text', text: 'Watch this:' }]); + expect(input[0]!.content).toEqual([ + { type: 'input_text', text: 'Watch this:' }, + { type: 'input_text', text: '(video omitted: not supported by this provider)' }, + ]); }); - it('audio_url with unsupported scheme is silently dropped from user content', async () => { + it('audio_url with unsupported scheme degrades to placeholder text', async () => { const provider = createProvider(); const history: Message[] = [ { @@ -1154,11 +1256,14 @@ describe('OpenAIResponsesChatProvider', () => { const body = await captureRequestBody(provider, '', [], history); const input = body['input'] as Array<{ content: unknown[] }>; - // file:// URL is unsupported → drop - expect(input[0]!.content).toEqual([{ type: 'input_text', text: 'Hear:' }]); + // file:// URL cannot be encoded as input_file → placeholder + expect(input[0]!.content).toEqual([ + { type: 'input_text', text: 'Hear:' }, + { type: 'input_text', text: '(audio omitted: unsupported audio format)' }, + ]); }); - it('audio_url data URL with unknown subtype is silently dropped', async () => { + it('audio_url data URL with unknown subtype degrades to placeholder text', async () => { const provider = createProvider(); const history: Message[] = [ { @@ -1173,8 +1278,11 @@ describe('OpenAIResponsesChatProvider', () => { const body = await captureRequestBody(provider, '', [], history); const input = body['input'] as Array<{ content: unknown[] }>; - // ogg subtype is not mp3/wav → drop - expect(input[0]!.content).toEqual([{ type: 'input_text', text: 'OGG:' }]); + // ogg subtype is not mp3/wav → placeholder + expect(input[0]!.content).toEqual([ + { type: 'input_text', text: 'OGG:' }, + { type: 'input_text', text: '(audio omitted: unsupported audio format)' }, + ]); }); });