diff --git a/.changeset/fix-tool-result-images.md b/.changeset/fix-tool-result-images.md new file mode 100644 index 000000000..e38dcd1eb --- /dev/null +++ b/.changeset/fix-tool-result-images.md @@ -0,0 +1,6 @@ +--- +"@moonshot-ai/kosong": patch +"@moonshot-ai/kimi-code": patch +--- + +Preserve image outputs from tools when using OpenAI-compatible chat completions. diff --git a/packages/kosong/src/providers/openai-legacy.ts b/packages/kosong/src/providers/openai-legacy.ts index 23820c8db..5fb313c22 100644 --- a/packages/kosong/src/providers/openai-legacy.ts +++ b/packages/kosong/src/providers/openai-legacy.ts @@ -50,6 +50,8 @@ const OPENAI_CHAT_TOOL_CALL_ID_POLICY: ToolCallIdPolicy = { normalize: (id) => sanitizeToolCallId(id, 64), maxLength: 64, }; +const TOOL_RESULT_IMAGE_PROMPT = 'Attached image(s) from tool result:'; +const TOOL_RESULT_IMAGE_PLACEHOLDER = '(see attached image)'; function extractReasoningContent( source: unknown, @@ -165,7 +167,7 @@ function convertMessage( : toolMessageConversion; if (effectiveConversion !== null) { - result.content = convertToolMessageContent(message, effectiveConversion); + result.content = convertToolMessageContentForChat(message, effectiveConversion); } else { // Pure-text tool result with no conversion configured: serialize via the // generic content-part path so single-text messages become a plain string. @@ -217,6 +219,70 @@ function convertMessage( return result; } + +function convertToolMessageContentForChat( + message: Message, + conversion: ToolMessageConversion, +): string | OpenAIContentPart[] { + const content = convertToolMessageContent(message, conversion); + if ( + typeof content === 'string' && + content.length === 0 && + message.content.some((part) => part.type === 'image_url') + ) { + return TOOL_RESULT_IMAGE_PLACEHOLDER; + } + return content; +} + +function toolResultImageParts(message: Message): OpenAIContentPart[] { + const images: OpenAIContentPart[] = []; + for (const part of message.content) { + if (part.type !== 'image_url') continue; + images.push({ + type: 'image_url', + image_url: + part.imageUrl.id === undefined + ? { url: part.imageUrl.url } + : { url: part.imageUrl.url, id: part.imageUrl.id }, + }); + } + return images; +} + +function appendToolResultImagesMessage( + messages: OpenAIMessage[], + pendingToolResultImages: OpenAIContentPart[], +): void { + if (pendingToolResultImages.length === 0) return; + messages.push({ + role: 'user', + content: [{ type: 'text', text: TOOL_RESULT_IMAGE_PROMPT }, ...pendingToolResultImages], + }); + pendingToolResultImages.length = 0; +} + +function convertHistoryMessages( + history: readonly Message[], + reasoningKey: string | undefined, + toolMessageConversion: ToolMessageConversion, +): OpenAIMessage[] { + const messages: OpenAIMessage[] = []; + const pendingToolResultImages: OpenAIContentPart[] = []; + + for (const msg of history) { + if (msg.role !== 'tool') { + appendToolResultImagesMessage(messages, pendingToolResultImages); + } + messages.push(convertMessage(msg, reasoningKey, toolMessageConversion)); + if (msg.role === 'tool') { + pendingToolResultImages.push(...toolResultImageParts(msg)); + } + } + + appendToolResultImagesMessage(messages, pendingToolResultImages); + return messages; +} export class OpenAILegacyStreamedMessage implements StreamedMessage { private _id: string | null = null; private _usage: TokenUsage | null = null; @@ -437,9 +503,9 @@ export class OpenAILegacyChatProvider implements ChatProvider { history, OPENAI_CHAT_TOOL_CALL_ID_POLICY, ); - for (const msg of normalizedHistory) { - messages.push(convertMessage(msg, this._reasoningKey, this._toolMessageConversion)); - } + messages.push( + ...convertHistoryMessages(normalizedHistory, this._reasoningKey, this._toolMessageConversion), + ); const kwargs: Record = normalizeGenerationKwargs( this._model, diff --git a/packages/kosong/test/openai-legacy.test.ts b/packages/kosong/test/openai-legacy.test.ts index 8847f6626..07aec0f01 100644 --- a/packages/kosong/test/openai-legacy.test.ts +++ b/packages/kosong/test/openai-legacy.test.ts @@ -287,7 +287,7 @@ describe('OpenAILegacyChatProvider', () => { ]); }); - it('tool call with image result flattens to text to satisfy API constraints', async () => { + it('tool call with image result keeps the tool result textual and reattaches images as user input', async () => { // OpenAI Chat Completions `tool` messages only accept text content. // Even when toolMessageConversion is unset, a tool result containing // image_url / audio_url / video_url parts must not be serialized as a @@ -319,15 +319,86 @@ describe('OpenAILegacyChatProvider', () => { ]; const body = await captureRequestBody(provider, '', [], history); - const toolMsg = (body['messages'] as Record[])[2]!; + const messages = body['messages'] as Record[]; + const toolMsg = messages[2]!; expect(toolMsg['role']).toBe('tool'); expect(toolMsg['tool_call_id']).toBe('call_abc123'); // Content must be a plain string, not a content-part array. expect(typeof toolMsg['content']).toBe('string'); // The text segment must survive; the image must not appear as a - // structured image_url part anywhere in the serialized content. + // structured image_url part inside the tool message. expect(toolMsg['content']).toContain('5'); expect(Array.isArray(toolMsg['content'])).toBe(false); + expect(messages[3]).toEqual({ + role: 'user', + content: [ + { type: 'text', text: 'Attached image(s) from tool result:' }, + { type: 'image_url', image_url: { url: 'https://example.com/image.png' } }, + ], + }); + }); + + it('groups consecutive tool result images after all matching tool messages', async () => { + const provider = createProvider(); + const history: Message[] = [ + { role: 'user', content: [{ type: 'text', text: 'Fetch both images' }], toolCalls: [] }, + { + role: 'assistant', + content: [{ type: 'text', text: 'ok' }], + toolCalls: [ + { type: 'function', id: 'call_first', name: 'first_image', arguments: '{}' }, + { type: 'function', id: 'call_second', name: 'second_image', arguments: '{}' }, + ], + }, + { + role: 'tool', + content: [ + { type: 'image_url', imageUrl: { url: 'https://example.com/first.png' } }, + ], + toolCallId: 'call_first', + toolCalls: [], + }, + { + role: 'tool', + content: [ + { type: 'text', text: 'second' }, + { type: 'image_url', imageUrl: { url: 'https://example.com/second.png' } }, + ], + toolCallId: 'call_second', + toolCalls: [], + }, + ]; + const body = await captureRequestBody(provider, '', [], history); + + expect(body['messages']).toEqual([ + { role: 'user', content: 'Fetch both images' }, + { + role: 'assistant', + content: 'ok', + tool_calls: [ + { + type: 'function', + id: 'call_first', + function: { name: 'first_image', arguments: '{}' }, + }, + { + type: 'function', + id: 'call_second', + function: { name: 'second_image', arguments: '{}' }, + }, + ], + }, + { role: 'tool', content: '(see attached image)', tool_call_id: 'call_first' }, + { role: 'tool', content: 'second', tool_call_id: 'call_second' }, + { + role: 'user', + content: [ + { type: 'text', text: 'Attached image(s) from tool result:' }, + { type: 'image_url', image_url: { url: 'https://example.com/first.png' } }, + { type: 'image_url', image_url: { url: 'https://example.com/second.png' } }, + ], + }, + ]); }); it('parallel tool calls', async () => {