diff --git a/.changeset/hide-image-compression-caption.md b/.changeset/hide-image-compression-caption.md new file mode 100644 index 000000000..dd30c6391 --- /dev/null +++ b/.changeset/hide-image-compression-caption.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Fix compressed-image prompts leaking an internal `` compression note into the visible message and the session title. diff --git a/packages/agent-core/src/agent/context/index.ts b/packages/agent-core/src/agent/context/index.ts index 5c93ebf45..5f6abc2f2 100644 --- a/packages/agent-core/src/agent/context/index.ts +++ b/packages/agent-core/src/agent/context/index.ts @@ -3,6 +3,7 @@ import { createToolMessage, type ContentPart, type Message } from '@moonshot-ai/ import type { Agent } from '..'; import { ErrorCodes, KimiError } from '../../errors'; import type { ExecutableToolResult, LoopRecordedEvent } from '../../loop'; +import { extractImageCompressionCaptions } from '../../tools/support/image-compress'; import { estimateTokens, estimateTokensForMessages } from '../../utils/tokens'; import { escapeXml } from '../../utils/xml-escape'; import { @@ -66,9 +67,23 @@ export class ContextMemory { origin: PromptOrigin = USER_PROMPT_ORIGIN, ): void { if (content.length === 0) return; + // Prompt ingestion (server upload/base64 route, TUI paste, ACP) annotates + // a compressed image with an inline `` caption next to the image. + // Left inside the user message, that raw markup is user-visible in every + // history projection (TUI replay, vis, export). Reroute each caption + // through the built-in system-reminder injection — hidden by its + // `injection` origin — and keep only the real user content here. + const { captions, parts } = + origin.kind === 'user' + ? splitImageCompressionCaptions(content) + : { captions: [], parts: [...content] }; + for (const caption of captions) { + this.appendSystemReminder(caption, { kind: 'injection', variant: 'image_compression' }); + } + if (parts.length === 0) return; this.appendMessage({ role: 'user', - content: [...content], + content: parts, toolCalls: [], origin, }); @@ -702,6 +717,35 @@ function isEmptyEquivalentContentArray(output: readonly ContentPart[]): boolean return output.every((part) => part.type === 'text' && part.text.trim().length === 0); } +// Split inline image-compression captions (see buildImageCompressionCaption) +// out of user prompt content. A caption may be a standalone text part (server +// route, ACP) or merged into an adjacent text segment (TUI paste), so each +// text part is scanned rather than matched whole. Text left empty once its +// captions are removed is dropped entirely. +function splitImageCompressionCaptions(content: readonly ContentPart[]): { + captions: readonly string[]; + parts: ContentPart[]; +} { + const captions: string[] = []; + const parts: ContentPart[] = []; + for (const part of content) { + if (part.type !== 'text') { + parts.push(part); + continue; + } + const extracted = extractImageCompressionCaptions(part.text); + if (extracted.captions.length === 0) { + parts.push(part); + continue; + } + captions.push(...extracted.captions); + if (extracted.text.trim().length > 0) { + parts.push({ type: 'text', text: extracted.text }); + } + } + return { captions, parts }; +} + function isEmptyOutputText(output: string): boolean { return output.trim().length === 0 || output.trim() === TOOL_OUTPUT_EMPTY_TEXT; } diff --git a/packages/agent-core/src/session/prompt-metadata.ts b/packages/agent-core/src/session/prompt-metadata.ts index b587ba00b..c43c056bb 100644 --- a/packages/agent-core/src/session/prompt-metadata.ts +++ b/packages/agent-core/src/session/prompt-metadata.ts @@ -1,4 +1,5 @@ import type { ActivatePluginCommandPayload, ActivateSkillPayload, PromptPayload } from '#/rpc'; +import { extractImageCompressionCaptions } from '#/tools/support/image-compress'; import type { ContentPart } from '@moonshot-ai/kosong'; const MAX_TITLE_LENGTH = 200; @@ -38,8 +39,13 @@ export function promptMetadataTextFromPluginCommand( function promptPartText(part: ContentPart): string | undefined { switch (part.type) { - case 'text': - return part.text; + case 'text': { + // Prompt ingestion may have annotated a compressed image with an inline + // caption (see buildImageCompressionCaption). It is harness metadata, + // not something the user typed, so keep it out of titles/lastPrompt. + const { text } = extractImageCompressionCaptions(part.text); + return text.trim().length === 0 ? undefined : text; + } case 'image_url': return '[image]'; case 'audio_url': diff --git a/packages/agent-core/src/tools/support/image-compress.ts b/packages/agent-core/src/tools/support/image-compress.ts index 2edf671f6..0c640b7be 100644 --- a/packages/agent-core/src/tools/support/image-compress.ts +++ b/packages/agent-core/src/tools/support/image-compress.ts @@ -20,7 +20,10 @@ * original dimensions, {@link buildImageCompressionCaption} renders the * shared "what was compressed, where is the original" note every ingestion * point can place next to the image, and {@link cropImageForModel} lets a - * caller read a region of the original back at full fidelity. + * caller read a region of the original back at full fidelity. In user + * prompts the context layer later reroutes that note through the hidden + * system-reminder injection via {@link extractImageCompressionCaptions}, + * so its raw `` markup never renders in the UI. */ import type { ContentPart } from '@moonshot-ai/kosong'; @@ -553,6 +556,14 @@ export interface ImageCompressionCaptionInput { * model knows it is looking at a downsampled copy: what the original was, what * was actually sent, and — when the original is on disk — where to read it * back (via ReadMediaFile `region`) for full-fidelity detail. + * + * Two channels consume this note differently: + * - Tool results (MCP images) keep it inline — `` status text inside + * tool output is the established convention there. + * - User prompts must not render raw `` markup in the UI, so the + * context layer detects the caption via + * {@link extractImageCompressionCaptions} and reroutes it through the + * built-in system-reminder injection (hidden by its `injection` origin). */ export function buildImageCompressionCaption(input: ImageCompressionCaptionInput): string { const sentences = [ @@ -572,6 +583,46 @@ export function buildImageCompressionCaption(input: ImageCompressionCaptionInput return `${sentences.join(' ')}`; } +/** + * Fixed opening every {@link buildImageCompressionCaption} note starts with — + * the anchor {@link extractImageCompressionCaptions} matches on. Keep the two + * in sync. + */ +const CAPTION_OPENING = 'Image compressed to fit model limits:'; + +/** + * A full caption embedded in arbitrary text. The body is sentences plus a + * quoted file path and never contains ``, so the non-greedy scan to + * the closing tag is exact. + */ +const CAPTION_PATTERN = /(Image compressed to fit model limits:[\s\S]*?)<\/system>/g; + +export interface ImageCompressionCaptionExtraction { + /** Caption bodies found, in order, without the `` wrapper. */ + readonly captions: readonly string[]; + /** The input text with every caption removed. */ + readonly text: string; +} + +/** + * Find every {@link buildImageCompressionCaption} note embedded in `text` and + * return the unwrapped caption bodies plus the text without them. Prompt + * ingestion (server upload/base64 route, TUI paste, ACP) places the caption + * inline next to the image — sometimes merged into an adjacent text segment — + * and the context layer uses this to reroute the note through the built-in + * system-reminder injection instead of leaving raw `` markup in the + * user-visible message. + */ +export function extractImageCompressionCaptions(text: string): ImageCompressionCaptionExtraction { + if (!text.includes(CAPTION_OPENING)) return { captions: [], text }; + const captions: string[] = []; + const remainder = text.replace(CAPTION_PATTERN, (_match, body: string) => { + captions.push(body); + return ''; + }); + return { captions, text: remainder }; +} + function describeImageVariant(variant: ImageVariantDescription): string { const size = `${variant.mimeType} (${formatByteSize(variant.byteLength)})`; if (variant.width > 0 && variant.height > 0) { diff --git a/packages/agent-core/test/agent/context.test.ts b/packages/agent-core/test/agent/context.test.ts index 4b613dde0..983cdfa0b 100644 --- a/packages/agent-core/test/agent/context.test.ts +++ b/packages/agent-core/test/agent/context.test.ts @@ -7,6 +7,7 @@ import { describe, expect, it, vi } from 'vitest'; import { renderNotificationXml } from '../../src/agent/context/notification-xml'; import { project } from '../../src/agent/context/projector'; import type { ContextMessage } from '../../src/agent/context/types'; +import { buildImageCompressionCaption } from '../../src/tools/support/image-compress'; import { estimateTokensForMessages } from '../../src/utils/tokens'; import { createFakeKaos } from '../tools/fixtures/fake-kaos'; import { recordingTelemetry, type TelemetryRecord } from '../fixtures/telemetry'; @@ -59,6 +60,79 @@ describe('Agent context', () => { expect(ctx.agent.context.messages.some((message) => 'origin' in message)).toBe(false); }); + it('reroutes an inline image-compression caption into a hidden system reminder', () => { + const ctx = testAgent(); + ctx.configure(); + + const caption = buildImageCompressionCaption({ + original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' }, + final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' }, + originalPath: '/tmp/originals/shot.png', + }); + // The TUI merges the caption into the preceding text segment; the server + // route emits it as a standalone part. Cover the merged (harder) shape. + ctx.agent.context.appendUserMessage([ + { type: 'text', text: `能展示但是没有快捷键提示${caption}` }, + { type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } }, + ]); + + const textOf = (message: ContextMessage): string => + message.content.map((part) => (part.type === 'text' ? part.text : '')).join(''); + + expect(ctx.agent.context.history.map(({ role, origin }) => ({ role, origin }))).toEqual([ + { role: 'user', origin: { kind: 'injection', variant: 'image_compression' } }, + { role: 'user', origin: { kind: 'user' } }, + ]); + const [reminder, userMessage] = ctx.agent.context.history; + expect(textOf(reminder!)).toContain(''); + expect(textOf(reminder!)).toContain('Image compressed to fit model limits'); + expect(textOf(reminder!)).toContain('/tmp/originals/shot.png'); + expect(textOf(reminder!)).not.toContain(''); + expect(textOf(userMessage!)).toBe('能展示但是没有快捷键提示'); + expect(userMessage!.content.some((part) => part.type === 'image_url')).toBe(true); + }); + + it('drops a caption-only text part instead of leaving an empty user text part', () => { + const ctx = testAgent(); + ctx.configure(); + + const caption = buildImageCompressionCaption({ + original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' }, + final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' }, + originalPath: '/tmp/originals/shot.png', + }); + ctx.agent.context.appendUserMessage([ + { type: 'text', text: caption }, + { type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } }, + ]); + + const [, userMessage] = ctx.agent.context.history; + expect(userMessage!.content).toEqual([ + { type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } }, + ]); + }); + + it('leaves caption-shaped text alone on non-user origins', () => { + const ctx = testAgent(); + ctx.configure(); + + const caption = buildImageCompressionCaption({ + original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' }, + final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' }, + originalPath: '/tmp/originals/shot.png', + }); + ctx.agent.context.appendUserMessage([{ type: 'text', text: caption }], { + kind: 'hook_result', + event: 'PostToolUse', + }); + + expect(ctx.agent.context.history).toHaveLength(1); + expect(ctx.agent.context.history[0]!.origin).toEqual({ + kind: 'hook_result', + event: 'PostToolUse', + }); + }); + it('tracks conversation_undo when undoHistory reverts a user message', async () => { const records: TelemetryRecord[] = []; const ctx = testAgent({ telemetry: recordingTelemetry(records) }); diff --git a/packages/agent-core/test/agent/image-caption-injection.test.ts b/packages/agent-core/test/agent/image-caption-injection.test.ts new file mode 100644 index 000000000..805a54861 --- /dev/null +++ b/packages/agent-core/test/agent/image-caption-injection.test.ts @@ -0,0 +1,172 @@ +/** + * End-to-end smoke for image-compression caption rerouting. + * + * Prompt ingestion (server route, TUI paste, ACP) annotates a compressed + * image with an inline `` caption inside the user's own message. + * The context layer must split that caption out into a hidden + * system-reminder injection so no raw `` markup is ever rendered in + * a user bubble, while the model still receives the note. + * + * These tests drive the REAL pipeline — rpc.prompt → turn → context → + * scripted provider — and assert at every seam: + * - the wire request the model receives (reminder present, user text clean) + * - the stored context history (origins drive UI hiding) + * - the recorded wire log (what session resume replays) + * - resume parity via expectResumeMatches (the TUI replay data source) + */ + +import { expect, it } from 'vitest'; + +import { buildImageCompressionCaption } from '../../src/tools/support/image-compress'; +import { testAgent } from './harness/agent'; + +const CAPTION = buildImageCompressionCaption({ + original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' }, + final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' }, + originalPath: '/tmp/originals/shot.png', +}); + +const SECOND_CAPTION = buildImageCompressionCaption({ + original: { width: 4000, height: 3000, byteLength: 9 * 1024 * 1024, mimeType: 'image/jpeg' }, + final: { width: 2000, height: 1500, byteLength: 1024 * 1024, mimeType: 'image/jpeg' }, + originalPath: '/tmp/originals/photo.jpg', +}); + +const IMAGE_URL = 'data:image/png;base64,AAAA'; + +it('smoke: a compressed-image prompt reaches the model with the caption as a system reminder', async () => { + const ctx = testAgent(); + ctx.configure(); + + ctx.mockNextResponse({ type: 'text', text: 'I can see the screenshot.' }); + // The TUI merges the caption into the preceding text segment — the exact + // shape from the bug report. + await ctx.rpc.prompt({ + input: [ + { type: 'text', text: `能展示但是没有快捷键提示${CAPTION}` }, + { type: 'image_url', imageUrl: { url: IMAGE_URL } }, + ], + }); + await ctx.untilTurnEnd(); + + // What the model actually received on the wire. + const llmInput = JSON.stringify(ctx.lastLlmInput().input); + expect(llmInput).toContain(''); + expect(llmInput).toContain('Image compressed to fit model limits'); + expect(llmInput).toContain('/tmp/originals/shot.png'); + expect(llmInput).not.toContain(''); + expect(llmInput).toContain('能展示但是没有快捷键提示'); + + // What the UI renders from: the reminder is a separate injection-origin + // message; the user message holds only what the user actually submitted. + const history = ctx.agent.context.history; + expect(history.map(({ role, origin }) => ({ role, origin }))).toEqual([ + { role: 'user', origin: { kind: 'injection', variant: 'image_compression' } }, + { role: 'user', origin: { kind: 'user' } }, + { role: 'assistant', origin: undefined }, + ]); + const userText = history[1]!.content + .map((part) => (part.type === 'text' ? part.text : '')) + .join(''); + expect(userText).toBe('能展示但是没有快捷键提示'); + expect(history[1]!.content.some((part) => part.type === 'image_url')).toBe(true); + + // The recorded wire log is what session resume replays into the TUI: the + // user append_message record must already be caption-free. + const appendRecords = ctx.allEvents.filter( + (event) => event.type === '[wire]' && event.event === 'context.append_message', + ); + expect(appendRecords).toHaveLength(2); + expect(JSON.stringify(appendRecords[0]!.args)).toContain('system-reminder'); + expect(JSON.stringify(appendRecords[1]!.args)).not.toContain(''); + + // Resume the session from the records and require identical state. + await ctx.expectResumeMatches(); +}); + +it('smoke: multiple compressed images in one prompt each announce via their own reminder', async () => { + const ctx = testAgent(); + ctx.configure(); + + ctx.mockNextResponse({ type: 'text', text: 'Two images noted.' }); + // Server-route shape: standalone caption part directly before each image. + await ctx.rpc.prompt({ + input: [ + { type: 'text', text: CAPTION }, + { type: 'image_url', imageUrl: { url: IMAGE_URL } }, + { type: 'text', text: SECOND_CAPTION }, + { type: 'image_url', imageUrl: { url: IMAGE_URL } }, + { type: 'text', text: '对比一下这两张截图' }, + ], + }); + await ctx.untilTurnEnd(); + + const llmInput = JSON.stringify(ctx.lastLlmInput().input); + expect(llmInput).toContain('/tmp/originals/shot.png'); + expect(llmInput).toContain('/tmp/originals/photo.jpg'); + expect(llmInput).not.toContain(''); + + const history = ctx.agent.context.history; + expect(history.map(({ role, origin }) => ({ role, origin }))).toEqual([ + { role: 'user', origin: { kind: 'injection', variant: 'image_compression' } }, + { role: 'user', origin: { kind: 'injection', variant: 'image_compression' } }, + { role: 'user', origin: { kind: 'user' } }, + { role: 'assistant', origin: undefined }, + ]); + const userMessage = history[2]!; + expect(userMessage.content.filter((part) => part.type === 'image_url')).toHaveLength(2); + const userText = userMessage.content + .map((part) => (part.type === 'text' ? part.text : '')) + .join(''); + expect(userText).toBe('对比一下这两张截图'); + + await ctx.expectResumeMatches(); +}); + +it('smoke: a steered prompt with a caption is split the same way', async () => { + const ctx = testAgent(); + ctx.configure(); + + ctx.mockNextResponse({ type: 'text', text: 'Steered.' }); + await ctx.rpc.steer({ + input: [ + { type: 'text', text: `看这张${CAPTION}` }, + { type: 'image_url', imageUrl: { url: IMAGE_URL } }, + ], + }); + await ctx.untilTurnEnd(); + + const history = ctx.agent.context.history; + expect(history.map(({ role, origin }) => ({ role, origin }))).toEqual([ + { role: 'user', origin: { kind: 'injection', variant: 'image_compression' } }, + { role: 'user', origin: { kind: 'user' } }, + { role: 'assistant', origin: undefined }, + ]); + const userText = history[1]!.content + .map((part) => (part.type === 'text' ? part.text : '')) + .join(''); + expect(userText).toBe('看这张'); + + await ctx.expectResumeMatches(); +}); + +it('smoke: a prompt without images is completely untouched', async () => { + const ctx = testAgent(); + ctx.configure(); + + ctx.mockNextResponse({ type: 'text', text: 'Hi.' }); + await ctx.rpc.prompt({ input: [{ type: 'text', text: 'plain hello' }] }); + await ctx.untilTurnEnd(); + + const history = ctx.agent.context.history; + expect(history.map(({ role, origin }) => ({ role, origin }))).toEqual([ + { role: 'user', origin: { kind: 'user' } }, + { role: 'assistant', origin: undefined }, + ]); + const userText = history[0]!.content + .map((part) => (part.type === 'text' ? part.text : '')) + .join(''); + expect(userText).toBe('plain hello'); + + await ctx.expectResumeMatches(); +}); diff --git a/packages/agent-core/test/session/prompt-metadata.test.ts b/packages/agent-core/test/session/prompt-metadata.test.ts new file mode 100644 index 000000000..248303d91 --- /dev/null +++ b/packages/agent-core/test/session/prompt-metadata.test.ts @@ -0,0 +1,55 @@ +/** + * prompt-metadata — the session title / lastPrompt text derived from a + * prompt payload. + * + * Tests pin: + * - media parts render as `[image]` / `[video]` / `[audio]` placeholders + * - an inline image-compression caption (harness metadata placed next to + * the image by prompt ingestion) never leaks into titles/lastPrompt, + * whether it is a standalone text part or merged into the user's text + */ + +import { describe, expect, it } from 'vitest'; + +import { promptMetadataTextFromPayload } from '../../src/session/prompt-metadata'; +import { buildImageCompressionCaption } from '../../src/tools/support/image-compress'; + +const CAPTION = buildImageCompressionCaption({ + original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' }, + final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' }, + originalPath: '/tmp/originals/shot.png', +}); + +describe('promptMetadataTextFromPayload', () => { + it('renders text and media placeholders', () => { + const text = promptMetadataTextFromPayload({ + input: [ + { type: 'text', text: 'look at this' }, + { type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } }, + ], + }); + expect(text).toBe('look at this [image]'); + }); + + it('keeps a standalone image-compression caption out of the metadata text', () => { + const text = promptMetadataTextFromPayload({ + input: [ + { type: 'text', text: CAPTION }, + { type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } }, + ], + }); + expect(text).toBe('[image]'); + }); + + it('strips a caption merged into the user text and keeps the rest', () => { + const text = promptMetadataTextFromPayload({ + input: [ + { type: 'text', text: `能展示但是没有快捷键提示${CAPTION}` }, + { type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } }, + ], + }); + expect(text).toBe('能展示但是没有快捷键提示 [image]'); + expect(text).not.toContain(''); + expect(text).not.toContain('Image compressed'); + }); +}); diff --git a/packages/agent-core/test/tools/image-compress.test.ts b/packages/agent-core/test/tools/image-compress.test.ts index 3c6a89b21..cd26adedb 100644 --- a/packages/agent-core/test/tools/image-compress.test.ts +++ b/packages/agent-core/test/tools/image-compress.test.ts @@ -36,6 +36,7 @@ import { compressImageContentParts, compressImageForModel, cropImageForModel, + extractImageCompressionCaptions, IMAGE_BYTE_BUDGET, MAX_IMAGE_EDGE_PX, } from '../../src/tools/support/image-compress'; @@ -592,6 +593,54 @@ describe('buildImageCompressionCaption', () => { }); }); +describe('extractImageCompressionCaptions', () => { + const caption = buildImageCompressionCaption({ + original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' }, + final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' }, + originalPath: '/tmp/originals/shot.png', + }); + + it('extracts a standalone caption, unwrapping the tag', () => { + const result = extractImageCompressionCaptions(caption); + expect(result.captions).toHaveLength(1); + expect(result.captions[0]).toContain('Image compressed to fit model limits'); + expect(result.captions[0]).toContain('/tmp/originals/shot.png'); + expect(result.captions[0]).not.toContain(''); + expect(result.text).toBe(''); + }); + + it('extracts a caption merged into surrounding user text', () => { + const result = extractImageCompressionCaptions(`能展示但是没有快捷键提示${caption}`); + expect(result.captions).toHaveLength(1); + expect(result.text).toBe('能展示但是没有快捷键提示'); + }); + + it('extracts multiple captions from one text', () => { + const other = buildImageCompressionCaption({ + original: { width: 4000, height: 3000, byteLength: 9 * 1024 * 1024, mimeType: 'image/jpeg' }, + final: { width: 2000, height: 1500, byteLength: 1024 * 1024, mimeType: 'image/jpeg' }, + originalPath: '/tmp/originals/photo.jpg', + }); + const result = extractImageCompressionCaptions(`看这两张图${caption}${other}`); + expect(result.captions).toHaveLength(2); + expect(result.captions[0]).toContain('/tmp/originals/shot.png'); + expect(result.captions[1]).toContain('/tmp/originals/photo.jpg'); + expect(result.text).toBe('看这两张图'); + }); + + it('leaves non-caption blocks and plain text untouched', () => { + const toolStatus = 'ERROR: Tool execution failed.'; + expect(extractImageCompressionCaptions(toolStatus)).toEqual({ + captions: [], + text: toolStatus, + }); + expect(extractImageCompressionCaptions('just some text')).toEqual({ + captions: [], + text: 'just some text', + }); + }); +}); + // ── content-part annotation ────────────────────────────────────────── describe('compressImageContentParts — annotate', () => {