fix(agent-core): route image-compression captions through hidden system reminders (#1348)

* fix(agent-core): route image-compression captions through hidden system reminders

Prompt ingestion (server upload/base64 route, TUI paste, ACP) annotates a
compressed image with an inline <system> caption inside the user's own
message. That raw markup rendered verbatim in every user-visible history
projection (TUI session replay, web UI) and leaked into session titles.

Split the caption out at the appendUserMessage chokepoint and deliver it
through the built-in system-reminder injection (origin
{kind: 'injection', variant: 'image_compression'}), which every UI already
hides. The model still receives the full note; ingestion sites and the wire
protocol are unchanged. Session titles/lastPrompt strip the caption the same
way. Tool-result captions (MCP) keep the established <system> convention.

Covered by unit tests plus an end-to-end smoke suite that drives
rpc.prompt/steer through the real turn pipeline and asserts the provider
wire request, stored history, replay records, and resume parity.

* chore: tighten changeset wording per gen-changesets conventions
This commit is contained in:
Kai 2026-07-03 16:21:10 +08:00 committed by GitHub
parent 021786f5a2
commit 175b95f3af
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 460 additions and 4 deletions

View file

@ -0,0 +1,5 @@
---
"@moonshot-ai/kimi-code": patch
---
Fix compressed-image prompts leaking an internal `<system>` compression note into the visible message and the session title.

View file

@ -3,6 +3,7 @@ import { createToolMessage, type ContentPart, type Message } from '@moonshot-ai/
import type { Agent } from '..';
import { ErrorCodes, KimiError } from '../../errors';
import type { ExecutableToolResult, LoopRecordedEvent } from '../../loop';
import { extractImageCompressionCaptions } from '../../tools/support/image-compress';
import { estimateTokens, estimateTokensForMessages } from '../../utils/tokens';
import { escapeXml } from '../../utils/xml-escape';
import {
@ -66,9 +67,23 @@ export class ContextMemory {
origin: PromptOrigin = USER_PROMPT_ORIGIN,
): void {
if (content.length === 0) return;
// Prompt ingestion (server upload/base64 route, TUI paste, ACP) annotates
// a compressed image with an inline `<system>` caption next to the image.
// Left inside the user message, that raw markup is user-visible in every
// history projection (TUI replay, vis, export). Reroute each caption
// through the built-in system-reminder injection — hidden by its
// `injection` origin — and keep only the real user content here.
const { captions, parts } =
origin.kind === 'user'
? splitImageCompressionCaptions(content)
: { captions: [], parts: [...content] };
for (const caption of captions) {
this.appendSystemReminder(caption, { kind: 'injection', variant: 'image_compression' });
}
if (parts.length === 0) return;
this.appendMessage({
role: 'user',
content: [...content],
content: parts,
toolCalls: [],
origin,
});
@ -702,6 +717,35 @@ function isEmptyEquivalentContentArray(output: readonly ContentPart[]): boolean
return output.every((part) => part.type === 'text' && part.text.trim().length === 0);
}
// Split inline image-compression captions (see buildImageCompressionCaption)
// out of user prompt content. A caption may be a standalone text part (server
// route, ACP) or merged into an adjacent text segment (TUI paste), so each
// text part is scanned rather than matched whole. Text left empty once its
// captions are removed is dropped entirely.
function splitImageCompressionCaptions(content: readonly ContentPart[]): {
captions: readonly string[];
parts: ContentPart[];
} {
const captions: string[] = [];
const parts: ContentPart[] = [];
for (const part of content) {
if (part.type !== 'text') {
parts.push(part);
continue;
}
const extracted = extractImageCompressionCaptions(part.text);
if (extracted.captions.length === 0) {
parts.push(part);
continue;
}
captions.push(...extracted.captions);
if (extracted.text.trim().length > 0) {
parts.push({ type: 'text', text: extracted.text });
}
}
return { captions, parts };
}
function isEmptyOutputText(output: string): boolean {
return output.trim().length === 0 || output.trim() === TOOL_OUTPUT_EMPTY_TEXT;
}

View file

@ -1,4 +1,5 @@
import type { ActivatePluginCommandPayload, ActivateSkillPayload, PromptPayload } from '#/rpc';
import { extractImageCompressionCaptions } from '#/tools/support/image-compress';
import type { ContentPart } from '@moonshot-ai/kosong';
const MAX_TITLE_LENGTH = 200;
@ -38,8 +39,13 @@ export function promptMetadataTextFromPluginCommand(
function promptPartText(part: ContentPart): string | undefined {
switch (part.type) {
case 'text':
return part.text;
case 'text': {
// Prompt ingestion may have annotated a compressed image with an inline
// caption (see buildImageCompressionCaption). It is harness metadata,
// not something the user typed, so keep it out of titles/lastPrompt.
const { text } = extractImageCompressionCaptions(part.text);
return text.trim().length === 0 ? undefined : text;
}
case 'image_url':
return '[image]';
case 'audio_url':

View file

@ -20,7 +20,10 @@
* original dimensions, {@link buildImageCompressionCaption} renders the
* shared "what was compressed, where is the original" note every ingestion
* point can place next to the image, and {@link cropImageForModel} lets a
* caller read a region of the original back at full fidelity.
* caller read a region of the original back at full fidelity. In user
* prompts the context layer later reroutes that note through the hidden
* system-reminder injection via {@link extractImageCompressionCaptions},
* so its raw `<system>` markup never renders in the UI.
*/
import type { ContentPart } from '@moonshot-ai/kosong';
@ -553,6 +556,14 @@ export interface ImageCompressionCaptionInput {
* model knows it is looking at a downsampled copy: what the original was, what
* was actually sent, and when the original is on disk where to read it
* back (via ReadMediaFile `region`) for full-fidelity detail.
*
* Two channels consume this note differently:
* - Tool results (MCP images) keep it inline `<system>` status text inside
* tool output is the established convention there.
* - User prompts must not render raw `<system>` markup in the UI, so the
* context layer detects the caption via
* {@link extractImageCompressionCaptions} and reroutes it through the
* built-in system-reminder injection (hidden by its `injection` origin).
*/
export function buildImageCompressionCaption(input: ImageCompressionCaptionInput): string {
const sentences = [
@ -572,6 +583,46 @@ export function buildImageCompressionCaption(input: ImageCompressionCaptionInput
return `<system>${sentences.join(' ')}</system>`;
}
/**
* Fixed opening every {@link buildImageCompressionCaption} note starts with
* the anchor {@link extractImageCompressionCaptions} matches on. Keep the two
* in sync.
*/
const CAPTION_OPENING = '<system>Image compressed to fit model limits:';
/**
* A full caption embedded in arbitrary text. The body is sentences plus a
* quoted file path and never contains `</system>`, so the non-greedy scan to
* the closing tag is exact.
*/
const CAPTION_PATTERN = /<system>(Image compressed to fit model limits:[\s\S]*?)<\/system>/g;
export interface ImageCompressionCaptionExtraction {
/** Caption bodies found, in order, without the `<system>` wrapper. */
readonly captions: readonly string[];
/** The input text with every caption removed. */
readonly text: string;
}
/**
* Find every {@link buildImageCompressionCaption} note embedded in `text` and
* return the unwrapped caption bodies plus the text without them. Prompt
* ingestion (server upload/base64 route, TUI paste, ACP) places the caption
* inline next to the image sometimes merged into an adjacent text segment
* and the context layer uses this to reroute the note through the built-in
* system-reminder injection instead of leaving raw `<system>` markup in the
* user-visible message.
*/
export function extractImageCompressionCaptions(text: string): ImageCompressionCaptionExtraction {
if (!text.includes(CAPTION_OPENING)) return { captions: [], text };
const captions: string[] = [];
const remainder = text.replace(CAPTION_PATTERN, (_match, body: string) => {
captions.push(body);
return '';
});
return { captions, text: remainder };
}
function describeImageVariant(variant: ImageVariantDescription): string {
const size = `${variant.mimeType} (${formatByteSize(variant.byteLength)})`;
if (variant.width > 0 && variant.height > 0) {

View file

@ -7,6 +7,7 @@ import { describe, expect, it, vi } from 'vitest';
import { renderNotificationXml } from '../../src/agent/context/notification-xml';
import { project } from '../../src/agent/context/projector';
import type { ContextMessage } from '../../src/agent/context/types';
import { buildImageCompressionCaption } from '../../src/tools/support/image-compress';
import { estimateTokensForMessages } from '../../src/utils/tokens';
import { createFakeKaos } from '../tools/fixtures/fake-kaos';
import { recordingTelemetry, type TelemetryRecord } from '../fixtures/telemetry';
@ -59,6 +60,79 @@ describe('Agent context', () => {
expect(ctx.agent.context.messages.some((message) => 'origin' in message)).toBe(false);
});
it('reroutes an inline image-compression caption into a hidden system reminder', () => {
const ctx = testAgent();
ctx.configure();
const caption = buildImageCompressionCaption({
original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' },
final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' },
originalPath: '/tmp/originals/shot.png',
});
// The TUI merges the caption into the preceding text segment; the server
// route emits it as a standalone part. Cover the merged (harder) shape.
ctx.agent.context.appendUserMessage([
{ type: 'text', text: `能展示但是没有快捷键提示${caption}` },
{ type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } },
]);
const textOf = (message: ContextMessage): string =>
message.content.map((part) => (part.type === 'text' ? part.text : '')).join('');
expect(ctx.agent.context.history.map(({ role, origin }) => ({ role, origin }))).toEqual([
{ role: 'user', origin: { kind: 'injection', variant: 'image_compression' } },
{ role: 'user', origin: { kind: 'user' } },
]);
const [reminder, userMessage] = ctx.agent.context.history;
expect(textOf(reminder!)).toContain('<system-reminder>');
expect(textOf(reminder!)).toContain('Image compressed to fit model limits');
expect(textOf(reminder!)).toContain('/tmp/originals/shot.png');
expect(textOf(reminder!)).not.toContain('<system>');
expect(textOf(userMessage!)).toBe('能展示但是没有快捷键提示');
expect(userMessage!.content.some((part) => part.type === 'image_url')).toBe(true);
});
it('drops a caption-only text part instead of leaving an empty user text part', () => {
const ctx = testAgent();
ctx.configure();
const caption = buildImageCompressionCaption({
original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' },
final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' },
originalPath: '/tmp/originals/shot.png',
});
ctx.agent.context.appendUserMessage([
{ type: 'text', text: caption },
{ type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } },
]);
const [, userMessage] = ctx.agent.context.history;
expect(userMessage!.content).toEqual([
{ type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } },
]);
});
it('leaves caption-shaped text alone on non-user origins', () => {
const ctx = testAgent();
ctx.configure();
const caption = buildImageCompressionCaption({
original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' },
final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' },
originalPath: '/tmp/originals/shot.png',
});
ctx.agent.context.appendUserMessage([{ type: 'text', text: caption }], {
kind: 'hook_result',
event: 'PostToolUse',
});
expect(ctx.agent.context.history).toHaveLength(1);
expect(ctx.agent.context.history[0]!.origin).toEqual({
kind: 'hook_result',
event: 'PostToolUse',
});
});
it('tracks conversation_undo when undoHistory reverts a user message', async () => {
const records: TelemetryRecord[] = [];
const ctx = testAgent({ telemetry: recordingTelemetry(records) });

View file

@ -0,0 +1,172 @@
/**
* End-to-end smoke for image-compression caption rerouting.
*
* Prompt ingestion (server route, TUI paste, ACP) annotates a compressed
* image with an inline `<system>` caption inside the user's own message.
* The context layer must split that caption out into a hidden
* system-reminder injection so no raw `<system>` markup is ever rendered in
* a user bubble, while the model still receives the note.
*
* These tests drive the REAL pipeline rpc.prompt turn context
* scripted provider and assert at every seam:
* - the wire request the model receives (reminder present, user text clean)
* - the stored context history (origins drive UI hiding)
* - the recorded wire log (what session resume replays)
* - resume parity via expectResumeMatches (the TUI replay data source)
*/
import { expect, it } from 'vitest';
import { buildImageCompressionCaption } from '../../src/tools/support/image-compress';
import { testAgent } from './harness/agent';
const CAPTION = buildImageCompressionCaption({
original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' },
final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' },
originalPath: '/tmp/originals/shot.png',
});
const SECOND_CAPTION = buildImageCompressionCaption({
original: { width: 4000, height: 3000, byteLength: 9 * 1024 * 1024, mimeType: 'image/jpeg' },
final: { width: 2000, height: 1500, byteLength: 1024 * 1024, mimeType: 'image/jpeg' },
originalPath: '/tmp/originals/photo.jpg',
});
const IMAGE_URL = 'data:image/png;base64,AAAA';
it('smoke: a compressed-image prompt reaches the model with the caption as a system reminder', async () => {
const ctx = testAgent();
ctx.configure();
ctx.mockNextResponse({ type: 'text', text: 'I can see the screenshot.' });
// The TUI merges the caption into the preceding text segment — the exact
// shape from the bug report.
await ctx.rpc.prompt({
input: [
{ type: 'text', text: `能展示但是没有快捷键提示${CAPTION}` },
{ type: 'image_url', imageUrl: { url: IMAGE_URL } },
],
});
await ctx.untilTurnEnd();
// What the model actually received on the wire.
const llmInput = JSON.stringify(ctx.lastLlmInput().input);
expect(llmInput).toContain('<system-reminder>');
expect(llmInput).toContain('Image compressed to fit model limits');
expect(llmInput).toContain('/tmp/originals/shot.png');
expect(llmInput).not.toContain('<system>');
expect(llmInput).toContain('能展示但是没有快捷键提示');
// What the UI renders from: the reminder is a separate injection-origin
// message; the user message holds only what the user actually submitted.
const history = ctx.agent.context.history;
expect(history.map(({ role, origin }) => ({ role, origin }))).toEqual([
{ role: 'user', origin: { kind: 'injection', variant: 'image_compression' } },
{ role: 'user', origin: { kind: 'user' } },
{ role: 'assistant', origin: undefined },
]);
const userText = history[1]!.content
.map((part) => (part.type === 'text' ? part.text : ''))
.join('');
expect(userText).toBe('能展示但是没有快捷键提示');
expect(history[1]!.content.some((part) => part.type === 'image_url')).toBe(true);
// The recorded wire log is what session resume replays into the TUI: the
// user append_message record must already be caption-free.
const appendRecords = ctx.allEvents.filter(
(event) => event.type === '[wire]' && event.event === 'context.append_message',
);
expect(appendRecords).toHaveLength(2);
expect(JSON.stringify(appendRecords[0]!.args)).toContain('system-reminder');
expect(JSON.stringify(appendRecords[1]!.args)).not.toContain('<system>');
// Resume the session from the records and require identical state.
await ctx.expectResumeMatches();
});
it('smoke: multiple compressed images in one prompt each announce via their own reminder', async () => {
const ctx = testAgent();
ctx.configure();
ctx.mockNextResponse({ type: 'text', text: 'Two images noted.' });
// Server-route shape: standalone caption part directly before each image.
await ctx.rpc.prompt({
input: [
{ type: 'text', text: CAPTION },
{ type: 'image_url', imageUrl: { url: IMAGE_URL } },
{ type: 'text', text: SECOND_CAPTION },
{ type: 'image_url', imageUrl: { url: IMAGE_URL } },
{ type: 'text', text: '对比一下这两张截图' },
],
});
await ctx.untilTurnEnd();
const llmInput = JSON.stringify(ctx.lastLlmInput().input);
expect(llmInput).toContain('/tmp/originals/shot.png');
expect(llmInput).toContain('/tmp/originals/photo.jpg');
expect(llmInput).not.toContain('<system>');
const history = ctx.agent.context.history;
expect(history.map(({ role, origin }) => ({ role, origin }))).toEqual([
{ role: 'user', origin: { kind: 'injection', variant: 'image_compression' } },
{ role: 'user', origin: { kind: 'injection', variant: 'image_compression' } },
{ role: 'user', origin: { kind: 'user' } },
{ role: 'assistant', origin: undefined },
]);
const userMessage = history[2]!;
expect(userMessage.content.filter((part) => part.type === 'image_url')).toHaveLength(2);
const userText = userMessage.content
.map((part) => (part.type === 'text' ? part.text : ''))
.join('');
expect(userText).toBe('对比一下这两张截图');
await ctx.expectResumeMatches();
});
it('smoke: a steered prompt with a caption is split the same way', async () => {
const ctx = testAgent();
ctx.configure();
ctx.mockNextResponse({ type: 'text', text: 'Steered.' });
await ctx.rpc.steer({
input: [
{ type: 'text', text: `看这张${CAPTION}` },
{ type: 'image_url', imageUrl: { url: IMAGE_URL } },
],
});
await ctx.untilTurnEnd();
const history = ctx.agent.context.history;
expect(history.map(({ role, origin }) => ({ role, origin }))).toEqual([
{ role: 'user', origin: { kind: 'injection', variant: 'image_compression' } },
{ role: 'user', origin: { kind: 'user' } },
{ role: 'assistant', origin: undefined },
]);
const userText = history[1]!.content
.map((part) => (part.type === 'text' ? part.text : ''))
.join('');
expect(userText).toBe('看这张');
await ctx.expectResumeMatches();
});
it('smoke: a prompt without images is completely untouched', async () => {
const ctx = testAgent();
ctx.configure();
ctx.mockNextResponse({ type: 'text', text: 'Hi.' });
await ctx.rpc.prompt({ input: [{ type: 'text', text: 'plain hello' }] });
await ctx.untilTurnEnd();
const history = ctx.agent.context.history;
expect(history.map(({ role, origin }) => ({ role, origin }))).toEqual([
{ role: 'user', origin: { kind: 'user' } },
{ role: 'assistant', origin: undefined },
]);
const userText = history[0]!.content
.map((part) => (part.type === 'text' ? part.text : ''))
.join('');
expect(userText).toBe('plain hello');
await ctx.expectResumeMatches();
});

View file

@ -0,0 +1,55 @@
/**
* prompt-metadata the session title / lastPrompt text derived from a
* prompt payload.
*
* Tests pin:
* - media parts render as `[image]` / `[video]` / `[audio]` placeholders
* - an inline image-compression caption (harness metadata placed next to
* the image by prompt ingestion) never leaks into titles/lastPrompt,
* whether it is a standalone text part or merged into the user's text
*/
import { describe, expect, it } from 'vitest';
import { promptMetadataTextFromPayload } from '../../src/session/prompt-metadata';
import { buildImageCompressionCaption } from '../../src/tools/support/image-compress';
const CAPTION = buildImageCompressionCaption({
original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' },
final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' },
originalPath: '/tmp/originals/shot.png',
});
describe('promptMetadataTextFromPayload', () => {
it('renders text and media placeholders', () => {
const text = promptMetadataTextFromPayload({
input: [
{ type: 'text', text: 'look at this' },
{ type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } },
],
});
expect(text).toBe('look at this [image]');
});
it('keeps a standalone image-compression caption out of the metadata text', () => {
const text = promptMetadataTextFromPayload({
input: [
{ type: 'text', text: CAPTION },
{ type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } },
],
});
expect(text).toBe('[image]');
});
it('strips a caption merged into the user text and keeps the rest', () => {
const text = promptMetadataTextFromPayload({
input: [
{ type: 'text', text: `能展示但是没有快捷键提示${CAPTION}` },
{ type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } },
],
});
expect(text).toBe('能展示但是没有快捷键提示 [image]');
expect(text).not.toContain('<system>');
expect(text).not.toContain('Image compressed');
});
});

View file

@ -36,6 +36,7 @@ import {
compressImageContentParts,
compressImageForModel,
cropImageForModel,
extractImageCompressionCaptions,
IMAGE_BYTE_BUDGET,
MAX_IMAGE_EDGE_PX,
} from '../../src/tools/support/image-compress';
@ -592,6 +593,54 @@ describe('buildImageCompressionCaption', () => {
});
});
describe('extractImageCompressionCaptions', () => {
const caption = buildImageCompressionCaption({
original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' },
final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' },
originalPath: '/tmp/originals/shot.png',
});
it('extracts a standalone caption, unwrapping the <system> tag', () => {
const result = extractImageCompressionCaptions(caption);
expect(result.captions).toHaveLength(1);
expect(result.captions[0]).toContain('Image compressed to fit model limits');
expect(result.captions[0]).toContain('/tmp/originals/shot.png');
expect(result.captions[0]).not.toContain('<system>');
expect(result.text).toBe('');
});
it('extracts a caption merged into surrounding user text', () => {
const result = extractImageCompressionCaptions(`能展示但是没有快捷键提示${caption}`);
expect(result.captions).toHaveLength(1);
expect(result.text).toBe('能展示但是没有快捷键提示');
});
it('extracts multiple captions from one text', () => {
const other = buildImageCompressionCaption({
original: { width: 4000, height: 3000, byteLength: 9 * 1024 * 1024, mimeType: 'image/jpeg' },
final: { width: 2000, height: 1500, byteLength: 1024 * 1024, mimeType: 'image/jpeg' },
originalPath: '/tmp/originals/photo.jpg',
});
const result = extractImageCompressionCaptions(`看这两张图${caption}${other}`);
expect(result.captions).toHaveLength(2);
expect(result.captions[0]).toContain('/tmp/originals/shot.png');
expect(result.captions[1]).toContain('/tmp/originals/photo.jpg');
expect(result.text).toBe('看这两张图');
});
it('leaves non-caption <system> blocks and plain text untouched', () => {
const toolStatus = '<system>ERROR: Tool execution failed.</system>';
expect(extractImageCompressionCaptions(toolStatus)).toEqual({
captions: [],
text: toolStatus,
});
expect(extractImageCompressionCaptions('just some text')).toEqual({
captions: [],
text: 'just some text',
});
});
});
// ── content-part annotation ──────────────────────────────────────────
describe('compressImageContentParts — annotate', () => {