mirror of
https://github.com/MoonshotAI/kimi-code.git
synced 2026-07-09 17:29:12 +00:00
fix(agent-core): route image-compression captions through hidden system reminders (#1348)
* fix(agent-core): route image-compression captions through hidden system reminders
Prompt ingestion (server upload/base64 route, TUI paste, ACP) annotates a
compressed image with an inline <system> caption inside the user's own
message. That raw markup rendered verbatim in every user-visible history
projection (TUI session replay, web UI) and leaked into session titles.
Split the caption out at the appendUserMessage chokepoint and deliver it
through the built-in system-reminder injection (origin
{kind: 'injection', variant: 'image_compression'}), which every UI already
hides. The model still receives the full note; ingestion sites and the wire
protocol are unchanged. Session titles/lastPrompt strip the caption the same
way. Tool-result captions (MCP) keep the established <system> convention.
Covered by unit tests plus an end-to-end smoke suite that drives
rpc.prompt/steer through the real turn pipeline and asserts the provider
wire request, stored history, replay records, and resume parity.
* chore: tighten changeset wording per gen-changesets conventions
This commit is contained in:
parent
021786f5a2
commit
175b95f3af
8 changed files with 460 additions and 4 deletions
5
.changeset/hide-image-compression-caption.md
Normal file
5
.changeset/hide-image-compression-caption.md
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
"@moonshot-ai/kimi-code": patch
|
||||
---
|
||||
|
||||
Fix compressed-image prompts leaking an internal `<system>` compression note into the visible message and the session title.
|
||||
|
|
@ -3,6 +3,7 @@ import { createToolMessage, type ContentPart, type Message } from '@moonshot-ai/
|
|||
import type { Agent } from '..';
|
||||
import { ErrorCodes, KimiError } from '../../errors';
|
||||
import type { ExecutableToolResult, LoopRecordedEvent } from '../../loop';
|
||||
import { extractImageCompressionCaptions } from '../../tools/support/image-compress';
|
||||
import { estimateTokens, estimateTokensForMessages } from '../../utils/tokens';
|
||||
import { escapeXml } from '../../utils/xml-escape';
|
||||
import {
|
||||
|
|
@ -66,9 +67,23 @@ export class ContextMemory {
|
|||
origin: PromptOrigin = USER_PROMPT_ORIGIN,
|
||||
): void {
|
||||
if (content.length === 0) return;
|
||||
// Prompt ingestion (server upload/base64 route, TUI paste, ACP) annotates
|
||||
// a compressed image with an inline `<system>` caption next to the image.
|
||||
// Left inside the user message, that raw markup is user-visible in every
|
||||
// history projection (TUI replay, vis, export). Reroute each caption
|
||||
// through the built-in system-reminder injection — hidden by its
|
||||
// `injection` origin — and keep only the real user content here.
|
||||
const { captions, parts } =
|
||||
origin.kind === 'user'
|
||||
? splitImageCompressionCaptions(content)
|
||||
: { captions: [], parts: [...content] };
|
||||
for (const caption of captions) {
|
||||
this.appendSystemReminder(caption, { kind: 'injection', variant: 'image_compression' });
|
||||
}
|
||||
if (parts.length === 0) return;
|
||||
this.appendMessage({
|
||||
role: 'user',
|
||||
content: [...content],
|
||||
content: parts,
|
||||
toolCalls: [],
|
||||
origin,
|
||||
});
|
||||
|
|
@ -702,6 +717,35 @@ function isEmptyEquivalentContentArray(output: readonly ContentPart[]): boolean
|
|||
return output.every((part) => part.type === 'text' && part.text.trim().length === 0);
|
||||
}
|
||||
|
||||
// Split inline image-compression captions (see buildImageCompressionCaption)
|
||||
// out of user prompt content. A caption may be a standalone text part (server
|
||||
// route, ACP) or merged into an adjacent text segment (TUI paste), so each
|
||||
// text part is scanned rather than matched whole. Text left empty once its
|
||||
// captions are removed is dropped entirely.
|
||||
function splitImageCompressionCaptions(content: readonly ContentPart[]): {
|
||||
captions: readonly string[];
|
||||
parts: ContentPart[];
|
||||
} {
|
||||
const captions: string[] = [];
|
||||
const parts: ContentPart[] = [];
|
||||
for (const part of content) {
|
||||
if (part.type !== 'text') {
|
||||
parts.push(part);
|
||||
continue;
|
||||
}
|
||||
const extracted = extractImageCompressionCaptions(part.text);
|
||||
if (extracted.captions.length === 0) {
|
||||
parts.push(part);
|
||||
continue;
|
||||
}
|
||||
captions.push(...extracted.captions);
|
||||
if (extracted.text.trim().length > 0) {
|
||||
parts.push({ type: 'text', text: extracted.text });
|
||||
}
|
||||
}
|
||||
return { captions, parts };
|
||||
}
|
||||
|
||||
function isEmptyOutputText(output: string): boolean {
|
||||
return output.trim().length === 0 || output.trim() === TOOL_OUTPUT_EMPTY_TEXT;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import type { ActivatePluginCommandPayload, ActivateSkillPayload, PromptPayload } from '#/rpc';
|
||||
import { extractImageCompressionCaptions } from '#/tools/support/image-compress';
|
||||
import type { ContentPart } from '@moonshot-ai/kosong';
|
||||
|
||||
const MAX_TITLE_LENGTH = 200;
|
||||
|
|
@ -38,8 +39,13 @@ export function promptMetadataTextFromPluginCommand(
|
|||
|
||||
function promptPartText(part: ContentPart): string | undefined {
|
||||
switch (part.type) {
|
||||
case 'text':
|
||||
return part.text;
|
||||
case 'text': {
|
||||
// Prompt ingestion may have annotated a compressed image with an inline
|
||||
// caption (see buildImageCompressionCaption). It is harness metadata,
|
||||
// not something the user typed, so keep it out of titles/lastPrompt.
|
||||
const { text } = extractImageCompressionCaptions(part.text);
|
||||
return text.trim().length === 0 ? undefined : text;
|
||||
}
|
||||
case 'image_url':
|
||||
return '[image]';
|
||||
case 'audio_url':
|
||||
|
|
|
|||
|
|
@ -20,7 +20,10 @@
|
|||
* original dimensions, {@link buildImageCompressionCaption} renders the
|
||||
* shared "what was compressed, where is the original" note every ingestion
|
||||
* point can place next to the image, and {@link cropImageForModel} lets a
|
||||
* caller read a region of the original back at full fidelity.
|
||||
* caller read a region of the original back at full fidelity. In user
|
||||
* prompts the context layer later reroutes that note through the hidden
|
||||
* system-reminder injection via {@link extractImageCompressionCaptions},
|
||||
* so its raw `<system>` markup never renders in the UI.
|
||||
*/
|
||||
|
||||
import type { ContentPart } from '@moonshot-ai/kosong';
|
||||
|
|
@ -553,6 +556,14 @@ export interface ImageCompressionCaptionInput {
|
|||
* model knows it is looking at a downsampled copy: what the original was, what
|
||||
* was actually sent, and — when the original is on disk — where to read it
|
||||
* back (via ReadMediaFile `region`) for full-fidelity detail.
|
||||
*
|
||||
* Two channels consume this note differently:
|
||||
* - Tool results (MCP images) keep it inline — `<system>` status text inside
|
||||
* tool output is the established convention there.
|
||||
* - User prompts must not render raw `<system>` markup in the UI, so the
|
||||
* context layer detects the caption via
|
||||
* {@link extractImageCompressionCaptions} and reroutes it through the
|
||||
* built-in system-reminder injection (hidden by its `injection` origin).
|
||||
*/
|
||||
export function buildImageCompressionCaption(input: ImageCompressionCaptionInput): string {
|
||||
const sentences = [
|
||||
|
|
@ -572,6 +583,46 @@ export function buildImageCompressionCaption(input: ImageCompressionCaptionInput
|
|||
return `<system>${sentences.join(' ')}</system>`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fixed opening every {@link buildImageCompressionCaption} note starts with —
|
||||
* the anchor {@link extractImageCompressionCaptions} matches on. Keep the two
|
||||
* in sync.
|
||||
*/
|
||||
const CAPTION_OPENING = '<system>Image compressed to fit model limits:';
|
||||
|
||||
/**
|
||||
* A full caption embedded in arbitrary text. The body is sentences plus a
|
||||
* quoted file path and never contains `</system>`, so the non-greedy scan to
|
||||
* the closing tag is exact.
|
||||
*/
|
||||
const CAPTION_PATTERN = /<system>(Image compressed to fit model limits:[\s\S]*?)<\/system>/g;
|
||||
|
||||
export interface ImageCompressionCaptionExtraction {
|
||||
/** Caption bodies found, in order, without the `<system>` wrapper. */
|
||||
readonly captions: readonly string[];
|
||||
/** The input text with every caption removed. */
|
||||
readonly text: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find every {@link buildImageCompressionCaption} note embedded in `text` and
|
||||
* return the unwrapped caption bodies plus the text without them. Prompt
|
||||
* ingestion (server upload/base64 route, TUI paste, ACP) places the caption
|
||||
* inline next to the image — sometimes merged into an adjacent text segment —
|
||||
* and the context layer uses this to reroute the note through the built-in
|
||||
* system-reminder injection instead of leaving raw `<system>` markup in the
|
||||
* user-visible message.
|
||||
*/
|
||||
export function extractImageCompressionCaptions(text: string): ImageCompressionCaptionExtraction {
|
||||
if (!text.includes(CAPTION_OPENING)) return { captions: [], text };
|
||||
const captions: string[] = [];
|
||||
const remainder = text.replace(CAPTION_PATTERN, (_match, body: string) => {
|
||||
captions.push(body);
|
||||
return '';
|
||||
});
|
||||
return { captions, text: remainder };
|
||||
}
|
||||
|
||||
function describeImageVariant(variant: ImageVariantDescription): string {
|
||||
const size = `${variant.mimeType} (${formatByteSize(variant.byteLength)})`;
|
||||
if (variant.width > 0 && variant.height > 0) {
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import { describe, expect, it, vi } from 'vitest';
|
|||
import { renderNotificationXml } from '../../src/agent/context/notification-xml';
|
||||
import { project } from '../../src/agent/context/projector';
|
||||
import type { ContextMessage } from '../../src/agent/context/types';
|
||||
import { buildImageCompressionCaption } from '../../src/tools/support/image-compress';
|
||||
import { estimateTokensForMessages } from '../../src/utils/tokens';
|
||||
import { createFakeKaos } from '../tools/fixtures/fake-kaos';
|
||||
import { recordingTelemetry, type TelemetryRecord } from '../fixtures/telemetry';
|
||||
|
|
@ -59,6 +60,79 @@ describe('Agent context', () => {
|
|||
expect(ctx.agent.context.messages.some((message) => 'origin' in message)).toBe(false);
|
||||
});
|
||||
|
||||
it('reroutes an inline image-compression caption into a hidden system reminder', () => {
|
||||
const ctx = testAgent();
|
||||
ctx.configure();
|
||||
|
||||
const caption = buildImageCompressionCaption({
|
||||
original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' },
|
||||
final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' },
|
||||
originalPath: '/tmp/originals/shot.png',
|
||||
});
|
||||
// The TUI merges the caption into the preceding text segment; the server
|
||||
// route emits it as a standalone part. Cover the merged (harder) shape.
|
||||
ctx.agent.context.appendUserMessage([
|
||||
{ type: 'text', text: `能展示但是没有快捷键提示${caption}` },
|
||||
{ type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } },
|
||||
]);
|
||||
|
||||
const textOf = (message: ContextMessage): string =>
|
||||
message.content.map((part) => (part.type === 'text' ? part.text : '')).join('');
|
||||
|
||||
expect(ctx.agent.context.history.map(({ role, origin }) => ({ role, origin }))).toEqual([
|
||||
{ role: 'user', origin: { kind: 'injection', variant: 'image_compression' } },
|
||||
{ role: 'user', origin: { kind: 'user' } },
|
||||
]);
|
||||
const [reminder, userMessage] = ctx.agent.context.history;
|
||||
expect(textOf(reminder!)).toContain('<system-reminder>');
|
||||
expect(textOf(reminder!)).toContain('Image compressed to fit model limits');
|
||||
expect(textOf(reminder!)).toContain('/tmp/originals/shot.png');
|
||||
expect(textOf(reminder!)).not.toContain('<system>');
|
||||
expect(textOf(userMessage!)).toBe('能展示但是没有快捷键提示');
|
||||
expect(userMessage!.content.some((part) => part.type === 'image_url')).toBe(true);
|
||||
});
|
||||
|
||||
it('drops a caption-only text part instead of leaving an empty user text part', () => {
|
||||
const ctx = testAgent();
|
||||
ctx.configure();
|
||||
|
||||
const caption = buildImageCompressionCaption({
|
||||
original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' },
|
||||
final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' },
|
||||
originalPath: '/tmp/originals/shot.png',
|
||||
});
|
||||
ctx.agent.context.appendUserMessage([
|
||||
{ type: 'text', text: caption },
|
||||
{ type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } },
|
||||
]);
|
||||
|
||||
const [, userMessage] = ctx.agent.context.history;
|
||||
expect(userMessage!.content).toEqual([
|
||||
{ type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } },
|
||||
]);
|
||||
});
|
||||
|
||||
it('leaves caption-shaped text alone on non-user origins', () => {
|
||||
const ctx = testAgent();
|
||||
ctx.configure();
|
||||
|
||||
const caption = buildImageCompressionCaption({
|
||||
original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' },
|
||||
final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' },
|
||||
originalPath: '/tmp/originals/shot.png',
|
||||
});
|
||||
ctx.agent.context.appendUserMessage([{ type: 'text', text: caption }], {
|
||||
kind: 'hook_result',
|
||||
event: 'PostToolUse',
|
||||
});
|
||||
|
||||
expect(ctx.agent.context.history).toHaveLength(1);
|
||||
expect(ctx.agent.context.history[0]!.origin).toEqual({
|
||||
kind: 'hook_result',
|
||||
event: 'PostToolUse',
|
||||
});
|
||||
});
|
||||
|
||||
it('tracks conversation_undo when undoHistory reverts a user message', async () => {
|
||||
const records: TelemetryRecord[] = [];
|
||||
const ctx = testAgent({ telemetry: recordingTelemetry(records) });
|
||||
|
|
|
|||
172
packages/agent-core/test/agent/image-caption-injection.test.ts
Normal file
172
packages/agent-core/test/agent/image-caption-injection.test.ts
Normal file
|
|
@ -0,0 +1,172 @@
|
|||
/**
|
||||
* End-to-end smoke for image-compression caption rerouting.
|
||||
*
|
||||
* Prompt ingestion (server route, TUI paste, ACP) annotates a compressed
|
||||
* image with an inline `<system>` caption inside the user's own message.
|
||||
* The context layer must split that caption out into a hidden
|
||||
* system-reminder injection so no raw `<system>` markup is ever rendered in
|
||||
* a user bubble, while the model still receives the note.
|
||||
*
|
||||
* These tests drive the REAL pipeline — rpc.prompt → turn → context →
|
||||
* scripted provider — and assert at every seam:
|
||||
* - the wire request the model receives (reminder present, user text clean)
|
||||
* - the stored context history (origins drive UI hiding)
|
||||
* - the recorded wire log (what session resume replays)
|
||||
* - resume parity via expectResumeMatches (the TUI replay data source)
|
||||
*/
|
||||
|
||||
import { expect, it } from 'vitest';
|
||||
|
||||
import { buildImageCompressionCaption } from '../../src/tools/support/image-compress';
|
||||
import { testAgent } from './harness/agent';
|
||||
|
||||
const CAPTION = buildImageCompressionCaption({
|
||||
original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' },
|
||||
final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' },
|
||||
originalPath: '/tmp/originals/shot.png',
|
||||
});
|
||||
|
||||
const SECOND_CAPTION = buildImageCompressionCaption({
|
||||
original: { width: 4000, height: 3000, byteLength: 9 * 1024 * 1024, mimeType: 'image/jpeg' },
|
||||
final: { width: 2000, height: 1500, byteLength: 1024 * 1024, mimeType: 'image/jpeg' },
|
||||
originalPath: '/tmp/originals/photo.jpg',
|
||||
});
|
||||
|
||||
const IMAGE_URL = 'data:image/png;base64,AAAA';
|
||||
|
||||
it('smoke: a compressed-image prompt reaches the model with the caption as a system reminder', async () => {
|
||||
const ctx = testAgent();
|
||||
ctx.configure();
|
||||
|
||||
ctx.mockNextResponse({ type: 'text', text: 'I can see the screenshot.' });
|
||||
// The TUI merges the caption into the preceding text segment — the exact
|
||||
// shape from the bug report.
|
||||
await ctx.rpc.prompt({
|
||||
input: [
|
||||
{ type: 'text', text: `能展示但是没有快捷键提示${CAPTION}` },
|
||||
{ type: 'image_url', imageUrl: { url: IMAGE_URL } },
|
||||
],
|
||||
});
|
||||
await ctx.untilTurnEnd();
|
||||
|
||||
// What the model actually received on the wire.
|
||||
const llmInput = JSON.stringify(ctx.lastLlmInput().input);
|
||||
expect(llmInput).toContain('<system-reminder>');
|
||||
expect(llmInput).toContain('Image compressed to fit model limits');
|
||||
expect(llmInput).toContain('/tmp/originals/shot.png');
|
||||
expect(llmInput).not.toContain('<system>');
|
||||
expect(llmInput).toContain('能展示但是没有快捷键提示');
|
||||
|
||||
// What the UI renders from: the reminder is a separate injection-origin
|
||||
// message; the user message holds only what the user actually submitted.
|
||||
const history = ctx.agent.context.history;
|
||||
expect(history.map(({ role, origin }) => ({ role, origin }))).toEqual([
|
||||
{ role: 'user', origin: { kind: 'injection', variant: 'image_compression' } },
|
||||
{ role: 'user', origin: { kind: 'user' } },
|
||||
{ role: 'assistant', origin: undefined },
|
||||
]);
|
||||
const userText = history[1]!.content
|
||||
.map((part) => (part.type === 'text' ? part.text : ''))
|
||||
.join('');
|
||||
expect(userText).toBe('能展示但是没有快捷键提示');
|
||||
expect(history[1]!.content.some((part) => part.type === 'image_url')).toBe(true);
|
||||
|
||||
// The recorded wire log is what session resume replays into the TUI: the
|
||||
// user append_message record must already be caption-free.
|
||||
const appendRecords = ctx.allEvents.filter(
|
||||
(event) => event.type === '[wire]' && event.event === 'context.append_message',
|
||||
);
|
||||
expect(appendRecords).toHaveLength(2);
|
||||
expect(JSON.stringify(appendRecords[0]!.args)).toContain('system-reminder');
|
||||
expect(JSON.stringify(appendRecords[1]!.args)).not.toContain('<system>');
|
||||
|
||||
// Resume the session from the records and require identical state.
|
||||
await ctx.expectResumeMatches();
|
||||
});
|
||||
|
||||
it('smoke: multiple compressed images in one prompt each announce via their own reminder', async () => {
|
||||
const ctx = testAgent();
|
||||
ctx.configure();
|
||||
|
||||
ctx.mockNextResponse({ type: 'text', text: 'Two images noted.' });
|
||||
// Server-route shape: standalone caption part directly before each image.
|
||||
await ctx.rpc.prompt({
|
||||
input: [
|
||||
{ type: 'text', text: CAPTION },
|
||||
{ type: 'image_url', imageUrl: { url: IMAGE_URL } },
|
||||
{ type: 'text', text: SECOND_CAPTION },
|
||||
{ type: 'image_url', imageUrl: { url: IMAGE_URL } },
|
||||
{ type: 'text', text: '对比一下这两张截图' },
|
||||
],
|
||||
});
|
||||
await ctx.untilTurnEnd();
|
||||
|
||||
const llmInput = JSON.stringify(ctx.lastLlmInput().input);
|
||||
expect(llmInput).toContain('/tmp/originals/shot.png');
|
||||
expect(llmInput).toContain('/tmp/originals/photo.jpg');
|
||||
expect(llmInput).not.toContain('<system>');
|
||||
|
||||
const history = ctx.agent.context.history;
|
||||
expect(history.map(({ role, origin }) => ({ role, origin }))).toEqual([
|
||||
{ role: 'user', origin: { kind: 'injection', variant: 'image_compression' } },
|
||||
{ role: 'user', origin: { kind: 'injection', variant: 'image_compression' } },
|
||||
{ role: 'user', origin: { kind: 'user' } },
|
||||
{ role: 'assistant', origin: undefined },
|
||||
]);
|
||||
const userMessage = history[2]!;
|
||||
expect(userMessage.content.filter((part) => part.type === 'image_url')).toHaveLength(2);
|
||||
const userText = userMessage.content
|
||||
.map((part) => (part.type === 'text' ? part.text : ''))
|
||||
.join('');
|
||||
expect(userText).toBe('对比一下这两张截图');
|
||||
|
||||
await ctx.expectResumeMatches();
|
||||
});
|
||||
|
||||
it('smoke: a steered prompt with a caption is split the same way', async () => {
|
||||
const ctx = testAgent();
|
||||
ctx.configure();
|
||||
|
||||
ctx.mockNextResponse({ type: 'text', text: 'Steered.' });
|
||||
await ctx.rpc.steer({
|
||||
input: [
|
||||
{ type: 'text', text: `看这张${CAPTION}` },
|
||||
{ type: 'image_url', imageUrl: { url: IMAGE_URL } },
|
||||
],
|
||||
});
|
||||
await ctx.untilTurnEnd();
|
||||
|
||||
const history = ctx.agent.context.history;
|
||||
expect(history.map(({ role, origin }) => ({ role, origin }))).toEqual([
|
||||
{ role: 'user', origin: { kind: 'injection', variant: 'image_compression' } },
|
||||
{ role: 'user', origin: { kind: 'user' } },
|
||||
{ role: 'assistant', origin: undefined },
|
||||
]);
|
||||
const userText = history[1]!.content
|
||||
.map((part) => (part.type === 'text' ? part.text : ''))
|
||||
.join('');
|
||||
expect(userText).toBe('看这张');
|
||||
|
||||
await ctx.expectResumeMatches();
|
||||
});
|
||||
|
||||
it('smoke: a prompt without images is completely untouched', async () => {
|
||||
const ctx = testAgent();
|
||||
ctx.configure();
|
||||
|
||||
ctx.mockNextResponse({ type: 'text', text: 'Hi.' });
|
||||
await ctx.rpc.prompt({ input: [{ type: 'text', text: 'plain hello' }] });
|
||||
await ctx.untilTurnEnd();
|
||||
|
||||
const history = ctx.agent.context.history;
|
||||
expect(history.map(({ role, origin }) => ({ role, origin }))).toEqual([
|
||||
{ role: 'user', origin: { kind: 'user' } },
|
||||
{ role: 'assistant', origin: undefined },
|
||||
]);
|
||||
const userText = history[0]!.content
|
||||
.map((part) => (part.type === 'text' ? part.text : ''))
|
||||
.join('');
|
||||
expect(userText).toBe('plain hello');
|
||||
|
||||
await ctx.expectResumeMatches();
|
||||
});
|
||||
55
packages/agent-core/test/session/prompt-metadata.test.ts
Normal file
55
packages/agent-core/test/session/prompt-metadata.test.ts
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
/**
|
||||
* prompt-metadata — the session title / lastPrompt text derived from a
|
||||
* prompt payload.
|
||||
*
|
||||
* Tests pin:
|
||||
* - media parts render as `[image]` / `[video]` / `[audio]` placeholders
|
||||
* - an inline image-compression caption (harness metadata placed next to
|
||||
* the image by prompt ingestion) never leaks into titles/lastPrompt,
|
||||
* whether it is a standalone text part or merged into the user's text
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { promptMetadataTextFromPayload } from '../../src/session/prompt-metadata';
|
||||
import { buildImageCompressionCaption } from '../../src/tools/support/image-compress';
|
||||
|
||||
const CAPTION = buildImageCompressionCaption({
|
||||
original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' },
|
||||
final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' },
|
||||
originalPath: '/tmp/originals/shot.png',
|
||||
});
|
||||
|
||||
describe('promptMetadataTextFromPayload', () => {
|
||||
it('renders text and media placeholders', () => {
|
||||
const text = promptMetadataTextFromPayload({
|
||||
input: [
|
||||
{ type: 'text', text: 'look at this' },
|
||||
{ type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } },
|
||||
],
|
||||
});
|
||||
expect(text).toBe('look at this [image]');
|
||||
});
|
||||
|
||||
it('keeps a standalone image-compression caption out of the metadata text', () => {
|
||||
const text = promptMetadataTextFromPayload({
|
||||
input: [
|
||||
{ type: 'text', text: CAPTION },
|
||||
{ type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } },
|
||||
],
|
||||
});
|
||||
expect(text).toBe('[image]');
|
||||
});
|
||||
|
||||
it('strips a caption merged into the user text and keeps the rest', () => {
|
||||
const text = promptMetadataTextFromPayload({
|
||||
input: [
|
||||
{ type: 'text', text: `能展示但是没有快捷键提示${CAPTION}` },
|
||||
{ type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAAA' } },
|
||||
],
|
||||
});
|
||||
expect(text).toBe('能展示但是没有快捷键提示 [image]');
|
||||
expect(text).not.toContain('<system>');
|
||||
expect(text).not.toContain('Image compressed');
|
||||
});
|
||||
});
|
||||
|
|
@ -36,6 +36,7 @@ import {
|
|||
compressImageContentParts,
|
||||
compressImageForModel,
|
||||
cropImageForModel,
|
||||
extractImageCompressionCaptions,
|
||||
IMAGE_BYTE_BUDGET,
|
||||
MAX_IMAGE_EDGE_PX,
|
||||
} from '../../src/tools/support/image-compress';
|
||||
|
|
@ -592,6 +593,54 @@ describe('buildImageCompressionCaption', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('extractImageCompressionCaptions', () => {
|
||||
const caption = buildImageCompressionCaption({
|
||||
original: { width: 3264, height: 666, byteLength: 344 * 1024, mimeType: 'image/png' },
|
||||
final: { width: 2000, height: 408, byteLength: 282 * 1024, mimeType: 'image/png' },
|
||||
originalPath: '/tmp/originals/shot.png',
|
||||
});
|
||||
|
||||
it('extracts a standalone caption, unwrapping the <system> tag', () => {
|
||||
const result = extractImageCompressionCaptions(caption);
|
||||
expect(result.captions).toHaveLength(1);
|
||||
expect(result.captions[0]).toContain('Image compressed to fit model limits');
|
||||
expect(result.captions[0]).toContain('/tmp/originals/shot.png');
|
||||
expect(result.captions[0]).not.toContain('<system>');
|
||||
expect(result.text).toBe('');
|
||||
});
|
||||
|
||||
it('extracts a caption merged into surrounding user text', () => {
|
||||
const result = extractImageCompressionCaptions(`能展示但是没有快捷键提示${caption}`);
|
||||
expect(result.captions).toHaveLength(1);
|
||||
expect(result.text).toBe('能展示但是没有快捷键提示');
|
||||
});
|
||||
|
||||
it('extracts multiple captions from one text', () => {
|
||||
const other = buildImageCompressionCaption({
|
||||
original: { width: 4000, height: 3000, byteLength: 9 * 1024 * 1024, mimeType: 'image/jpeg' },
|
||||
final: { width: 2000, height: 1500, byteLength: 1024 * 1024, mimeType: 'image/jpeg' },
|
||||
originalPath: '/tmp/originals/photo.jpg',
|
||||
});
|
||||
const result = extractImageCompressionCaptions(`看这两张图${caption}${other}`);
|
||||
expect(result.captions).toHaveLength(2);
|
||||
expect(result.captions[0]).toContain('/tmp/originals/shot.png');
|
||||
expect(result.captions[1]).toContain('/tmp/originals/photo.jpg');
|
||||
expect(result.text).toBe('看这两张图');
|
||||
});
|
||||
|
||||
it('leaves non-caption <system> blocks and plain text untouched', () => {
|
||||
const toolStatus = '<system>ERROR: Tool execution failed.</system>';
|
||||
expect(extractImageCompressionCaptions(toolStatus)).toEqual({
|
||||
captions: [],
|
||||
text: toolStatus,
|
||||
});
|
||||
expect(extractImageCompressionCaptions('just some text')).toEqual({
|
||||
captions: [],
|
||||
text: 'just some text',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// ── content-part annotation ──────────────────────────────────────────
|
||||
|
||||
describe('compressImageContentParts — annotate', () => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue