diff --git a/.changeset/image-compression-improvements.md b/.changeset/image-compression-improvements.md new file mode 100644 index 000000000..70e762223 --- /dev/null +++ b/.changeset/image-compression-improvements.md @@ -0,0 +1,6 @@ +--- +"@moonshot-ai/kimi-code": patch +"@moonshot-ai/kimi-code-sdk": patch +--- + +Raise the image downscale cap from 2000px to 3000px, and fix swapped width/height for EXIF-rotated (portrait) photos in compression captions and media read notes so region readback coordinates map correctly. diff --git a/apps/kimi-code/src/tui/controllers/editor-keyboard.ts b/apps/kimi-code/src/tui/controllers/editor-keyboard.ts index af2d8efd6..e76492f7b 100644 --- a/apps/kimi-code/src/tui/controllers/editor-keyboard.ts +++ b/apps/kimi-code/src/tui/controllers/editor-keyboard.ts @@ -407,8 +407,20 @@ export class EditorKeyboardController { // session's media-originals dir when known, else the temp-dir fallback) // and recorded on the attachment, so submit-time expansion can announce // the compression and point the model at the full-fidelity copy. - const compressed = await compressImageForModel(media.bytes, meta.mime); + const compressed = await compressImageForModel(media.bytes, meta.mime, { + telemetry: { + client: { + track: (event, properties) => + this.host.track(event, properties === undefined ? undefined : { ...properties }), + }, + source: 'tui_paste', + }, + }); const sessionDir = this.host.session?.summary?.sessionDir; + // Dimensions come from the compression result, not parseImageMeta: the + // compressor reports display space (EXIF orientation applied) — the space + // the sent image, the caption, and ReadMediaFile region readback share — + // while parseImageMeta reads the raw pre-rotation header. const attachment = compressed.changed ? this.imageStore.addImage( compressed.data, @@ -421,13 +433,18 @@ export class EditorKeyboardController { meta.mime, sessionDir === undefined ? {} : { dir: sessionMediaOriginalsDir(sessionDir) }, ), - width: meta.width, - height: meta.height, + width: compressed.originalWidth, + height: compressed.originalHeight, byteLength: media.bytes.length, mime: meta.mime, }, ) - : this.imageStore.addImage(media.bytes, meta.mime, meta.width, meta.height); + : this.imageStore.addImage( + media.bytes, + meta.mime, + compressed.width || meta.width, + compressed.height || meta.height, + ); this.host.state.editor.insertTextAtCursor?.(`${attachment.placeholder} `); this.host.state.ui.requestRender(); this.host.track('shortcut_paste', { kind: 'image' }); diff --git a/apps/kimi-code/test/tui/controllers/editor-keyboard-image-paste.test.ts b/apps/kimi-code/test/tui/controllers/editor-keyboard-image-paste.test.ts index 4be348fa2..0fe14bc09 100644 --- a/apps/kimi-code/test/tui/controllers/editor-keyboard-image-paste.test.ts +++ b/apps/kimi-code/test/tui/controllers/editor-keyboard-image-paste.test.ts @@ -37,6 +37,7 @@ vi.mock('#/utils/clipboard/clipboard-image', async (importActual) => { interface PasteHarness { readonly store: ImageAttachmentStore; + readonly track: ReturnType; pasteImage(): Promise; } @@ -45,6 +46,7 @@ function createPasteHarness(options: { sessionDir?: string } = {}): PasteHarness setHistoryFilter: vi.fn() as unknown as (...args: never[]) => unknown, }; const store = new ImageAttachmentStore(); + const track = vi.fn(); const host = { state: { editor, @@ -58,7 +60,7 @@ function createPasteHarness(options: { sessionDir?: string } = {}): PasteHarness ? undefined : { summary: { sessionDir: options.sessionDir } }, btwPanelController: { closeOrCancel: vi.fn(() => false) }, - track: vi.fn(), + track, showError: vi.fn(), openUndoSelector: vi.fn(), cancelRunningShellCommand: vi.fn(), @@ -69,6 +71,7 @@ function createPasteHarness(options: { sessionDir?: string } = {}): PasteHarness return { store, + track, async pasteImage() { const handler = editor['onPasteImage']; if (handler === undefined) throw new Error('onPasteImage handler not installed'); @@ -83,13 +86,50 @@ async function solidPng(width: number, height: number): Promise { ); } +async function solidJpeg(width: number, height: number): Promise { + return new Uint8Array( + await new Jimp({ width, height, color: 0x3366ccff }).getBuffer('image/jpeg', { quality: 90 }), + ); +} + +/** + * Insert a minimal EXIF APP1 segment carrying only an Orientation tag right + * after the JPEG SOI marker (jimp itself never writes EXIF). Mirrors the + * fixture in agent-core's image-compress tests. + */ +function withExifOrientation(jpeg: Uint8Array, orientation: number): Uint8Array { + // TIFF body, little-endian: 8-byte header + IFD0 with a single entry. + const tiff = Buffer.alloc(26); + tiff.write('II', 0, 'latin1'); + tiff.writeUInt16LE(42, 2); + tiff.writeUInt32LE(8, 4); // offset of IFD0 + tiff.writeUInt16LE(1, 8); // one directory entry + tiff.writeUInt16LE(0x0112, 10); // tag: Orientation + tiff.writeUInt16LE(3, 12); // type: SHORT + tiff.writeUInt32LE(1, 14); // count + tiff.writeUInt16LE(orientation, 18); // value, left-aligned in the 4-byte field + tiff.writeUInt32LE(0, 22); // no next IFD + const exifBody = Buffer.concat([Buffer.from('Exif\0\0', 'latin1'), tiff]); + const app1Header = Buffer.alloc(4); + app1Header.writeUInt16BE(0xff_e1, 0); + app1Header.writeUInt16BE(exifBody.length + 2, 2); + return new Uint8Array( + Buffer.concat([ + Buffer.from(jpeg.subarray(0, 2)), // SOI + app1Header, + exifBody, + Buffer.from(jpeg.subarray(2)), + ]), + ); +} + describe('clipboard image paste compression', () => { beforeEach(() => { readClipboardMedia.mockReset(); }); it('downsamples an oversized pasted image before storing it', async () => { - const big = await solidPng(2600, 2600); + const big = await solidPng(3600, 1800); readClipboardMedia.mockResolvedValue({ kind: 'image', bytes: big, mimeType: 'image/png' }); const { store, pasteImage } = createPasteHarness(); @@ -101,18 +141,18 @@ describe('clipboard image paste compression', () => { if (att?.kind !== 'image') throw new Error('expected image attachment'); // Stored metadata reflects the compressed size. - expect(Math.max(att.width, att.height)).toBeLessThanOrEqual(2000); - expect(att.placeholder).toContain('2000×2000'); + expect(Math.max(att.width, att.height)).toBeLessThanOrEqual(3000); + expect(att.placeholder).toContain('3000×1500'); // The stored bytes decode to the compressed dimensions — the thumbnail and // the submitted image both read from these bytes, so they cannot diverge. const dims = parseImageMeta(att.bytes); expect(dims).not.toBeNull(); - expect(Math.max(dims!.width, dims!.height)).toBeLessThanOrEqual(2000); + expect(Math.max(dims!.width, dims!.height)).toBeLessThanOrEqual(3000); }); it('records and persists the pre-compression original for an oversized paste', async () => { - const big = await solidPng(2600, 2600); + const big = await solidPng(3600, 1800); readClipboardMedia.mockResolvedValue({ kind: 'image', bytes: big, mimeType: 'image/png' }); const { store, pasteImage } = createPasteHarness(); @@ -121,8 +161,8 @@ describe('clipboard image paste compression', () => { const att = store.get(1); if (att?.kind !== 'image') throw new Error('expected image attachment'); expect(att.original).toBeDefined(); - expect(att.original?.width).toBe(2600); - expect(att.original?.height).toBe(2600); + expect(att.original?.width).toBe(3600); + expect(att.original?.height).toBe(1800); expect(att.original?.byteLength).toBe(big.length); expect(att.original?.mime).toBe('image/png'); @@ -135,7 +175,7 @@ describe('clipboard image paste compression', () => { it('persists the original into the session media-originals dir when the session is known', async () => { const sessionDir = await mkdtemp(join(tmpdir(), 'kimi-paste-session-')); - const big = await solidPng(2600, 2600); + const big = await solidPng(3600, 1800); readClipboardMedia.mockResolvedValue({ kind: 'image', bytes: big, mimeType: 'image/png' }); const { store, pasteImage } = createPasteHarness({ sessionDir }); @@ -164,4 +204,71 @@ describe('clipboard image paste compression', () => { expect(att.bytes).toBe(small); // identity: no re-encode on the fast path expect(att.original).toBeUndefined(); }); + + it( + 'records an EXIF-rotated compressed original in display space', + async () => { + // Orientation 6 (rotate 90° CW): the header says 3600x400, but the image + // decodes to 400x3600 — the space the compressed bytes and any later + // ReadMediaFile region readback live in. The recorded original (which + // drives the submit-time compression caption) must match that space, or + // the caption contradicts the sent image's aspect and region coordinates + // land axis-swapped. (Kept narrow: pure-JS decode+rotate+encode of a + // larger frame can outlast the test timeout on slow CI runners.) + const portrait = withExifOrientation(await solidJpeg(3600, 400), 6); + readClipboardMedia.mockResolvedValue({ + kind: 'image', + bytes: portrait, + mimeType: 'image/jpeg', + }); + + const { store, pasteImage } = createPasteHarness(); + await pasteImage(); + + const att = store.get(1); + if (att?.kind !== 'image') throw new Error('expected image attachment'); + expect(att.original?.width).toBe(400); + expect(att.original?.height).toBe(3600); + // The compressed attachment itself keeps the portrait aspect. + expect(att.width).toBeLessThan(att.height); + await unlink(att.original!.path!).catch(() => undefined); + }, + 15_000, + ); + + it('stores display-space dimensions for an EXIF-rotated untouched paste', async () => { + // Within budgets → sent byte-for-byte, but the placeholder and metadata + // must still describe the display (rotated) space. + const portrait = withExifOrientation(await solidJpeg(120, 80), 6); + readClipboardMedia.mockResolvedValue({ + kind: 'image', + bytes: portrait, + mimeType: 'image/jpeg', + }); + + const { store, pasteImage } = createPasteHarness(); + await pasteImage(); + + const att = store.get(1); + if (att?.kind !== 'image') throw new Error('expected image attachment'); + expect(att.bytes).toBe(portrait); // fast path — untouched + expect(att.original).toBeUndefined(); + expect(att.width).toBe(80); + expect(att.height).toBe(120); + expect(att.placeholder).toContain('80×120'); + }); + + it('emits image_compress telemetry tagged tui_paste through host.track', async () => { + const big = await solidPng(3600, 1800); + readClipboardMedia.mockResolvedValue({ kind: 'image', bytes: big, mimeType: 'image/png' }); + + const { track, pasteImage } = createPasteHarness(); + await pasteImage(); + + const compressCalls = track.mock.calls.filter(([event]) => event === 'image_compress'); + expect(compressCalls).toHaveLength(1); + const props = compressCalls[0]![1] as Record; + expect(props['source']).toBe('tui_paste'); + expect(props['outcome']).toBe('compressed'); + }); }); diff --git a/packages/acp-adapter/src/convert.ts b/packages/acp-adapter/src/convert.ts index c2a3ca8d7..bf8cdb197 100644 --- a/packages/acp-adapter/src/convert.ts +++ b/packages/acp-adapter/src/convert.ts @@ -5,6 +5,7 @@ import { compressBase64ForModel, persistOriginalImage, type PromptPart, + type TelemetryClient, type ToolInputDisplay, type ToolResultEvent, } from '@moonshot-ai/kimi-code-sdk'; @@ -88,14 +89,23 @@ export function acpBlocksToPromptParts( */ export async function compressPromptImageParts( parts: readonly PromptPart[], - options: { readonly originalsDir?: string | undefined } = {}, + options: { + readonly originalsDir?: string | undefined; + /** Report an `image_compress` event per prompt image (source `acp_prompt`). */ + readonly telemetry?: TelemetryClient | undefined; + } = {}, ): Promise { const out: PromptPart[] = []; for (const part of parts) { if (part.type === 'image_url') { const parsed = parseImageDataUrl(part.imageUrl.url); if (parsed !== null) { - const result = await compressBase64ForModel(parsed.base64, parsed.mimeType); + const result = await compressBase64ForModel(parsed.base64, parsed.mimeType, { + telemetry: + options.telemetry === undefined + ? undefined + : { client: options.telemetry, source: 'acp_prompt' }, + }); if (result.changed) { const originalPath = await persistOriginalImage( Buffer.from(parsed.base64, 'base64'), diff --git a/packages/acp-adapter/src/session.ts b/packages/acp-adapter/src/session.ts index 377fbcc99..64a3bbf14 100644 --- a/packages/acp-adapter/src/session.ts +++ b/packages/acp-adapter/src/session.ts @@ -736,9 +736,17 @@ export class AcpSession { let parts: readonly PromptPart[]; try { const sessionDir = this.session.summary?.sessionDir; + const track = this.track; parts = await compressPromptImageParts(acpBlocksToPromptParts(blocks), { originalsDir: sessionDir === undefined ? undefined : sessionMediaOriginalsDir(sessionDir), + telemetry: + track === undefined + ? undefined + : { + track: (event, properties) => + track(event, properties === undefined ? undefined : { ...properties }), + }, }); } finally { this.pendingPromptAborts.delete(pending); diff --git a/packages/acp-adapter/test/cancel.test.ts b/packages/acp-adapter/test/cancel.test.ts index e741db135..bbe243ada 100644 --- a/packages/acp-adapter/test/cancel.test.ts +++ b/packages/acp-adapter/test/cancel.test.ts @@ -163,10 +163,11 @@ describe('AcpServer cancel', () => { const { sessionId } = await client.newSession({ cwd: '/tmp/x', mcpServers: [] }); - // A solid 2600×2600 image is small in bytes but slow enough to compress - // that the cancel below reliably lands mid-compression, before any turn. + // A solid 3600×1800 image is small in bytes but slow enough to compress + // that the cancel below reliably lands mid-compression, before any turn + // — while staying safely inside the 5s test timeout on slow CI runners. const data = Buffer.from( - await new Jimp({ width: 2600, height: 2600, color: 0x3366ccff }).getBuffer('image/png'), + await new Jimp({ width: 3600, height: 1800, color: 0x3366ccff }).getBuffer('image/png'), ).toString('base64'); const promptP = client.prompt({ @@ -203,7 +204,7 @@ describe('AcpServer cancel', () => { const { sessionId } = await client.newSession({ cwd: '/tmp/x', mcpServers: [] }); const data = Buffer.from( - await new Jimp({ width: 2600, height: 2600, color: 0x3366ccff }).getBuffer('image/png'), + await new Jimp({ width: 3600, height: 1800, color: 0x3366ccff }).getBuffer('image/png'), ).toString('base64'); const imageBlock = { type: 'image' as const, data, mimeType: 'image/png' }; diff --git a/packages/acp-adapter/test/convert.test.ts b/packages/acp-adapter/test/convert.test.ts index 79c8b96ac..cb4117f48 100644 --- a/packages/acp-adapter/test/convert.test.ts +++ b/packages/acp-adapter/test/convert.test.ts @@ -338,7 +338,7 @@ describe('compressPromptImageParts', () => { it('downsamples an oversized inline image part and announces the compression', async () => { const originalsDir = await mkdtemp(join(tmpdir(), 'acp-originals-')); - const originalBase64 = await pngBase64(2600, 2600); + const originalBase64 = await pngBase64(3600, 1800); const parts = acpBlocksToPromptParts([imageBlock(originalBase64, 'image/png')]); const compressed = await compressPromptImageParts(parts, { originalsDir }); @@ -348,14 +348,14 @@ describe('compressPromptImageParts', () => { const caption = compressed[0]; if (caption?.type !== 'text') throw new Error('expected a caption text part'); expect(caption.text).toContain('Image compressed'); - expect(caption.text).toContain('2600x2600'); + expect(caption.text).toContain('3600x1800'); const part = compressed[1]; if (part?.type !== 'image_url') throw new Error('expected an image_url part'); const match = /^data:(image\/[a-z]+);base64,(.+)$/.exec(part.imageUrl.url); expect(match).not.toBeNull(); const decoded = await Jimp.fromBuffer(Buffer.from(match![2]!, 'base64')); - expect(Math.max(decoded.width, decoded.height)).toBeLessThanOrEqual(2000); + expect(Math.max(decoded.width, decoded.height)).toBeLessThanOrEqual(3000); // The caption points at a persisted copy of the ORIGINAL bytes, placed in // the provided (session-scoped) originals dir. @@ -367,6 +367,24 @@ describe('compressPromptImageParts', () => { await rm(originalsDir, { recursive: true, force: true }); }); + it('emits image_compress telemetry tagged acp_prompt', async () => { + const originalsDir = await mkdtemp(join(tmpdir(), 'acp-originals-')); + const events: { event: string; props: Record }[] = []; + const parts = acpBlocksToPromptParts([ + imageBlock(await pngBase64(3600, 1800), 'image/png'), + ]); + await compressPromptImageParts(parts, { + originalsDir, + telemetry: { track: (event, props) => events.push({ event, props: { ...props } }) }, + }); + + expect(events).toHaveLength(1); + expect(events[0]!.event).toBe('image_compress'); + expect(events[0]!.props['source']).toBe('acp_prompt'); + expect(events[0]!.props['outcome']).toBe('compressed'); + await rm(originalsDir, { recursive: true, force: true }); + }); + it('passes a within-budget image and text through unchanged', async () => { const parts = acpBlocksToPromptParts([ imageBlock(await pngBase64(32, 32), 'image/png'), diff --git a/packages/agent-core/src/agent/tool/index.ts b/packages/agent-core/src/agent/tool/index.ts index 5ddd0dc3c..f46dcd546 100644 --- a/packages/agent-core/src/agent/tool/index.ts +++ b/packages/agent-core/src/agent/tool/index.ts @@ -316,6 +316,7 @@ export class ToolManager { ); return mcpResultToExecutableOutput(result, qualified, { originalsDir: this.agent.mediaOriginalsDir, + telemetry: this.agent.telemetry, }); }, }; @@ -700,7 +701,13 @@ export class ToolManager { allowBackground, }), (modelCapabilities.image_in || modelCapabilities.video_in) && - new b.ReadMediaFileTool(kaos, workspace, modelCapabilities, videoUploader), + new b.ReadMediaFileTool( + kaos, + workspace, + modelCapabilities, + videoUploader, + this.agent.telemetry, + ), new b.EnterPlanModeTool(this.agent), new b.ExitPlanModeTool(this.agent), // Registered unconditionally: the tool-select flag can flip at runtime diff --git a/packages/agent-core/src/index.ts b/packages/agent-core/src/index.ts index 01e159c0e..b347ac5d6 100644 --- a/packages/agent-core/src/index.ts +++ b/packages/agent-core/src/index.ts @@ -76,6 +76,7 @@ export type { CropImageOptions, CropImageOutcome, ImageCompressionCaptionInput, + ImageCompressionTelemetry, ImageCropRegion, ImageVariantDescription, } from './tools/support/image-compress'; diff --git a/packages/agent-core/src/mcp/output.ts b/packages/agent-core/src/mcp/output.ts index 6c5b5a777..0f77683f9 100644 --- a/packages/agent-core/src/mcp/output.ts +++ b/packages/agent-core/src/mcp/output.ts @@ -29,6 +29,8 @@ import type { ContentPart } from '@moonshot-ai/kosong'; +import type { TelemetryClient } from '#/telemetry'; + import { compressImageContentParts } from '../tools/support/image-compress'; import { persistOriginalImage } from '../tools/support/image-originals'; import type { MCPContentBlock, MCPToolResult } from './types'; @@ -40,6 +42,8 @@ export interface McpOutputOptions { * Falls back to the shared temp-dir cache when absent. */ readonly originalsDir?: string | undefined; + /** Report an `image_compress` event per compressed tool-result image. */ + readonly telemetry?: TelemetryClient | undefined; } // MCP servers can produce arbitrarily large outputs; cap what we feed back to @@ -184,6 +188,10 @@ export async function mcpResultToExecutableOutput( // DATA (never inserted into the parts), so tool output that merely quotes // a caption can never be mistaken for a generated one. const compressed = await compressImageContentParts(budgeted.parts, { + telemetry: + options.telemetry === undefined + ? undefined + : { client: options.telemetry, source: 'mcp_tool_result' }, annotate: { persistOriginal: (bytes, mimeType) => persistOriginalImage( diff --git a/packages/agent-core/src/services/coreProcess/coreProcess.ts b/packages/agent-core/src/services/coreProcess/coreProcess.ts index 2a8b5ff03..73b8b27c7 100644 --- a/packages/agent-core/src/services/coreProcess/coreProcess.ts +++ b/packages/agent-core/src/services/coreProcess/coreProcess.ts @@ -31,6 +31,7 @@ import { createDecorator } from '../../di'; import type { CoreRPC, KimiCoreOptions } from '../../rpc'; +import type { TelemetryClient } from '../../telemetry'; import { type KimiHostIdentity } from '@moonshot-ai/kimi-code-oauth'; export interface CoreProcessServiceOptions extends KimiCoreOptions { @@ -60,6 +61,13 @@ export interface ICoreProcessService { readonly kimiRequestHeaders?: Record | undefined; + /** + * The telemetry client the host wired into `KimiCore` (noop when the host + * supplied none), so daemon-side code — e.g. prompt-ingestion image + * compression — reports through the same sink as core events. + */ + readonly telemetry?: TelemetryClient | undefined; + /** * Resolves once `KimiCore` is fully constructed and the SDK side of the * in-process RPC has been bound. Repeated calls return the cached promise. diff --git a/packages/agent-core/src/services/coreProcess/coreProcessService.ts b/packages/agent-core/src/services/coreProcess/coreProcessService.ts index 58bb34452..c66422bff 100644 --- a/packages/agent-core/src/services/coreProcess/coreProcessService.ts +++ b/packages/agent-core/src/services/coreProcess/coreProcessService.ts @@ -6,6 +6,7 @@ import { createRPC, KimiCore } from '../../rpc'; import { Disposable, registerSingleton, SyncDescriptor } from '../../di'; import type { CoreAPI, CoreRPC, SDKAPI } from '../../rpc'; import type { OAuthTokenProviderResolver } from '../../session/provider-manager'; +import { noopTelemetryClient, type TelemetryClient } from '../../telemetry'; import { createKimiDefaultHeaders, type KimiHostIdentity, @@ -33,6 +34,8 @@ export class CoreProcessService extends Disposable implements ICoreProcessServic public readonly kimiRequestHeaders: Record | undefined; + public readonly telemetry: TelemetryClient; + /** * The in-process `KimiCore` instance. Kept private so daemon-side code can't * grab it and bypass the peer-service indirection. @@ -96,6 +99,7 @@ export class CoreProcessService extends Disposable implements ICoreProcessServic this.kimiRequestHeaders = options.kimiRequestHeaders ?? CoreProcessService._defaultKimiRequestHeaders(env.homeDir, options.identity); + this.telemetry = options.telemetry ?? noopTelemetryClient; // `appVersion` flows into Session records (`app_version`) and tool // call ctx. Prefer explicit > identity.version so callers can pin diff --git a/packages/agent-core/src/tools/builtin/file/read-media.ts b/packages/agent-core/src/tools/builtin/file/read-media.ts index f654829a8..5676cbc40 100644 --- a/packages/agent-core/src/tools/builtin/file/read-media.ts +++ b/packages/agent-core/src/tools/builtin/file/read-media.ts @@ -36,6 +36,7 @@ import { z } from 'zod'; import type { BuiltinTool } from '../../../agent/tool'; import { ToolAccesses } from '../../../loop/tool-access'; import type { ExecutableToolResult, ToolExecution } from '../../../loop/types'; +import type { TelemetryClient } from '../../../telemetry'; import { renderPrompt } from '../../../utils/render-prompt'; import { resolvePathAccessPath } from '../../policies/path-access'; import { MEDIA_SNIFF_BYTES, detectFileType, sniffImageDimensions } from '../../support/file-type'; @@ -44,6 +45,7 @@ import { compressImageForModel, cropImageForModel, formatByteSize, + type ImageCompressionTelemetry, type ImageCropRegion, } from '../../support/image-compress'; import { toInputJsonSchema } from '../../support/input-schema'; @@ -215,11 +217,13 @@ export class ReadMediaFileTool implements BuiltinTool { readonly name = 'ReadMediaFile' as const; readonly description: string; readonly parameters: Record = toInputJsonSchema(ReadMediaFileInputSchema); + private readonly compressTelemetry: ImageCompressionTelemetry | undefined; constructor( private readonly kaos: Kaos, private readonly workspace: WorkspaceConfig, private readonly capabilities: ModelCapability, private readonly videoUploader?: VideoUploader | undefined, + telemetry?: TelemetryClient, ) { if (!capabilities.image_in && !capabilities.video_in) { const skip = new Error('ReadMediaFile requires image_in or video_in capability'); @@ -227,6 +231,8 @@ export class ReadMediaFileTool implements BuiltinTool { throw skip; } this.description = buildDescription(capabilities); + this.compressTelemetry = + telemetry === undefined ? undefined : { client: telemetry, source: 'read_media' }; } resolveExecution(args: ReadMediaFileInput): ToolExecution { @@ -330,6 +336,7 @@ export class ReadMediaFileTool implements BuiltinTool { // full fidelity, so a prior downsampled view can be zoomed into. const outcome = await cropImageForModel(data, fileType.mimeType, args.region, { skipResize: args.full_resolution === true, + telemetry: this.compressTelemetry, }); if (!outcome.ok) { return { isError: true, output: `Cannot read region from "${args.path}": ${outcome.error}` }; @@ -348,8 +355,10 @@ export class ReadMediaFileTool implements BuiltinTool { region: outcome.region, resized: outcome.resized, }; - // The decode knows the true size even when header sniffing does not. - dimensions ??= { width: outcome.originalWidth, height: outcome.originalHeight }; + // The decode is authoritative: it covers formats and nonconforming + // EXIF the header sniff cannot read, and region coordinates live + // in the decoded space, so the note must report it. + dimensions = { width: outcome.originalWidth, height: outcome.originalHeight }; } else if (args.full_resolution === true) { // Native resolution on request — but the provider's per-image byte // ceiling is a hard limit, so refuse explicitly rather than degrade. @@ -382,7 +391,9 @@ export class ReadMediaFileTool implements BuiltinTool { // tokens nor trips the provider's per-image byte ceiling. Best effort: // on any failure compressImageForModel returns the original bytes, so // the read still succeeds with the uncompressed image. - const compressed = await compressImageForModel(data, fileType.mimeType); + const compressed = await compressImageForModel(data, fileType.mimeType, { + telemetry: this.compressTelemetry, + }); const base64 = Buffer.from(compressed.data).toString('base64'); mediaPart = { type: 'image_url', @@ -395,6 +406,11 @@ export class ReadMediaFileTool implements BuiltinTool { byteLength: compressed.finalByteLength, mimeType: compressed.mimeType, }; + if (compressed.changed) { + // Same as the crop path: once a decode happened, its dimensions + // are authoritative over the header sniff. + dimensions = { width: compressed.originalWidth, height: compressed.originalHeight }; + } } } else if (this.videoUploader !== undefined) { mediaPart = await this.videoUploader({ diff --git a/packages/agent-core/src/tools/support/file-type.ts b/packages/agent-core/src/tools/support/file-type.ts index 49b6d1980..ea9058774 100644 --- a/packages/agent-core/src/tools/support/file-type.ts +++ b/packages/agent-core/src/tools/support/file-type.ts @@ -237,6 +237,11 @@ export function sniffMediaFromMagic(data: Buffer | Uint8Array): FileType | null export interface ImageDimensions { readonly width: number; readonly height: number; + /** + * Present (true) when a JPEG EXIF orientation of 5-8 swapped the reported + * width/height into display space. + */ + readonly transposed?: boolean; } /** @@ -247,6 +252,11 @@ export interface ImageDimensions { * after the `WEBP` tag, or the first JPEG SOFn segment). Returns `null` * for formats whose dimensions are not locatable from that region, or * when the supplied buffer is too short to cover it. + * + * JPEG dimensions are reported in DISPLAY space: an EXIF Orientation of + * 5-8 transposes the image at decode time, so the SOF width/height are + * swapped to match what decoders (and this codebase's crop regions and + * compression captions) actually operate in. */ export function sniffImageDimensions(data: Buffer | Uint8Array): ImageDimensions | null { const buf = toBuffer(data); @@ -297,8 +307,10 @@ export function sniffImageDimensions(data: Buffer | Uint8Array): ImageDimensions } // JPEG — scan segment markers for a Start-Of-Frame (SOFn) marker, - // whose payload carries height/width as big-endian uint16. + // whose payload carries height/width as big-endian uint16. An EXIF + // APP1 segment encountered on the way supplies the orientation. if (startsWith(buf, [0xff, 0xd8])) { + let orientation: number | null = null; let offset = 2; while (offset + 9 < buf.length) { if (buf[offset] !== 0xff) { @@ -314,10 +326,11 @@ export function sniffImageDimensions(data: Buffer | Uint8Array): ImageDimensions marker !== 0xc8 && marker !== 0xcc ) { - return { - height: buf.readUInt16BE(offset + 5), - width: buf.readUInt16BE(offset + 7), - }; + const height = buf.readUInt16BE(offset + 5); + const width = buf.readUInt16BE(offset + 7); + return orientation !== null && orientation >= 5 + ? { width: height, height: width, transposed: true } + : { width, height }; } // Standalone markers (RSTn, SOI, EOI) carry no length field. if (marker === 0xd8 || marker === 0xd9 || (marker >= 0xd0 && marker <= 0xd7)) { @@ -326,6 +339,9 @@ export function sniffImageDimensions(data: Buffer | Uint8Array): ImageDimensions } const segmentLength = buf.readUInt16BE(offset + 2); if (segmentLength < 2) break; + if (marker === 0xe1 && orientation === null) { + orientation = readExifOrientation(buf, offset + 4, offset + 2 + segmentLength); + } offset += 2 + segmentLength; } } @@ -333,6 +349,42 @@ export function sniffImageDimensions(data: Buffer | Uint8Array): ImageDimensions return null; } +/** + * Read the Orientation tag (0x0112) out of a JPEG APP1 payload spanning + * [`start`, `end`). Returns 1-8, or `null` when the payload is not EXIF, + * is truncated, or carries no valid orientation. Only IFD0 is examined — + * that is where the tag lives; nothing here follows nested IFDs. + */ +function readExifOrientation(buf: Buffer, start: number, end: number): number | null { + const boundedEnd = Math.min(end, buf.length); + // 'Exif\0\0' preamble, then the TIFF header. + if (start + 6 > boundedEnd || buf.toString('latin1', start, start + 6) !== 'Exif\0\0') { + return null; + } + const tiff = start + 6; + if (tiff + 8 > boundedEnd) return null; + const byteOrder = buf.toString('latin1', tiff, tiff + 2); + const le = byteOrder === 'II'; + if (!le && byteOrder !== 'MM') return null; + const u16 = (offset: number): number => (le ? buf.readUInt16LE(offset) : buf.readUInt16BE(offset)); + const u32 = (offset: number): number => (le ? buf.readUInt32LE(offset) : buf.readUInt32BE(offset)); + if (u16(tiff + 2) !== 42) return null; + const ifd = tiff + u32(tiff + 4); + if (ifd + 2 > boundedEnd) return null; + const entryCount = u16(ifd); + for (let i = 0; i < entryCount; i += 1) { + const entry = ifd + 2 + i * 12; + if (entry + 12 > boundedEnd) return null; + if (u16(entry) === 0x0112) { + // Type SHORT: the value sits in the first two bytes of the 4-byte + // value field, in the TIFF byte order. + const value = u16(entry + 8); + return value >= 1 && value <= 8 ? value : null; + } + } + return null; +} + function getSuffix(path: string): string { const idx = path.lastIndexOf('.'); if (idx === -1) return ''; diff --git a/packages/agent-core/src/tools/support/image-compress.ts b/packages/agent-core/src/tools/support/image-compress.ts index 34f62ecb5..8804eb1c0 100644 --- a/packages/agent-core/src/tools/support/image-compress.ts +++ b/packages/agent-core/src/tools/support/image-compress.ts @@ -28,10 +28,12 @@ import type { ContentPart } from '@moonshot-ai/kosong'; +import type { TelemetryClient } from '#/telemetry'; + import { sniffImageDimensions } from './file-type'; /** Longest-edge ceiling (px). Larger images are scaled down to fit. */ -export const MAX_IMAGE_EDGE_PX = 2000; +export const MAX_IMAGE_EDGE_PX = 3000; /** * Raw-byte budget for a single image. base64 inflates bytes by ~4/3, so a @@ -43,8 +45,13 @@ export const IMAGE_BYTE_BUDGET = 3.75 * 1024 * 1024; /** Progressively lower JPEG quality until the payload fits the byte budget. */ const JPEG_QUALITY_STEPS = [80, 60, 40, 20] as const; -/** Last-ditch longest edge when the budget cannot be met at MAX_IMAGE_EDGE_PX. */ -const FALLBACK_EDGE_PX = 1000; +/** + * Longest-edge step-downs tried when the budget cannot be met at the fitted + * size. The 2000px step preserves the behavior of the previous 2000px cap: + * an image whose 2000px encode fits the budget keeps that resolution + * instead of dropping straight to the 1000px last resort. + */ +const FALLBACK_EDGES_PX = [2000, 1000] as const; /** * Pixel-count ceiling above which we skip compression entirely. A tiny-byte, @@ -79,8 +86,36 @@ export interface CompressImageOptions { readonly byteBudget?: number; /** Override the raw-byte ceiling above which compression is skipped. */ readonly maxDecodeBytes?: number; + /** + * Report an `image_compress` event per compression call (and an + * `image_crop` event per {@link cropImageForModel} call). Absent → silent. + */ + readonly telemetry?: ImageCompressionTelemetry; } +/** Wiring for the optional compression telemetry events. */ +export interface ImageCompressionTelemetry { + readonly client: TelemetryClient; + /** Where the image entered the pipeline, e.g. 'read_media', 'tui_paste'. */ + readonly source: string; +} + +/** + * How a compression call ended, as reported in the `image_compress` event. + * Every `passthrough_*` variant returns the input bytes unchanged: `fast` is + * the within-budgets hot path, `guard` a decode-safety refusal (pixel bomb or + * byte cap), `unsupported` a format the codec cannot re-encode (or empty + * input), `unhelpful` a re-encode that saved neither bytes nor pixels, and + * `error` a decode/encode failure. + */ +type CompressOutcome = + | 'compressed' + | 'passthrough_fast' + | 'passthrough_guard' + | 'passthrough_unsupported' + | 'passthrough_unhelpful' + | 'passthrough_error'; + export interface CompressImageResult { /** Bytes to send: the re-encoded image, or the original when unchanged. */ readonly data: Uint8Array; @@ -90,9 +125,13 @@ export interface CompressImageResult { readonly width: number; /** Pixel height of `data`; falls back to the input size when unknown. */ readonly height: number; - /** Pixel width of the input image; 0 when it cannot be sniffed. */ + /** + * Pixel width of the input image, in display space (EXIF orientation + * applied): the decoded width when re-encoded, the header sniff on + * passthrough (0 when it cannot be determined). + */ readonly originalWidth: number; - /** Pixel height of the input image; 0 when it cannot be sniffed. */ + /** Pixel height of the input image; see {@link originalWidth}. */ readonly originalHeight: number; /** True only when `data` differs from the input bytes. */ readonly changed: boolean; @@ -112,6 +151,7 @@ export async function compressImageForModel( mimeType: string, options: CompressImageOptions = {}, ): Promise { + const startedAt = Date.now(); const maxEdge = options.maxEdge ?? MAX_IMAGE_EDGE_PX; const byteBudget = options.byteBudget ?? IMAGE_BYTE_BUDGET; const maxDecodeBytes = options.maxDecodeBytes ?? MAX_DECODE_BYTES; @@ -129,28 +169,50 @@ export async function compressImageForModel( originalByteLength: bytes.length, finalByteLength: bytes.length, }); + const finish = (outcome: CompressOutcome, result: CompressImageResult): CompressImageResult => { + reportCompressEvent(options.telemetry, { + outcome, + startedAt, + inputMime: normalizedMime, + exifTransposed: dims?.transposed === true, + result, + }); + return result; + }; - if (bytes.length === 0) return passthrough(); + if (bytes.length === 0) return finish('passthrough_unsupported', passthrough()); // Only re-encode formats the codec handles; everything else passes through. - if (!RECODABLE_MIME.has(normalizedMime)) return passthrough(); + if (!RECODABLE_MIME.has(normalizedMime)) return finish('passthrough_unsupported', passthrough()); // Fast path: already within both budgets — no codec load, no allocation. const longestEdge = dims ? Math.max(dims.width, dims.height) : 0; const withinBytes = bytes.length <= byteBudget; const withinEdge = longestEdge > 0 && longestEdge <= maxEdge; - if (withinBytes && (withinEdge || longestEdge === 0)) return passthrough(); + if (withinBytes && (withinEdge || longestEdge === 0)) { + return finish('passthrough_fast', passthrough()); + } // Decompression-bomb guard: refuse to decode absurd pixel counts. The sniff // above gave us the dimensions without decoding, so this costs nothing. - if (dims && dims.width * dims.height > MAX_DECODE_PIXELS) return passthrough(); + if (dims && dims.width * dims.height > MAX_DECODE_PIXELS) { + return finish('passthrough_guard', passthrough()); + } // Refuse to decode very large byte payloads (e.g. a huge or invalid image // from an MCP tool) that would be loaded just to be dropped downstream. - if (bytes.length > maxDecodeBytes) return passthrough(); + if (bytes.length > maxDecodeBytes) return finish('passthrough_guard', passthrough()); try { const { Jimp } = await import('jimp'); const image = await Jimp.fromBuffer(Buffer.from(bytes)); const sourceIsPng = normalizedMime === 'image/png'; + // The decoded bitmap is authoritative for the original size: jimp + // applies EXIF orientation while decoding, and this is the coordinate + // space the encoded result and any later crop region (see + // cropImageForModel, which decodes the same way) actually live in. The + // header sniff also reports display space, but can miss formats or + // nonconforming EXIF that the decoder still handles. + const decodedWidth = image.width; + const decodedHeight = image.height; // Scale so the longest edge fits maxEdge (never enlarges). fitWithinEdge(image, maxEdge); @@ -158,33 +220,33 @@ export async function compressImageForModel( const encoded = await encodeWithinBudget(image, { sourceIsPng, byteBudget, - fallbackEdge: FALLBACK_EDGE_PX, + fallbackEdges: FALLBACK_EDGES_PX, }); // Keep the result when it actually helps: fewer bytes, or fewer pixels // (a smaller image costs fewer vision tokens even if the byte count is // flat, as with near-solid graphics). Otherwise the re-encode bought us // nothing — send the original. - const originalPixels = (dims?.width ?? 0) * (dims?.height ?? 0); + const originalPixels = decodedWidth * decodedHeight; const finalPixels = encoded.width * encoded.height; const shrankBytes = encoded.data.length < bytes.length; - const shrankPixels = originalPixels > 0 && finalPixels < originalPixels; - if (!shrankBytes && !shrankPixels) return passthrough(); + const shrankPixels = finalPixels < originalPixels; + if (!shrankBytes && !shrankPixels) return finish('passthrough_unhelpful', passthrough()); - return { + return finish('compressed', { data: encoded.data, mimeType: encoded.mimeType, width: encoded.width, height: encoded.height, - originalWidth: dims?.width ?? 0, - originalHeight: dims?.height ?? 0, + originalWidth: decodedWidth, + originalHeight: decodedHeight, changed: true, originalByteLength: bytes.length, finalByteLength: encoded.data.length, - }; + }); } catch { // Decode/encode failure — keep the original bytes. - return passthrough(); + return finish('passthrough_error', passthrough()); } } @@ -195,9 +257,13 @@ export interface CompressBase64Result { readonly width: number; /** Pixel height of the (possibly re-encoded) payload; 0 when unknown. */ readonly height: number; - /** Pixel width of the input image; 0 when it cannot be sniffed. */ + /** + * Pixel width of the input image, in display space (EXIF orientation + * applied): the decoded width when re-encoded, the header sniff on + * passthrough (0 when it cannot be determined). + */ readonly originalWidth: number; - /** Pixel height of the input image; 0 when it cannot be sniffed. */ + /** Pixel height of the input image; see {@link originalWidth}. */ readonly originalHeight: number; readonly changed: boolean; readonly originalByteLength: number; @@ -217,10 +283,11 @@ export async function compressBase64ForModel( // Skip very large payloads before allocating: base64 decodes to ~3/4 its // length, so a payload whose decoded size would exceed the cap is passed // through without the Buffer.from allocation (and without touching Jimp). + const startedAt = Date.now(); const maxDecodeBytes = options.maxDecodeBytes ?? MAX_DECODE_BYTES; const approxBytes = Math.floor((base64.length * 3) / 4); if (approxBytes > maxDecodeBytes) { - return { + const result: CompressBase64Result = { base64, mimeType, width: 0, @@ -231,12 +298,20 @@ export async function compressBase64ForModel( originalByteLength: approxBytes, finalByteLength: approxBytes, }; + reportCompressEvent(options.telemetry, { + outcome: 'passthrough_guard', + startedAt, + inputMime: normalizeMime(mimeType), + exifTransposed: false, + result, + }); + return result; } let bytes: Buffer; try { bytes = Buffer.from(base64, 'base64'); } catch { - return { + const result: CompressBase64Result = { base64, mimeType, width: 0, @@ -247,7 +322,16 @@ export async function compressBase64ForModel( originalByteLength: 0, finalByteLength: 0, }; + reportCompressEvent(options.telemetry, { + outcome: 'passthrough_error', + startedAt, + inputMime: normalizeMime(mimeType), + exifTransposed: false, + result, + }); + return result; } + // The event for this call is emitted inside compressImageForModel. const result = await compressImageForModel(bytes, mimeType, options); if (!result.changed) { return { @@ -369,7 +453,10 @@ export interface CompressAnnotateOptions { // ── crop ───────────────────────────────────────────────────────────── -/** Crop rectangle in ORIGINAL-image pixel coordinates. */ +/** + * Crop rectangle in ORIGINAL-image pixel coordinates — the decoded, + * EXIF-rotated space that compression results report as the original size. + */ export interface ImageCropRegion { readonly x: number; readonly y: number; @@ -430,41 +517,50 @@ export async function cropImageForModel( region: ImageCropRegion, options: CropImageOptions = {}, ): Promise { + const startedAt = Date.now(); const maxEdge = options.maxEdge ?? MAX_IMAGE_EDGE_PX; const byteBudget = options.byteBudget ?? IMAGE_BYTE_BUDGET; const maxDecodeBytes = options.maxDecodeBytes ?? MAX_DECODE_BYTES; const normalizedMime = normalizeMime(mimeType); + const fail = (errorKind: CropErrorKind, error: string): CropImageFailure => { + reportCropEvent(options.telemetry, { startedAt, ok: false, errorKind }); + return { ok: false, error }; + }; + const succeed = (result: CropImageSuccess): CropImageSuccess => { + reportCropEvent(options.telemetry, { startedAt, ok: true, result }); + return result; + }; + if (bytes.length === 0) { - return { ok: false, error: 'The image is empty.' }; + return fail('empty', 'The image is empty.'); } if (!RECODABLE_MIME.has(normalizedMime)) { - return { - ok: false, - error: `Cropping is only supported for PNG and JPEG images; got ${mimeType}.`, - }; + return fail( + 'unsupported_format', + `Cropping is only supported for PNG and JPEG images; got ${mimeType}.`, + ); } // NaN slips past every = comparison in the bounds guard below, so gate // on finiteness explicitly rather than surfacing a codec-internal error. if ( ![region.x, region.y, region.width, region.height].every((value) => Number.isFinite(value)) ) { - return { - ok: false, - error: - `Region coordinates must be finite numbers; got x=${String(region.x)}, ` + + return fail( + 'region_invalid', + `Region coordinates must be finite numbers; got x=${String(region.x)}, ` + `y=${String(region.y)}, width=${String(region.width)}, height=${String(region.height)}.`, - }; + ); } const dims = sniffImageDimensions(bytes); if (dims && dims.width * dims.height > MAX_DECODE_PIXELS) { - return { - ok: false, - error: `The image (${String(dims.width)}x${String(dims.height)} pixels) is too large to decode for cropping.`, - }; + return fail( + 'too_large', + `The image (${String(dims.width)}x${String(dims.height)} pixels) is too large to decode for cropping.`, + ); } if (bytes.length > maxDecodeBytes) { - return { ok: false, error: 'The image is too large to decode for cropping.' }; + return fail('too_large', 'The image is too large to decode for cropping.'); } try { @@ -476,12 +572,11 @@ export async function cropImageForModel( const x = Math.floor(region.x); const y = Math.floor(region.y); if (x < 0 || y < 0 || x >= originalWidth || y >= originalHeight || region.width < 1 || region.height < 1) { - return { - ok: false, - error: - `Region (x=${String(region.x)}, y=${String(region.y)}, width=${String(region.width)}, ` + + return fail( + 'out_of_bounds', + `Region (x=${String(region.x)}, y=${String(region.y)}, width=${String(region.width)}, ` + `height=${String(region.height)}) lies outside the ${String(originalWidth)}x${String(originalHeight)} image.`, - }; + ); } const w = Math.min(Math.floor(region.width), originalWidth - x); const h = Math.min(Math.floor(region.height), originalHeight - y); @@ -497,16 +592,15 @@ export async function cropImageForModel( ? await image.getBuffer('image/png', { deflateLevel: 9 }) : await image.getBuffer('image/jpeg', { quality: 90 }); if (buffer.length > byteBudget) { - return { - ok: false, - error: - `The cropped region encodes to ${String(buffer.length)} bytes ` + + return fail( + 'budget', + `The cropped region encodes to ${String(buffer.length)} bytes ` + `(${formatByteSize(buffer.length)}), over the ${String(byteBudget)}-byte ` + `(${formatByteSize(byteBudget)}) per-image limit. ` + 'Choose a smaller region, or allow downscaling.', - }; + ); } - return { + return succeed({ ok: true, data: new Uint8Array(buffer), mimeType: sourceIsPng ? 'image/png' : 'image/jpeg', @@ -518,16 +612,16 @@ export async function cropImageForModel( resized: false, originalByteLength: bytes.length, finalByteLength: buffer.length, - }; + }); } fitWithinEdge(image, maxEdge); const encoded = await encodeWithinBudget(image, { sourceIsPng, byteBudget, - fallbackEdge: FALLBACK_EDGE_PX, + fallbackEdges: FALLBACK_EDGES_PX, }); - return { + return succeed({ ok: true, data: new Uint8Array(encoded.data), mimeType: encoded.mimeType, @@ -539,12 +633,12 @@ export async function cropImageForModel( resized: encoded.width !== w || encoded.height !== h, originalByteLength: bytes.length, finalByteLength: encoded.data.length, - }; + }); } catch (error) { - return { - ok: false, - error: `Failed to decode the image for cropping: ${error instanceof Error ? error.message : String(error)}`, - }; + return fail( + 'decode_failed', + `Failed to decode the image for cropping: ${error instanceof Error ? error.message : String(error)}`, + ); } } @@ -675,7 +769,7 @@ interface EncodedImage { interface EncodeOptions { readonly sourceIsPng: boolean; readonly byteBudget: number; - readonly fallbackEdge: number; + readonly fallbackEdges: readonly number[]; } /** @@ -684,14 +778,17 @@ interface EncodeOptions { * Strategy — prefer the source format so a downscaled screenshot stays lossless * PNG (preserving text and transparency), and only fall back to lossy JPEG when * PNG cannot meet the byte budget: - * - PNG source: PNG at the fitted size → a smaller PNG rescale → JPEG ladder. - * - JPEG source: JPEG quality ladder → a smaller JPEG rescale. + * - PNG source: PNG at the fitted size → smaller PNG rescales, stepping down + * the fallback edges → JPEG ladder. + * - JPEG source: the full quality ladder at the fitted size, then again at + * each fallback edge — a smaller rescale must not skip the high-quality + * rungs its extra pixels just paid for. * * Always returns the smallest buffer it produced, even if no attempt met the * budget — the caller still gates on whether it actually helped. */ async function encodeWithinBudget(image: JimpImage, opts: EncodeOptions): Promise { - const { sourceIsPng, byteBudget, fallbackEdge } = opts; + const { sourceIsPng, byteBudget, fallbackEdges } = opts; let smallest: EncodedImage | null = null; const consider = (data: Buffer, mimeType: string): EncodedImage => { @@ -708,8 +805,9 @@ async function encodeWithinBudget(image: JimpImage, opts: EncodeOptions): Promis if (png.length <= byteBudget) return consider(png, 'image/png'); consider(png, 'image/png'); - // Over budget: a smaller PNG before going lossy. - if (fitWithinEdge(image, fallbackEdge)) { + // Over budget: progressively smaller PNGs before going lossy. + for (const edge of fallbackEdges) { + if (!fitWithinEdge(image, edge)) continue; const smallerPng = await image.getBuffer('image/png', { deflateLevel: 9 }); if (smallerPng.length <= byteBudget) return consider(smallerPng, 'image/png'); consider(smallerPng, 'image/png'); @@ -724,15 +822,20 @@ async function encodeWithinBudget(image: JimpImage, opts: EncodeOptions): Promis return smallest!; } - // JPEG source: quality ladder, then a smaller rescale at the lowest quality. + // JPEG source: quality ladder at the fitted size, then the full ladder + // again at each fallback rescale. for (const quality of JPEG_QUALITY_STEPS) { const jpeg = await image.getBuffer('image/jpeg', { quality }); if (jpeg.length <= byteBudget) return consider(jpeg, 'image/jpeg'); consider(jpeg, 'image/jpeg'); } - if (fitWithinEdge(image, fallbackEdge)) { - const jpeg = await image.getBuffer('image/jpeg', { quality: JPEG_QUALITY_STEPS.at(-1) }); - consider(jpeg, 'image/jpeg'); + for (const edge of fallbackEdges) { + if (!fitWithinEdge(image, edge)) continue; + for (const quality of JPEG_QUALITY_STEPS) { + const jpeg = await image.getBuffer('image/jpeg', { quality }); + if (jpeg.length <= byteBudget) return consider(jpeg, 'image/jpeg'); + consider(jpeg, 'image/jpeg'); + } } return smallest!; @@ -741,6 +844,13 @@ async function encodeWithinBudget(image: JimpImage, opts: EncodeOptions): Promis /** * Scale `image` so its longest edge is at most `edge`, preserving aspect * ratio. No-op (returns false) when the image already fits. + * + * Deliberately passes no `mode`: without one, jimp's default resizer + * downscales with a full-coverage area average (every source pixel + * contributes to the output), which does not alias. The named + * ResizeStrategy modes (BILINEAR, BICUBIC, …) switch to point-sampled + * interpolation that skips source pixels beyond ~2x reduction and produces + * moiré on text and fine patterns — do not "upgrade" this call to one. */ function fitWithinEdge(image: JimpImage, edge: number): boolean { const longest = Math.max(image.width, image.height); @@ -757,3 +867,99 @@ function normalizeMime(mimeType: string): string { const lower = mimeType.trim().toLowerCase(); return lower === 'image/jpg' ? 'image/jpeg' : lower; } + +// ── telemetry ──────────────────────────────────────────────────────── + +/** Failure classification carried by the `image_crop` event. */ +type CropErrorKind = + | 'empty' + | 'unsupported_format' + | 'region_invalid' + | 'too_large' + | 'out_of_bounds' + | 'budget' + | 'decode_failed'; + +/** The subset of a compression result the `image_compress` event reads. */ +interface CompressEventResult { + readonly mimeType: string; + readonly width: number; + readonly height: number; + readonly originalWidth: number; + readonly originalHeight: number; + readonly originalByteLength: number; + readonly finalByteLength: number; +} + +/** + * Emit the `image_compress` event. Properties are all numeric/enum — never + * paths or content — and a throwing client is swallowed so telemetry can + * never affect the compression result. + */ +function reportCompressEvent( + telemetry: ImageCompressionTelemetry | undefined, + input: { + readonly outcome: CompressOutcome; + readonly startedAt: number; + readonly inputMime: string; + readonly exifTransposed: boolean; + readonly result: CompressEventResult; + }, +): void { + if (telemetry === undefined) return; + try { + telemetry.client.track('image_compress', { + source: telemetry.source, + outcome: input.outcome, + input_mime: input.inputMime, + output_mime: normalizeMime(input.result.mimeType), + original_bytes: input.result.originalByteLength, + final_bytes: input.result.finalByteLength, + original_width: input.result.originalWidth, + original_height: input.result.originalHeight, + final_width: input.result.width, + final_height: input.result.height, + exif_transposed: input.exifTransposed, + duration_ms: Date.now() - input.startedAt, + }); + } catch { + // Telemetry must never affect the compression result. + } +} + +/** + * Emit the `image_crop` event. Reports the region as a share of the original + * pixel area rather than raw coordinates. + */ +function reportCropEvent( + telemetry: ImageCompressionTelemetry | undefined, + input: { + readonly startedAt: number; + readonly ok: boolean; + readonly errorKind?: CropErrorKind; + readonly result?: CropImageSuccess; + }, +): void { + if (telemetry === undefined) return; + try { + const { result } = input; + const originalPixels = + result === undefined ? 0 : result.originalWidth * result.originalHeight; + telemetry.client.track('image_crop', { + source: telemetry.source, + ok: input.ok, + error_kind: input.errorKind, + resized: result?.resized, + original_width: result?.originalWidth, + original_height: result?.originalHeight, + region_area_ratio: + result === undefined || originalPixels === 0 + ? undefined + : (result.region.width * result.region.height) / originalPixels, + final_bytes: result?.finalByteLength, + duration_ms: Date.now() - input.startedAt, + }); + } catch { + // Telemetry must never affect the crop outcome. + } +} diff --git a/packages/agent-core/test/mcp/output.test.ts b/packages/agent-core/test/mcp/output.test.ts index 11edc738c..24d2f9fd3 100644 --- a/packages/agent-core/test/mcp/output.test.ts +++ b/packages/agent-core/test/mcp/output.test.ts @@ -8,6 +8,7 @@ import { describe, expect, test } from 'vitest'; import { convertMCPContentBlock, mcpResultToExecutableOutput } from '../../src/mcp/output'; import type { MCPContentBlock, MCPToolResult } from '../../src/mcp/types'; +import type { TelemetryClient } from '../../src/telemetry'; import { sniffImageDimensions } from '../../src/tools/support/file-type'; const MCP_OUTPUT_TRUNCATED_TEXT = @@ -210,6 +211,27 @@ describe('mcpResultToExecutableOutput', () => { return { content, isError }; } + test('emits image_compress telemetry tagged mcp_tool_result', async () => { + const events: { event: string; props: Record }[] = []; + const telemetry: TelemetryClient = { + track: (event, props) => events.push({ event, props: props ?? {} }), + }; + const big = Buffer.from( + await new Jimp({ width: 3600, height: 1800, color: 0x3366ccff }).getBuffer('image/png'), + ).toString('base64'); + + await mcpResultToExecutableOutput( + result([{ type: 'image', data: big, mimeType: 'image/png' }]), + 'mcp__s__shot', + { telemetry }, + ); + + expect(events).toHaveLength(1); + expect(events[0]!.event).toBe('image_compress'); + expect(events[0]!.props['source']).toBe('mcp_tool_result'); + expect(events[0]!.props['outcome']).toBe('compressed'); + }); + test('collapses a single text part into a plain string', async () => { const out = await mcpResultToExecutableOutput( result([{ type: 'text', text: 'hello' }]), @@ -337,7 +359,7 @@ describe('mcpResultToExecutableOutput', () => { test('downsamples an oversized real image instead of leaving it full-size', async () => { const big = Buffer.from( - await new Jimp({ width: 2600, height: 2600, color: 0x3366ccff }).getBuffer('image/png'), + await new Jimp({ width: 3600, height: 1800, color: 0x3366ccff }).getBuffer('image/png'), ).toString('base64'); const out = await mcpResultToExecutableOutput( @@ -353,7 +375,7 @@ describe('mcpResultToExecutableOutput', () => { ); expect(match).not.toBeNull(); const dims = sniffImageDimensions(Buffer.from(match![2]!, 'base64')); - expect(Math.max(dims!.width, dims!.height)).toBeLessThanOrEqual(2000); + expect(Math.max(dims!.width, dims!.height)).toBeLessThanOrEqual(3000); // The image was compressed and kept, not dropped to a notice. const joined = parts.map((p) => (p.type === 'text' ? p.text : '')).join(''); expect(joined).not.toContain('image_url dropped'); @@ -361,7 +383,7 @@ describe('mcpResultToExecutableOutput', () => { test('annotates a downsampled image with a caption note and a readable original', async () => { const bigBytes = Buffer.from( - await new Jimp({ width: 2600, height: 2600, color: 0x3366ccff }).getBuffer('image/png'), + await new Jimp({ width: 3600, height: 1800, color: 0x3366ccff }).getBuffer('image/png'), ); const out = await mcpResultToExecutableOutput( @@ -372,7 +394,7 @@ describe('mcpResultToExecutableOutput', () => { // The caption rides the `note` side channel (model-only), keeping its // `` wrapping; the output itself carries only the media. expect(out.note).toContain('Image compressed'); - expect(out.note).toContain('2600x2600'); + expect(out.note).toContain('3600x1800'); const parts = out.output as ContentPart[]; expect(parts.some((p) => p.type === 'text' && p.text.includes('Image compressed'))).toBe( false, @@ -407,7 +429,7 @@ describe('mcpResultToExecutableOutput', () => { test('persists originals into the provided session originals dir', async () => { const dir = await mkdtemp(join(tmpdir(), 'mcp-originals-')); const bigBytes = Buffer.from( - await new Jimp({ width: 2600, height: 2600, color: 0x3366ccff }).getBuffer('image/png'), + await new Jimp({ width: 3600, height: 1800, color: 0x3366ccff }).getBuffer('image/png'), ); const out = await mcpResultToExecutableOutput( @@ -432,7 +454,7 @@ describe('mcpResultToExecutableOutput', () => { // to prevent — and orphaning the persisted original. const dir = await mkdtemp(join(tmpdir(), 'mcp-originals-')); const big = Buffer.from( - await new Jimp({ width: 2600, height: 2600, color: 0x3366ccff }).getBuffer('image/png'), + await new Jimp({ width: 3600, height: 1800, color: 0x3366ccff }).getBuffer('image/png'), ).toString('base64'); const out = await mcpResultToExecutableOutput( @@ -491,7 +513,7 @@ describe('mcpResultToExecutableOutput', () => { 'sent 50x50 image/jpeg (100 KB). Fine detail may be lost. ' + 'The uncompressed original was not preserved.'; const big = Buffer.from( - await new Jimp({ width: 2600, height: 2600, color: 0x3366ccff }).getBuffer('image/png'), + await new Jimp({ width: 3600, height: 1800, color: 0x3366ccff }).getBuffer('image/png'), ).toString('base64'); const out = await mcpResultToExecutableOutput( @@ -502,9 +524,9 @@ describe('mcpResultToExecutableOutput', () => { 'mcp__s__t', ); - // The real caption (2600x2600) rides the note; the quoted one (100x100) + // The real caption (3600x1800) rides the note; the quoted one (100x100) // is tool output and stays where the tool put it. - expect(out.note).toContain('2600x2600'); + expect(out.note).toContain('3600x1800'); expect(out.note).not.toContain('100x100'); const parts = out.output as ContentPart[]; const joined = parts.map((p) => (p.type === 'text' ? p.text : '')).join(''); @@ -517,7 +539,7 @@ describe('mcpResultToExecutableOutput', () => { // into an unclosed fragment. const dir = await mkdtemp(join(tmpdir(), 'mcp-originals-')); const big = Buffer.from( - await new Jimp({ width: 2600, height: 2600, color: 0x3366ccff }).getBuffer('image/png'), + await new Jimp({ width: 3600, height: 1800, color: 0x3366ccff }).getBuffer('image/png'), ).toString('base64'); const out = await mcpResultToExecutableOutput( diff --git a/packages/agent-core/test/tools/file-type.test.ts b/packages/agent-core/test/tools/file-type.test.ts index c0587411a..eff175cc2 100644 --- a/packages/agent-core/test/tools/file-type.test.ts +++ b/packages/agent-core/test/tools/file-type.test.ts @@ -382,6 +382,48 @@ function buildJpeg(width: number, height: number): Buffer { return Buffer.concat([soi, app0, sof0]); } +/** + * A minimal EXIF APP1 segment: 'Exif\0\0' + TIFF header + IFD0 holding a + * single Orientation (0x0112) SHORT entry, in the requested byte order. + */ +function exifApp1(orientation: number, byteOrder: 'II' | 'MM'): Buffer { + const le = byteOrder === 'II'; + const tiff = Buffer.alloc(26); + tiff.write(byteOrder, 0, 'latin1'); + const u16 = (value: number, offset: number): void => { + if (le) tiff.writeUInt16LE(value, offset); + else tiff.writeUInt16BE(value, offset); + }; + const u32 = (value: number, offset: number): void => { + if (le) tiff.writeUInt32LE(value, offset); + else tiff.writeUInt32BE(value, offset); + }; + u16(42, 2); + u32(8, 4); // offset of IFD0 + u16(1, 8); // one directory entry + u16(0x0112, 10); // tag: Orientation + u16(3, 12); // type: SHORT + u32(1, 14); // count + u16(orientation, 18); // value, left-aligned in the 4-byte field + u32(0, 22); // no next IFD + const body = Buffer.concat([Buffer.from('Exif\0\0', 'latin1'), tiff]); + const header = Buffer.alloc(4); + header.writeUInt16BE(0xff_e1, 0); + header.writeUInt16BE(body.length + 2, 2); + return Buffer.concat([header, body]); +} + +/** A JPEG whose EXIF APP1 sits between SOI and the remaining segments. */ +function buildJpegWithOrientation( + width: number, + height: number, + orientation: number, + byteOrder: 'II' | 'MM' = 'II', +): Buffer { + const jpeg = buildJpeg(width, height); + return Buffer.concat([jpeg.subarray(0, 2), exifApp1(orientation, byteOrder), jpeg.subarray(2)]); +} + describe('sniffImageDimensions', () => { const cases: ReadonlyArray<{ name: string; @@ -443,6 +485,50 @@ describe('sniffImageDimensions', () => { expect(sniffImageDimensions(data)).toEqual({ width: 100, height: 700 }); }); + describe('JPEG EXIF orientation (dimensions are display-space)', () => { + it.each([5, 6, 7, 8])('swaps width/height for transposing orientation %i', (orientation) => { + // Orientations 5-8 rotate/transpose at decode time: a 120x80 sensor + // frame displays as 80x120. The sniff must report the display space — + // the space decoded images, crop regions, and captions live in. + const data = buildJpegWithOrientation(120, 80, orientation); + expect(sniffImageDimensions(data)).toEqual({ width: 80, height: 120, transposed: true }); + }); + + it.each([1, 2, 3, 4])('keeps width/height for non-transposing orientation %i', (orientation) => { + const data = buildJpegWithOrientation(120, 80, orientation); + expect(sniffImageDimensions(data)).toEqual({ width: 120, height: 80 }); + }); + + it('honors big-endian (MM) TIFF byte order', () => { + const data = buildJpegWithOrientation(120, 80, 6, 'MM'); + expect(sniffImageDimensions(data)).toEqual({ width: 80, height: 120, transposed: true }); + }); + + it('ignores out-of-range orientation values', () => { + expect(sniffImageDimensions(buildJpegWithOrientation(120, 80, 0))).toEqual({ + width: 120, + height: 80, + }); + expect(sniffImageDimensions(buildJpegWithOrientation(120, 80, 9))).toEqual({ + width: 120, + height: 80, + }); + }); + + it('survives a truncated APP1 payload without throwing', () => { + // Declared APP1 length points past the actual TIFF bytes. Whatever + // the sniff returns (unswapped dims or null), it must not throw. + const jpeg = buildJpeg(120, 80); + const app1 = exifApp1(6, 'II'); + const truncated = Buffer.concat([ + jpeg.subarray(0, 2), + app1.subarray(0, 10), + jpeg.subarray(2), + ]); + expect(() => sniffImageDimensions(truncated)).not.toThrow(); + }); + }); + describe('truncated / malformed input returns null without throwing', () => { const malformed: ReadonlyArray<{ name: string; data: Buffer }> = [ { diff --git a/packages/agent-core/test/tools/image-compress.test.ts b/packages/agent-core/test/tools/image-compress.test.ts index 3eb3f32f7..a063327eb 100644 --- a/packages/agent-core/test/tools/image-compress.test.ts +++ b/packages/agent-core/test/tools/image-compress.test.ts @@ -24,9 +24,15 @@ * `` note (dims, sizes, readback path) * - annotate: compressImageContentParts can collect that caption for * each compressed image and persist the original via a callback + * - quality guards: a 1px checkerboard downscales to flat gray (no + * spectral aliasing) at integer and fractional ratios, with jimp's + * point-sampled BILINEAR mode pinned as the aliasing counter-example; + * fully transparent pixels never bleed color into opaque edges; mean + * brightness survives the downscale; recompressing a compressed + * result is a no-op; extreme aspect ratios never collapse to zero */ -import { Jimp } from 'jimp'; +import { Jimp, ResizeStrategy } from 'jimp'; import { describe, expect, it } from 'vitest'; // eslint-disable-next-line import/no-unresolved @@ -42,6 +48,8 @@ import { } from '../../src/tools/support/image-compress'; // eslint-disable-next-line import/no-unresolved import { sniffImageDimensions } from '../../src/tools/support/file-type'; +// eslint-disable-next-line import/no-unresolved +import type { TelemetryClient, TelemetryProperties } from '../../src/telemetry'; // ── fixtures ───────────────────────────────────────────────────────── @@ -76,11 +84,77 @@ async function noisePng(width: number, height: number, alpha = false): Promise { + const image = new Jimp({ width, height, color: 0x000000ff }); + fillXorshiftNoise(image.bitmap.data); + return new Uint8Array(await image.getBuffer('image/png')); +} + +/** JPEG twin of {@link randomNoisePng}, for exercising the JPEG source path. */ +async function randomNoiseJpeg(width: number, height: number): Promise { + const image = new Jimp({ width, height, color: 0x000000ff }); + fillXorshiftNoise(image.bitmap.data); + return new Uint8Array(await image.getBuffer('image/jpeg', { quality: 90 })); +} + +function fillXorshiftNoise(data: Buffer | Uint8Array): void { + let state = 0x9e3779b9; + const next = (): number => { + state ^= (state << 13) >>> 0; + state ^= state >>> 17; + state ^= (state << 5) >>> 0; + state >>>= 0; + return state & 0xff; + }; + for (let i = 0; i < data.length; i += 4) { + data[i] = next(); + data[i + 1] = next(); + data[i + 2] = next(); + data[i + 3] = 0xff; + } +} + async function decodeAlpha(bytes: Uint8Array): Promise { const image = await Jimp.fromBuffer(Buffer.from(bytes)); return image.hasAlpha(); } +/** + * Insert a minimal EXIF APP1 segment carrying only an Orientation tag right + * after the JPEG SOI marker (jimp itself never writes EXIF). + */ +function withExifOrientation(jpeg: Uint8Array, orientation: number): Uint8Array { + // TIFF body, little-endian: 8-byte header + IFD0 with a single entry. + const tiff = Buffer.alloc(26); + tiff.write('II', 0, 'latin1'); + tiff.writeUInt16LE(42, 2); + tiff.writeUInt32LE(8, 4); // offset of IFD0 + tiff.writeUInt16LE(1, 8); // one directory entry + tiff.writeUInt16LE(0x0112, 10); // tag: Orientation + tiff.writeUInt16LE(3, 12); // type: SHORT + tiff.writeUInt32LE(1, 14); // count + tiff.writeUInt16LE(orientation, 18); // value, left-aligned in the 4-byte field + tiff.writeUInt32LE(0, 22); // no next IFD + const exifBody = Buffer.concat([Buffer.from('Exif\0\0', 'latin1'), tiff]); + const app1Header = Buffer.alloc(4); + app1Header.writeUInt16BE(0xff_e1, 0); + app1Header.writeUInt16BE(exifBody.length + 2, 2); + return new Uint8Array( + Buffer.concat([ + Buffer.from(jpeg.subarray(0, 2)), // SOI + app1Header, + exifBody, + Buffer.from(jpeg.subarray(2)), + ]), + ); +} + // ── fast path ──────────────────────────────────────────────────────── describe('compressImageForModel — fast path', () => { @@ -106,15 +180,15 @@ describe('compressImageForModel — fast path', () => { describe('compressImageForModel — dimension cap', () => { it('scales the longest edge down to MAX_IMAGE_EDGE_PX, preserving aspect', async () => { - const png = await solidPng(3000, 1500); + const png = await solidPng(4500, 2250); const result = await compressImageForModel(png, 'image/png'); expect(result.changed).toBe(true); expect(Math.max(result.width, result.height)).toBe(MAX_IMAGE_EDGE_PX); - // 3000x1500 → 2000x1000 (aspect 2:1 preserved). - expect(result.width).toBe(2000); - expect(result.height).toBe(1000); + // 4500x2250 → 3000x1500 (aspect 2:1 preserved). + expect(result.width).toBe(3000); + expect(result.height).toBe(1500); const dims = sniffImageDimensions(result.data); - expect(dims).toEqual({ width: 2000, height: 1000 }); + expect(dims).toEqual({ width: 3000, height: 1500 }); }); it('respects a custom maxEdge', async () => { @@ -128,7 +202,7 @@ describe('compressImageForModel — dimension cap', () => { it('keeps a downscaled opaque PNG lossless (no needless JPEG conversion)', async () => { // A screenshot-like opaque PNG that only needs downscaling must stay PNG so // sharp text is not degraded by JPEG artifacts. - const png = await solidPng(3000, 1500); + const png = await solidPng(4500, 2250); const result = await compressImageForModel(png, 'image/png'); expect(result.changed).toBe(true); expect(result.mimeType).toBe('image/png'); @@ -148,7 +222,7 @@ describe('compressImageForModel — byte budget', () => { }); it('keeps a translucent PNG as PNG when the budget allows', async () => { - const png = await translucentPng(2600, 2600); + const png = await translucentPng(3600, 1800); const result = await compressImageForModel(png, 'image/png'); expect(result.changed).toBe(true); expect(result.mimeType).toBe('image/png'); @@ -163,6 +237,66 @@ describe('compressImageForModel — byte budget', () => { expect(result.mimeType).toBe('image/jpeg'); expect(result.finalByteLength).toBeLessThan(result.originalByteLength); }); + + it('steps down through the 2000px edge before the 1000px fallback', async () => { + // Regression guard for the 3000px cap raise: a PNG whose fitted encode + // is over budget but whose 2000px encode fits must come back at 2000px + // (as it did under the old cap), not skip straight to 1000px. + // The budget is anchored to the actual 2000px encode size (probed with + // an unlimited budget) so the test does not depend on exact deflate + // output sizes. + const png = await randomNoisePng(2400, 600); + const probe = await compressImageForModel(png, 'image/png', { + maxEdge: 2000, + byteBudget: Number.MAX_SAFE_INTEGER, + }); + expect(probe.changed).toBe(true); + expect(probe.mimeType).toBe('image/png'); + expect(Math.max(probe.width, probe.height)).toBe(2000); + // Sanity: the anchor budget must sit below the input size, or the run + // below would pass through on the fast path instead of re-encoding. + expect(probe.finalByteLength + 1024).toBeLessThan(png.length); + + const result = await compressImageForModel(png, 'image/png', { + byteBudget: probe.finalByteLength + 1024, + }); + expect(result.changed).toBe(true); + expect(result.mimeType).toBe('image/png'); + expect(Math.max(result.width, result.height)).toBe(2000); + }); + + it( + 're-runs the JPEG quality ladder at fallback sizes instead of jumping to q20', + async () => { + // A JPEG whose quality ladder fails at every size above 1000px, with the + // budget tuned so that at 1000px a mid-quality (q60) encode fits. The + // fallback must walk the ladder again and return that q60 encode — not + // collapse straight to q20 and needlessly destroy detail. + // The probe replays the implementation's exact resize chain + // (2400 → 2000 → 1000): box-resizing twice does not yield the same + // bitmap as resizing once, and JPEG encoding is deterministic, so the + // probed q60 size matches the implementation's encode byte-for-byte. + // The width must exceed 2000px to exercise the full fallback chain; the + // height is kept small so the ~11 JPEG encodes stay fast on slow CI. + const jpeg = await randomNoiseJpeg(2400, 300); + const probe = await Jimp.fromBuffer(Buffer.from(jpeg)); + probe.resize({ w: 2000, h: 250 }); + probe.resize({ w: 1000, h: 125 }); + const q60Size = (await probe.getBuffer('image/jpeg', { quality: 60 })).length; + const q20Size = (await probe.getBuffer('image/jpeg', { quality: 20 })).length; + expect(q60Size).toBeGreaterThan(q20Size); // sanity: the anchor separates the rungs + + const result = await compressImageForModel(jpeg, 'image/jpeg', { + byteBudget: q60Size + 256, + }); + expect(result.changed).toBe(true); + expect(result.mimeType).toBe('image/jpeg'); + expect(Math.max(result.width, result.height)).toBe(1000); + // The highest quality that fits the budget at 1000px is q60. + expect(result.finalByteLength).toBe(q60Size); + }, + 15_000, + ); }); // ── fallback / robustness ──────────────────────────────────────────── @@ -219,7 +353,7 @@ describe('compressImageForModel — fallback', () => { it('skips compression for payloads over the byte cap without decoding', async () => { // Over the edge (so not the fast path), but capped by maxDecodeBytes. - const png = await solidPng(3000, 100); + const png = await solidPng(3200, 100); const result = await compressImageForModel(png, 'image/png', { maxDecodeBytes: 64 }); expect(result.changed).toBe(false); expect(result.data).toBe(png); // passthrough → Jimp was never called @@ -231,9 +365,9 @@ describe('compressImageForModel — fallback', () => { describe('compressImageForModel — invariants', () => { it('changed always yields a within-cap, decodable payload', async () => { const cases: Uint8Array[] = [ - await solidPng(3000, 1500), + await solidPng(4500, 2250), await noisePng(900, 900), - await translucentPng(2600, 2600), + await translucentPng(3600, 1800), ]; for (const bytes of cases) { const result = await compressImageForModel(bytes, 'image/png'); @@ -276,7 +410,7 @@ describe('compressBase64ForModel', () => { }); it('skips a base64 payload over the byte cap without decoding', async () => { - const png = await solidPng(3000, 100); // over edge, would otherwise compress + const png = await solidPng(3200, 100); // over edge, would otherwise compress const base64 = Buffer.from(png).toString('base64'); const result = await compressBase64ForModel(base64, 'image/png', { maxDecodeBytes: 64 }); expect(result.changed).toBe(false); @@ -300,7 +434,7 @@ describe('compressImageForModel — performance', () => { }); it('compresses a large image within a generous time bound', async () => { - const png = await solidPng(3000, 2000); + const png = await solidPng(4500, 3000); const start = performance.now(); const result = await compressImageForModel(png, 'image/png'); const elapsed = performance.now() - start; @@ -310,7 +444,7 @@ describe('compressImageForModel — performance', () => { it('exposes a sane default budget', () => { expect(IMAGE_BYTE_BUDGET).toBeGreaterThan(0); - expect(MAX_IMAGE_EDGE_PX).toBe(2000); + expect(MAX_IMAGE_EDGE_PX).toBe(3000); }); }); @@ -322,7 +456,7 @@ describe('compressImageContentParts', () => { } it('compresses an oversized inline image part, leaving other parts untouched', async () => { - const big = await solidPng(2600, 2600); + const big = await solidPng(3600, 1800); const parts = [ { type: 'text' as const, text: 'look at this' }, { type: 'image_url' as const, imageUrl: { url: dataUrl('image/png', big) } }, @@ -355,7 +489,7 @@ describe('compressImageContentParts', () => { }); it('keeps an image part id when rewriting the compressed url', async () => { - const big = await solidPng(2600, 2600); + const big = await solidPng(3600, 1800); const parts = [ { type: 'image_url' as const, imageUrl: { url: dataUrl('image/png', big), id: 'att-1' } }, ]; @@ -369,6 +503,36 @@ describe('compressImageContentParts', () => { // ── original-dimension metadata ────────────────────────────────────── +describe('compressImageForModel — EXIF orientation', () => { + it('reports original dimensions in the decoded (EXIF-rotated) space', async () => { + // Orientation 6 (rotate 90° CW): the file header says 120x80, but jimp + // decodes to 80x120 — the space the sent image and any later crop region + // actually live in. The reported original dimensions must match it, not + // the pre-rotation header sniff. + const jpeg = withExifOrientation(await solidJpeg(120, 80), 6); + const result = await compressImageForModel(jpeg, 'image/jpeg', { maxEdge: 64 }); + expect(result.changed).toBe(true); + expect(result.originalWidth).toBe(80); + expect(result.originalHeight).toBe(120); + // The sent image keeps the rotated (portrait) aspect. + expect(result.width).toBeLessThan(result.height); + }); + + it('reports display-space dimensions for an EXIF-rotated passthrough', async () => { + // Within both budgets → no decode ever happens. The header sniff itself + // must account for EXIF orientation so passthrough metadata agrees with + // the space a later region readback (which decodes) will use. + const jpeg = withExifOrientation(await solidJpeg(120, 80), 6); + const result = await compressImageForModel(jpeg, 'image/jpeg'); + expect(result.changed).toBe(false); + expect(result.data).toBe(jpeg); // fast path — not decoded + expect(result.originalWidth).toBe(80); + expect(result.originalHeight).toBe(120); + expect(result.width).toBe(80); + expect(result.height).toBe(120); + }); +}); + describe('compressImageForModel — original dimensions metadata', () => { it('reports original dimensions on passthrough and compressed results', async () => { const small = await solidPng(64, 64); @@ -377,23 +541,23 @@ describe('compressImageForModel — original dimensions metadata', () => { expect(pass.originalWidth).toBe(64); expect(pass.originalHeight).toBe(64); - const big = await solidPng(3000, 1500); + const big = await solidPng(4500, 2250); const shrunk = await compressImageForModel(big, 'image/png'); expect(shrunk.changed).toBe(true); - expect(shrunk.originalWidth).toBe(3000); - expect(shrunk.originalHeight).toBe(1500); - expect(shrunk.width).toBe(2000); + expect(shrunk.originalWidth).toBe(4500); + expect(shrunk.originalHeight).toBe(2250); + expect(shrunk.width).toBe(3000); }); it('reports original dimensions through the base64 wrapper', async () => { - const big = await solidPng(2600, 1300); + const big = await solidPng(3900, 1950); const base64 = Buffer.from(big).toString('base64'); const result = await compressBase64ForModel(base64, 'image/png'); expect(result.changed).toBe(true); - expect(result.originalWidth).toBe(2600); - expect(result.originalHeight).toBe(1300); - expect(result.width).toBe(2000); - expect(result.height).toBe(1000); + expect(result.originalWidth).toBe(3900); + expect(result.originalHeight).toBe(1950); + expect(result.width).toBe(3000); + expect(result.height).toBe(1500); }); }); @@ -464,18 +628,18 @@ describe('cropImageForModel', () => { }); it('downscales an oversized crop to the edge cap by default', async () => { - const png = await solidPng(3000, 1500); + const png = await solidPng(4500, 2250); const result = await cropImageForModel(png, 'image/png', { x: 0, y: 0, - width: 2500, + width: 4000, height: 1200, }); expect(result.ok).toBe(true); if (!result.ok) return; expect(result.resized).toBe(true); expect(Math.max(result.width, result.height)).toBeLessThanOrEqual(MAX_IMAGE_EDGE_PX); - expect(result.region).toEqual({ x: 0, y: 0, width: 2500, height: 1200 }); + expect(result.region).toEqual({ x: 0, y: 0, width: 4000, height: 1200 }); }); it('keeps native resolution with skipResize', async () => { @@ -649,7 +813,7 @@ describe('compressImageContentParts — annotate', () => { } it('collects a caption for a compressed image and persists the original', async () => { - const big = await solidPng(2600, 2600); + const big = await solidPng(3600, 1800); const persisted: { bytes: Uint8Array; mimeType: string }[] = []; const parts = [{ type: 'image_url' as const, imageUrl: { url: dataUrl('image/png', big) } }]; const out = await compressImageContentParts(parts, { @@ -665,7 +829,7 @@ describe('compressImageContentParts — annotate', () => { expect(out.parts).toHaveLength(1); expect(out.parts[0]?.type).toBe('image_url'); expect(out.captions).toHaveLength(1); - expect(out.captions[0]).toContain('2600x2600'); + expect(out.captions[0]).toContain('3600x1800'); expect(out.captions[0]).toContain('/tmp/originals/big.png'); expect(persisted).toHaveLength(1); expect(persisted[0]?.mimeType).toBe('image/png'); @@ -684,7 +848,7 @@ describe('compressImageContentParts — annotate', () => { }); it('captions without a path when persistence fails', async () => { - const big = await solidPng(2600, 2600); + const big = await solidPng(3600, 1800); const parts = [{ type: 'image_url' as const, imageUrl: { url: dataUrl('image/png', big) } }]; const out = await compressImageContentParts(parts, { annotate: { persistOriginal: () => Promise.resolve(null) }, @@ -694,3 +858,390 @@ describe('compressImageContentParts — annotate', () => { expect(out.captions[0]).toMatch(/not preserved/i); }); }); + +// ── downscale quality guards ───────────────────────────────────────── +// +// Downscaling is a resampling operation: input frequencies above the output +// Nyquist limit must be filtered out (averaged), or they fold back as +// low-frequency moiré — spectral aliasing. A 1px checkerboard is the +// worst-case probe: ALL of its energy sits at the input Nyquist frequency, +// so a resampler that skips source pixels turns it into high-contrast +// artifacts, while a correct full-coverage average yields flat ~50% gray. +// These tests pin the compressor to the correct behavior, keep the aliasing +// counter-example executable, and cover the other classic downscale bugs +// (transparent-pixel bleed, brightness drift, iterative degradation, +// degenerate aspect ratios). + +/** 1px checkerboard: every pixel alternates black/white in both axes. */ +async function checkerboardPng(size: number): Promise { + const image = new Jimp({ width: size, height: size, color: 0x000000ff }); + const data = image.bitmap.data; + for (let y = 0; y < size; y += 1) { + for (let x = 0; x < size; x += 1) { + const v = (x + y) % 2 === 0 ? 0 : 255; + const i = (y * size + x) * 4; + data[i] = v; + data[i + 1] = v; + data[i + 2] = v; + data[i + 3] = 0xff; + } + } + return new Uint8Array(await image.getBuffer('image/png')); +} + +interface GrayStats { + readonly min: number; + readonly max: number; + readonly mean: number; +} + +/** Min/max/mean over the red channel (all probes here are grayscale). */ +function grayStats(image: { bitmap: { data: Buffer | Uint8Array } }): GrayStats { + const data = image.bitmap.data; + let min = 255; + let max = 0; + let sum = 0; + for (let i = 0; i < data.length; i += 4) { + const v = data[i]!; + if (v < min) min = v; + if (v > max) max = v; + sum += v; + } + return { min, max, mean: sum / (data.length / 4) }; +} + +describe('compressImageForModel — downscale quality guards', () => { + it('averages a 1px checkerboard to flat gray at an integer ratio (no aliasing)', async () => { + // 2000 → 500 (4:1). Every output pixel covers a 4×4 block holding 8 + // black and 8 white pixels, so a full-coverage average lands on ~127. + // Aliasing would instead show up as black/white patches or moiré bands. + const png = await checkerboardPng(2000); + const result = await compressImageForModel(png, 'image/png', { maxEdge: 500 }); + expect(result.changed).toBe(true); + expect(Math.max(result.width, result.height)).toBe(500); + + const decoded = await Jimp.fromBuffer(Buffer.from(result.data)); + const { min, max } = grayStats(decoded); + expect(min).toBeGreaterThanOrEqual(118); + expect(max).toBeLessThanOrEqual(138); + }); + + it('stays alias-free at a non-integer ratio (fractional pixel coverage)', async () => { + // 2000 → 780 (≈2.56:1). Non-integer ratios are where phase-dependent + // point sampling degrades worst. Fractional window coverage leaves the + // average some mild texture, but nothing may approach black or white. + const png = await checkerboardPng(2000); + const result = await compressImageForModel(png, 'image/png', { maxEdge: 780 }); + expect(result.changed).toBe(true); + + const decoded = await Jimp.fromBuffer(Buffer.from(result.data)); + const { min, max } = grayStats(decoded); + expect(min).toBeGreaterThanOrEqual(90); + expect(max).toBeLessThanOrEqual(165); + }); + + it('control: jimp point-sampled BILINEAR aliases the same input (keeps the probe honest)', async () => { + // Executable counter-example for the constraint documented on + // fitWithinEdge: the named ResizeStrategy modes sample a fixed 2×2 + // neighborhood around the mapped point and skip the rest. At 4:1 the + // sample grid lands on a single checkerboard phase and the 50%-gray + // pattern collapses to solid black — the pattern's energy is entirely + // misrepresented. This proves the two tests above can fail (the probe + // distinguishes resamplers) and pins the library behavior the + // mode-less default call relies on — if jimp ever changes either + // side, revisit the fitWithinEdge comment. + const image = await Jimp.fromBuffer(Buffer.from(await checkerboardPng(2000))); + image.resize({ w: 500, h: 500, mode: ResizeStrategy.BILINEAR }); + const { min, max, mean } = grayStats(image); + // The correct answer is flat ~127 gray (mean ≈ 127, max-min ≈ 0). + // Aliasing shows up as a solid black/white collapse or full-contrast + // banding — far from that answer regardless of sampling phase. + const aliased = mean < 60 || mean > 195 || max - min > 200; + expect(aliased).toBe(true); + }); + + it('never bleeds color from fully transparent pixels into visible ones', async () => { + // Fully transparent pixels still carry RGB values. A resizer that + // blends them into the average tints every transparency edge (halo). + // Probe: a fully transparent BRIGHT RED field around an opaque blue + // square — after a 4:1 downscale no visible pixel may pick up red. + const size = 1600; + const image = new Jimp({ width: size, height: size, color: 0xff000000 }); // red, alpha 0 + const data = image.bitmap.data; + for (let y = 400; y < 1200; y += 1) { + for (let x = 400; x < 1200; x += 1) { + const i = (y * size + x) * 4; + data[i] = 0; + data[i + 1] = 0; + data[i + 2] = 0xff; + data[i + 3] = 0xff; + } + } + const png = new Uint8Array(await image.getBuffer('image/png')); + + const result = await compressImageForModel(png, 'image/png', { maxEdge: 400 }); + expect(result.changed).toBe(true); + expect(result.mimeType).toBe('image/png'); // alpha survives + + const decoded = await Jimp.fromBuffer(Buffer.from(result.data)); + const out = decoded.bitmap.data; + let visible = 0; + for (let i = 0; i < out.length; i += 4) { + if (out[i + 3]! >= 8) { + visible += 1; + expect(out[i]!).toBeLessThanOrEqual(16); // red channel stays ~0 + } + } + expect(visible).toBeGreaterThan(0); // the blue square is still there + }); + + it('preserves mean brightness through the downscale (no energy drift)', async () => { + // A normalized filter keeps the image mean; drift here would indicate + // non-normalized weights (or a broken gamma pipeline stage). + const png = await noisePng(900, 900); + const input = await Jimp.fromBuffer(Buffer.from(png)); + const inputMean = grayStats(input).mean; + + const result = await compressImageForModel(png, 'image/png', { maxEdge: 225 }); + expect(result.changed).toBe(true); + const output = await Jimp.fromBuffer(Buffer.from(result.data)); + expect(Math.abs(grayStats(output).mean - inputMean)).toBeLessThan(3); + }); + + it('recompressing a compressed result is a no-op (no iterative degradation)', async () => { + // Model-bound bytes can re-enter the pipeline (session replay, MCP + // round-trips). Once within budget they must pass through untouched + // instead of being shaved a little smaller on every pass. + const first = await compressImageForModel(await solidPng(4500, 2250), 'image/png'); + expect(first.changed).toBe(true); + + const second = await compressImageForModel(first.data, first.mimeType); + expect(second.changed).toBe(false); + expect(second.data).toBe(first.data); // identity — not even re-decoded + }); + + it('keeps a degenerate aspect ratio at least 1px tall (no zero-size collapse)', async () => { + // 9000×2 scaled to a 3000px edge would round the short side to 0.67px; + // the resizer must clamp to 1, not produce an undecodable 3000×0 image. + const png = await solidPng(9000, 2); + const result = await compressImageForModel(png, 'image/png'); + expect(result.changed).toBe(true); + expect(result.width).toBe(3000); + expect(result.height).toBe(1); + expect(sniffImageDimensions(result.data)).toEqual({ width: 3000, height: 1 }); + }); +}); + +// ── telemetry ──────────────────────────────────────────────────────── + +interface CapturedEvent { + readonly event: string; + readonly props: TelemetryProperties; +} + +function captureTelemetry(): { client: TelemetryClient; events: CapturedEvent[] } { + const events: CapturedEvent[] = []; + return { + client: { track: (event, props) => events.push({ event, props: props ?? {} }) }, + events, + }; +} + +describe('compressImageForModel — telemetry', () => { + it('reports a compressed image with sizes, formats, and duration', async () => { + const { client, events } = captureTelemetry(); + const png = await solidPng(4500, 2250); + const result = await compressImageForModel(png, 'image/png', { + telemetry: { client, source: 'read_media' }, + }); + expect(result.changed).toBe(true); + + expect(events).toHaveLength(1); + const { event, props } = events[0]!; + expect(event).toBe('image_compress'); + expect(props['source']).toBe('read_media'); + expect(props['outcome']).toBe('compressed'); + expect(props['input_mime']).toBe('image/png'); + expect(props['output_mime']).toBe(result.mimeType); + expect(props['original_bytes']).toBe(png.length); + expect(props['final_bytes']).toBe(result.finalByteLength); + expect(props['original_width']).toBe(4500); + expect(props['original_height']).toBe(2250); + expect(props['final_width']).toBe(3000); + expect(props['final_height']).toBe(1500); + expect(props['exif_transposed']).toBe(false); + expect(typeof props['duration_ms']).toBe('number'); + }); + + it('reports the fast path as passthrough_fast', async () => { + const { client, events } = captureTelemetry(); + await compressImageForModel(await solidPng(64, 64), 'image/png', { + telemetry: { client, source: 'tui_paste' }, + }); + expect(events).toHaveLength(1); + expect(events[0]!.props['outcome']).toBe('passthrough_fast'); + expect(events[0]!.props['source']).toBe('tui_paste'); + }); + + it('reports decode guards as passthrough_guard', async () => { + // Decompression-bomb header: 30000×30000 with no pixel data. + const header = Buffer.alloc(24); + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).copy(header, 0); + header.writeUInt32BE(13, 8); + header.write('IHDR', 12, 'latin1'); + header.writeUInt32BE(30000, 16); + header.writeUInt32BE(30000, 20); + + const bomb = captureTelemetry(); + await compressImageForModel(new Uint8Array(header), 'image/png', { + telemetry: { client: bomb.client, source: 'mcp_tool_result' }, + }); + expect(bomb.events[0]!.props['outcome']).toBe('passthrough_guard'); + + const byteCap = captureTelemetry(); + await compressImageForModel(await solidPng(3200, 100), 'image/png', { + maxDecodeBytes: 64, + telemetry: { client: byteCap.client, source: 'mcp_tool_result' }, + }); + expect(byteCap.events[0]!.props['outcome']).toBe('passthrough_guard'); + }); + + it('reports non-recodable formats and empty input as passthrough_unsupported', async () => { + const gif = captureTelemetry(); + await compressImageForModel( + new Uint8Array([0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 1, 0, 1, 0]), + 'image/gif', + { telemetry: { client: gif.client, source: 'mcp_tool_result' } }, + ); + expect(gif.events[0]!.props['outcome']).toBe('passthrough_unsupported'); + + const empty = captureTelemetry(); + await compressImageForModel(new Uint8Array(0), 'image/png', { + telemetry: { client: empty.client, source: 'mcp_tool_result' }, + }); + expect(empty.events[0]!.props['outcome']).toBe('passthrough_unsupported'); + }); + + it('reports undecodable bytes as passthrough_error', async () => { + // A tiny corrupt blob would pass through on the fast path (unknown dims, + // small bytes) without ever decoding; to reach the decoder the header + // must claim an over-cap size. 4000×4000 forces a decode of garbage. + const { client, events } = captureTelemetry(); + const corrupt = Buffer.alloc(32); + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).copy(corrupt, 0); + corrupt.writeUInt32BE(13, 8); + corrupt.write('IHDR', 12, 'latin1'); + corrupt.writeUInt32BE(4000, 16); + corrupt.writeUInt32BE(4000, 20); + await compressImageForModel(new Uint8Array(corrupt), 'image/png', { + telemetry: { client, source: 'prompt_inline' }, + }); + expect(events[0]!.props['outcome']).toBe('passthrough_error'); + }); + + it('marks EXIF-transposed inputs', async () => { + const { client, events } = captureTelemetry(); + const jpeg = withExifOrientation(await solidJpeg(120, 80), 6); + await compressImageForModel(jpeg, 'image/jpeg', { + maxEdge: 64, + telemetry: { client, source: 'read_media' }, + }); + expect(events[0]!.props['outcome']).toBe('compressed'); + expect(events[0]!.props['exif_transposed']).toBe(true); + }); + + it('reports the base64 early size-skip as passthrough_guard', async () => { + const { client, events } = captureTelemetry(); + const base64 = Buffer.from(await solidPng(3200, 100)).toString('base64'); + await compressBase64ForModel(base64, 'image/png', { + maxDecodeBytes: 64, + telemetry: { client, source: 'prompt_file' }, + }); + expect(events).toHaveLength(1); + expect(events[0]!.props['outcome']).toBe('passthrough_guard'); + expect(events[0]!.props['source']).toBe('prompt_file'); + }); + + it('threads telemetry through compressImageContentParts', async () => { + const { client, events } = captureTelemetry(); + const big = await solidPng(3600, 1800); + const url = `data:image/png;base64,${Buffer.from(big).toString('base64')}`; + await compressImageContentParts([{ type: 'image_url', imageUrl: { url } }], { + telemetry: { client, source: 'mcp_tool_result' }, + }); + expect(events).toHaveLength(1); + expect(events[0]!.event).toBe('image_compress'); + expect(events[0]!.props['outcome']).toBe('compressed'); + expect(events[0]!.props['source']).toBe('mcp_tool_result'); + }); + + it('never lets a throwing telemetry client break compression', async () => { + const throwing: TelemetryClient = { + track: () => { + throw new Error('sink down'); + }, + }; + const png = await solidPng(4500, 2250); + const result = await compressImageForModel(png, 'image/png', { + telemetry: { client: throwing, source: 'read_media' }, + }); + expect(result.changed).toBe(true); // compression outcome unaffected + }); +}); + +describe('cropImageForModel — telemetry', () => { + it('reports a successful crop with the region share of the original', async () => { + const { client, events } = captureTelemetry(); + const png = await solidPng(1000, 500); + const outcome = await cropImageForModel( + png, + 'image/png', + { x: 0, y: 0, width: 500, height: 250 }, + { telemetry: { client, source: 'read_media' } }, + ); + expect(outcome.ok).toBe(true); + + expect(events).toHaveLength(1); + const { event, props } = events[0]!; + expect(event).toBe('image_crop'); + expect(props['source']).toBe('read_media'); + expect(props['ok']).toBe(true); + expect(props['resized']).toBe(false); + expect(props['original_width']).toBe(1000); + expect(props['original_height']).toBe(500); + // 500×250 of 1000×500 → a quarter of the pixels. + expect(props['region_area_ratio']).toBeCloseTo(0.25, 5); + expect(typeof props['duration_ms']).toBe('number'); + expect(typeof props['final_bytes']).toBe('number'); + }); + + it('classifies failures by kind', async () => { + const oob = captureTelemetry(); + await cropImageForModel( + await solidPng(100, 100), + 'image/png', + { x: 200, y: 0, width: 10, height: 10 }, + { telemetry: { client: oob.client, source: 'read_media' } }, + ); + expect(oob.events[0]!.props['ok']).toBe(false); + expect(oob.events[0]!.props['error_kind']).toBe('out_of_bounds'); + + const format = captureTelemetry(); + await cropImageForModel( + new Uint8Array([0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 1, 0, 1, 0]), + 'image/gif', + { x: 0, y: 0, width: 1, height: 1 }, + { telemetry: { client: format.client, source: 'read_media' } }, + ); + expect(format.events[0]!.props['error_kind']).toBe('unsupported_format'); + + const budget = captureTelemetry(); + await cropImageForModel( + await noisePng(900, 900), + 'image/png', + { x: 0, y: 0, width: 900, height: 900 }, + { skipResize: true, byteBudget: 8 * 1024, telemetry: { client: budget.client, source: 'read_media' } }, + ); + expect(budget.events[0]!.props['error_kind']).toBe('budget'); + }); +}); diff --git a/packages/agent-core/test/tools/read-media.test.ts b/packages/agent-core/test/tools/read-media.test.ts index b04adde77..5fd848549 100644 --- a/packages/agent-core/test/tools/read-media.test.ts +++ b/packages/agent-core/test/tools/read-media.test.ts @@ -14,6 +14,7 @@ import { ReadMediaFileTool, } from '../../src/tools/builtin/file/read-media'; import { MEDIA_SNIFF_BYTES, sniffImageDimensions } from '../../src/tools/support/file-type'; +import type { TelemetryClient } from '../../src/telemetry'; import { createFakeKaos, PERMISSIVE_WORKSPACE } from './fixtures/fake-kaos'; import { executeTool } from './fixtures/execute-tool'; @@ -41,6 +42,35 @@ const MP4_HEADER = Buffer.concat([ Buffer.from('mp42isom'), ]); +/** + * Insert a minimal EXIF APP1 segment carrying only an Orientation tag right + * after the JPEG SOI marker (jimp itself never writes EXIF). Mirrors the + * fixture in image-compress.test.ts. + */ +function withExifOrientation(jpeg: Uint8Array, orientation: number): Buffer { + // TIFF body, little-endian: 8-byte header + IFD0 with a single entry. + const tiff = Buffer.alloc(26); + tiff.write('II', 0, 'latin1'); + tiff.writeUInt16LE(42, 2); + tiff.writeUInt32LE(8, 4); // offset of IFD0 + tiff.writeUInt16LE(1, 8); // one directory entry + tiff.writeUInt16LE(0x0112, 10); // tag: Orientation + tiff.writeUInt16LE(3, 12); // type: SHORT + tiff.writeUInt32LE(1, 14); // count + tiff.writeUInt16LE(orientation, 18); // value, left-aligned in the 4-byte field + tiff.writeUInt32LE(0, 22); // no next IFD + const exifBody = Buffer.concat([Buffer.from('Exif\0\0', 'latin1'), tiff]); + const app1Header = Buffer.alloc(4); + app1Header.writeUInt16BE(0xff_e1, 0); + app1Header.writeUInt16BE(exifBody.length + 2, 2); + return Buffer.concat([ + Buffer.from(jpeg.subarray(0, 2)), // SOI + app1Header, + exifBody, + Buffer.from(jpeg.subarray(2)), + ]); +} + function capabilities(overrides: Partial = {}): ModelCapability { return { image_in: true, @@ -58,6 +88,7 @@ function makeReadMediaTool( readonly stat?: Kaos['stat'] | undefined; readonly readBytes?: Kaos['readBytes'] | undefined; readonly modelCapabilities?: ModelCapability | undefined; + readonly telemetry?: TelemetryClient | undefined; } = {}, ): ReadMediaFileTool { const kaos = createFakeKaos({ @@ -68,6 +99,8 @@ function makeReadMediaTool( kaos, PERMISSIVE_WORKSPACE, input.modelCapabilities ?? capabilities(), + undefined, + input.telemetry, ); } @@ -655,9 +688,9 @@ describe('ReadMediaFileTool', () => { it('downsamples an oversized image but reports original dimensions', async () => { const big = Buffer.from( - await new Jimp({ width: 2600, height: 2600, color: 0x3366ccff }).getBuffer('image/png'), + await new Jimp({ width: 3600, height: 3600, color: 0x3366ccff }).getBuffer('image/png'), ); - expect(sniffImageDimensions(big)).toEqual({ width: 2600, height: 2600 }); + expect(sniffImageDimensions(big)).toEqual({ width: 3600, height: 3600 }); const tool = makeReadMediaTool({ stat: vi.fn().mockResolvedValue({ ...DEFAULT_STAT, stSize: big.length }), @@ -678,14 +711,136 @@ describe('ReadMediaFileTool', () => { // The image actually sent to the model is downsampled to the edge cap. const sentBytes = Buffer.from(match![2]!, 'base64'); const sentDims = sniffImageDimensions(sentBytes); - expect(Math.max(sentDims!.width, sentDims!.height)).toBeLessThanOrEqual(2000); + expect(Math.max(sentDims!.width, sentDims!.height)).toBeLessThanOrEqual(3000); // The note keeps the ORIGINAL size so coordinate mapping holds. const systemText = noteText(result); - expect(systemText).toContain('2600x2600'); + expect(systemText).toContain('3600x3600'); expect(systemText).toContain(`${String(big.length)} bytes`); }); + it('reports an EXIF-rotated original in the decoded coordinate space', async () => { + // Orientation 6 (rotate 90° CW): the header says 3600x1800, but jimp + // decodes to 1800x3600 — the space the sent image and any region + // readback live in. The note's original size must match that space, + // not the pre-rotation header sniff. + const portrait = withExifOrientation( + new Uint8Array( + await new Jimp({ width: 3600, height: 1800, color: 0x3366ccff }).getBuffer('image/jpeg', { + quality: 90, + }), + ), + 6, + ); + const tool = makeReadMediaTool({ + stat: vi.fn().mockResolvedValue({ ...DEFAULT_STAT, stSize: portrait.length }), + readBytes: vi.fn().mockResolvedValue(portrait), + }); + + const result = await executeTool(tool, { + turnId: 't1', + toolCallId: 'c_exif', + args: { path: '/workspace/portrait.jpg' }, + signal, + }); + + const systemText = noteText(result); + expect(systemText).toContain('Original dimensions: 1800x3600'); + expect(systemText).toMatch(/downsampled to 1500x3000/); + }); + + it('reports the decoded size for a region read of an EXIF-rotated image', async () => { + // Region coordinates live in the decoded (rotated) space; the note's + // original size must agree with it even when the header sniff succeeds. + const portrait = withExifOrientation( + new Uint8Array( + await new Jimp({ width: 120, height: 80, color: 0x3366ccff }).getBuffer('image/jpeg', { + quality: 90, + }), + ), + 6, + ); + const tool = makeReadMediaTool({ + stat: vi.fn().mockResolvedValue({ ...DEFAULT_STAT, stSize: portrait.length }), + readBytes: vi.fn().mockResolvedValue(portrait), + }); + + const result = await executeTool(tool, { + turnId: 't1', + toolCallId: 'c_exif_region', + args: { path: '/workspace/portrait.jpg', region: { x: 0, y: 0, width: 40, height: 40 } }, + signal, + }); + + expect(noteText(result)).toContain('Original dimensions: 80x120'); + }); + + it('reports display-space dimensions for an EXIF-rotated image sent untouched', async () => { + // Within both budgets the original bytes are sent without decoding; the + // note must still report the display-space size so coordinates derived + // from it agree with a later region readback (which decodes). + const portrait = withExifOrientation( + new Uint8Array( + await new Jimp({ width: 120, height: 80, color: 0x3366ccff }).getBuffer('image/jpeg', { + quality: 90, + }), + ), + 6, + ); + const tool = makeReadMediaTool({ + stat: vi.fn().mockResolvedValue({ ...DEFAULT_STAT, stSize: portrait.length }), + readBytes: vi.fn().mockResolvedValue(portrait), + }); + + const result = await executeTool(tool, { + turnId: 't1', + toolCallId: 'c_exif_untouched', + args: { path: '/workspace/portrait.jpg' }, + signal, + }); + + const systemText = noteText(result); + expect(systemText).toContain('Original dimensions: 80x120'); + expect(systemText).not.toMatch(/downsampled/i); + }); + + it('emits image_compress and image_crop telemetry tagged read_media', async () => { + const events: { event: string; props: Record }[] = []; + const telemetry: TelemetryClient = { + track: (event, props) => events.push({ event, props: props ?? {} }), + }; + const big = Buffer.from( + await new Jimp({ width: 3600, height: 1800, color: 0x3366ccff }).getBuffer('image/png'), + ); + const tool = makeReadMediaTool({ + stat: vi.fn().mockResolvedValue({ ...DEFAULT_STAT, stSize: big.length }), + readBytes: vi.fn().mockResolvedValue(big), + telemetry, + }); + + await executeTool(tool, { + turnId: 't1', + toolCallId: 'c_tele_read', + args: { path: '/workspace/big.png' }, + signal, + }); + expect(events).toHaveLength(1); + expect(events[0]!.event).toBe('image_compress'); + expect(events[0]!.props['source']).toBe('read_media'); + expect(events[0]!.props['outcome']).toBe('compressed'); + + await executeTool(tool, { + turnId: 't1', + toolCallId: 'c_tele_crop', + args: { path: '/workspace/big.png', region: { x: 0, y: 0, width: 100, height: 100 } }, + signal, + }); + expect(events).toHaveLength(2); + expect(events[1]!.event).toBe('image_crop'); + expect(events[1]!.props['source']).toBe('read_media'); + expect(events[1]!.props['ok']).toBe(true); + }); + describe('region and full_resolution', () => { async function bigPng(width: number, height: number): Promise { return Buffer.from( @@ -725,7 +880,7 @@ describe('ReadMediaFileTool', () => { }); it('announces a downsampled delivery and the region readback in the block', async () => { - const big = await bigPng(2600, 2600); + const big = await bigPng(3600, 3600); const result = await executeTool(toolFor(big), { turnId: 't1', toolCallId: 'c_note', @@ -734,11 +889,11 @@ describe('ReadMediaFileTool', () => { }); const systemText = noteText(result); - expect(systemText).toContain('2600x2600'); + expect(systemText).toContain('3600x3600'); // Wording must not depend on serialization order: some providers keep // the note inline after the media, others flatten tool text and // re-attach the image after it — so no "above"/"below". - expect(systemText).toMatch(/The attached image was downsampled to 2000x2000/); + expect(systemText).toMatch(/The attached image was downsampled to 3000x3000/); expect(systemText).toMatch(/fine detail/i); expect(systemText).toContain('region'); }); @@ -797,7 +952,7 @@ describe('ReadMediaFileTool', () => { }); it('serves full_resolution when the bytes fit the per-image budget', async () => { - const big = await bigPng(2600, 1300); // over the edge cap, tiny in bytes + const big = await bigPng(3900, 1950); // over the edge cap, tiny in bytes const result = await executeTool(toolFor(big), { turnId: 't1', toolCallId: 'c_fullres', diff --git a/packages/node-sdk/src/index.ts b/packages/node-sdk/src/index.ts index 0fb4cc928..d84e395f9 100644 --- a/packages/node-sdk/src/index.ts +++ b/packages/node-sdk/src/index.ts @@ -87,6 +87,7 @@ export type { CompressImageResult, CompressBase64Result, ImageCompressionCaptionInput, + ImageCompressionTelemetry, } from '@moonshot-ai/agent-core'; // Experimental feature flags — types only. Resolved values come from diff --git a/packages/server/src/routes/prompts.ts b/packages/server/src/routes/prompts.ts index 3d089a799..50062f695 100644 --- a/packages/server/src/routes/prompts.ts +++ b/packages/server/src/routes/prompts.ts @@ -13,7 +13,7 @@ import { promptSteerResultSchema, type PromptSubmission, } from '@moonshot-ai/protocol'; -import { IPromptService, AuthModelNotResolvedError, AuthProvisioningRequiredError, AuthTokenMissingError, AuthTokenUnauthorizedError, PromptAlreadyCompletedError, PromptNotFoundError, SessionBusyError, SessionNotFoundError, FileNotFoundError, ICoreProcessService, IEnvironmentService, IFileStore, buildImageCompressionCaption, compressImageForModel, compressBase64ForModel, persistOriginalImage, sessionMediaOriginalsDir, type IInstantiationService, type GetResult } from '@moonshot-ai/agent-core'; +import { IPromptService, AuthModelNotResolvedError, AuthProvisioningRequiredError, AuthTokenMissingError, AuthTokenUnauthorizedError, PromptAlreadyCompletedError, PromptNotFoundError, SessionBusyError, SessionNotFoundError, FileNotFoundError, ICoreProcessService, IEnvironmentService, IFileStore, buildImageCompressionCaption, compressImageForModel, compressBase64ForModel, persistOriginalImage, sessionMediaOriginalsDir, withTelemetryContext, type IInstantiationService, type GetResult, type ImageCompressionTelemetry, type TelemetryClient } from '@moonshot-ai/agent-core'; import { z } from 'zod'; @@ -132,6 +132,12 @@ export function registerPromptsRoutes( const core = a.get(ICoreProcessService); const cacheDir = join(a.get(IEnvironmentService).homeDir, 'cache'); const resolved = await resolvePromptMediaFiles(body, fileStore, cacheDir, { + // Scoped to the session so prompt-ingestion image_compress events + // carry the same context as the per-session agent telemetry. + telemetry: + core.telemetry === undefined + ? undefined + : withTelemetryContext(core.telemetry, { sessionId: session_id }), // Resolved lazily — only when an inline base64 image actually // got compressed — so image-free prompts never pay the lookup. resolveOriginalsDir: async () => { @@ -270,6 +276,8 @@ interface ResolvePromptMediaOptions { * to the shared temp-dir cache. */ readonly resolveOriginalsDir?: () => Promise; + /** Report an `image_compress` event per compressed prompt image. */ + readonly telemetry?: TelemetryClient; } async function resolvePromptMediaFiles( @@ -288,13 +296,17 @@ async function resolvePromptMediaFiles( } return originalsDir; }; + const telemetryFor = (source: string): ImageCompressionTelemetry | undefined => + options.telemetry === undefined ? undefined : { client: options.telemetry, source }; const content: PromptSubmission['content'] = []; for (const part of body.content) { // Inline base64 image: compress the payload in place. This is the same // input-stage step as the file path below, for REST clients that submit an // image as `{ source: { kind: 'base64' } }` instead of uploading a file. if (part.type === 'image' && part.source.kind === 'base64') { - const compressed = await compressBase64ForModel(part.source.data, part.source.media_type); + const compressed = await compressBase64ForModel(part.source.data, part.source.media_type, { + telemetry: telemetryFor('prompt_inline'), + }); if (compressed.changed) { // There is no stored file to point at, so persist the original into // the session's media-originals dir (best effort, temp-dir fallback) @@ -357,7 +369,9 @@ async function resolvePromptMediaFiles( let mediaType = file.meta.media_type; let bytes: Uint8Array = data; if (part.type === 'image') { - const compressed = await compressImageForModel(data, mediaType); + const compressed = await compressImageForModel(data, mediaType, { + telemetry: telemetryFor('prompt_file'), + }); if (compressed.changed) { // The stored file already holds the original bytes — the caption // points straight at it, no extra copy needed. diff --git a/packages/server/test/prompt.e2e.test.ts b/packages/server/test/prompt.e2e.test.ts index 1019715c5..1df6c6e48 100644 --- a/packages/server/test/prompt.e2e.test.ts +++ b/packages/server/test/prompt.e2e.test.ts @@ -32,7 +32,12 @@ import { WebSocket } from 'ws'; import { Jimp } from 'jimp'; import type { Event, PromptSubmission } from '@moonshot-ai/protocol'; -import { IEventService, IPromptService, PromptService } from '@moonshot-ai/agent-core'; +import { + IEventService, + IPromptService, + PromptService, + type TelemetryClient, +} from '@moonshot-ai/agent-core'; import { IRestGateway, startServer, type ServerStartOptions, type RunningServer } from '../src'; import { fixedTokenAuth } from './helpers/serverHarness'; @@ -61,13 +66,14 @@ afterEach(async () => { async function bootDaemon( serviceOverrides?: ServerStartOptions['serviceOverrides'], + telemetry?: TelemetryClient, ): Promise { server = await startServer({ host: '127.0.0.1', port: 0, lockPath, logger: pino({ level: 'silent' }), - coreProcessOptions: { homeDir: bridgeHome }, + coreProcessOptions: { homeDir: bridgeHome, telemetry }, wsGatewayOptions: { pingIntervalMs: 5_000, pongTimeoutMs: 5_000 }, serviceOverrides: [fixedTokenAuth(), ...(serviceOverrides ?? [])], }); @@ -445,7 +451,7 @@ describe('POST /api/v1/sessions/{sid}/prompts — submit validation (W7.2 / Chai const sid = await createSession(r); const bigPng = Buffer.from( - await new Jimp({ width: 2600, height: 2600, color: 0x3366ccff }).getBuffer('image/png'), + await new Jimp({ width: 3600, height: 1800, color: 0x3366ccff }).getBuffer('image/png'), ); const upload = buildMultipart({ file: { fieldName: 'file', filename: 'big.png', contentType: 'image/png', data: bigPng }, @@ -479,7 +485,7 @@ describe('POST /api/v1/sessions/{sid}/prompts — submit validation (W7.2 / Chai const sentBytes = Buffer.from(part.source.data, 'base64'); const decoded = await Jimp.fromBuffer(sentBytes); // The model-facing copy is downsampled to the edge cap. - expect(Math.max(decoded.width, decoded.height)).toBeLessThanOrEqual(2000); + expect(Math.max(decoded.width, decoded.height)).toBeLessThanOrEqual(3000); // Compression is announced next to the image, and the caption points at // the stored file (which keeps the original bytes) for readback. @@ -488,7 +494,7 @@ describe('POST /api/v1/sessions/{sid}/prompts — submit validation (W7.2 / Chai throw new Error('expected a compression caption before the image part'); } expect(caption.text).toContain('Image compressed'); - expect(caption.text).toContain('2600x2600'); + expect(caption.text).toContain('3600x1800'); const pathMatch = /saved at "([^"]+)"/.exec(caption.text); expect(pathMatch).not.toBeNull(); const persisted = await readFile(pathMatch![1]!); @@ -516,10 +522,10 @@ describe('POST /api/v1/sessions/{sid}/prompts — submit validation (W7.2 / Chai ]); const sid = await createSession(r); - // Solid 2600×2600: over the edge cap but tiny in bytes, so it stays well + // Solid 3600×1800: over the edge cap but tiny in bytes, so it stays well // under Fastify's inline-JSON limit yet still benefits from downscaling. const base64 = Buffer.from( - await new Jimp({ width: 2600, height: 2600, color: 0x3366ccff }).getBuffer('image/png'), + await new Jimp({ width: 3600, height: 1800, color: 0x3366ccff }).getBuffer('image/png'), ).toString('base64'); const res = await appOf(r).inject({ @@ -538,7 +544,7 @@ describe('POST /api/v1/sessions/{sid}/prompts — submit validation (W7.2 / Chai throw new Error('expected a base64 image part'); } const decoded = await Jimp.fromBuffer(Buffer.from(part.source.data, 'base64')); - expect(Math.max(decoded.width, decoded.height)).toBeLessThanOrEqual(2000); + expect(Math.max(decoded.width, decoded.height)).toBeLessThanOrEqual(3000); // Inline base64 has no stored file, so the original is persisted into the // session's media-originals dir and the caption points there. @@ -559,6 +565,54 @@ describe('POST /api/v1/sessions/{sid}/prompts — submit validation (W7.2 / Chai expect(persisted.equals(Buffer.from(base64, 'base64'))).toBe(true); }); + it('scopes prompt image compression telemetry to the session', async () => { + // The agent-side image_compress sources inherit the session context from + // their per-session telemetry client; the prompt-ingestion sources must + // match, or prompt_inline/prompt_file events cannot be correlated with + // the session they belong to. + const events: { event: string; sessionId: string | null | undefined }[] = []; + const makeClient = (ctx: { sessionId?: string | null }): TelemetryClient => ({ + track: (event) => events.push({ event, sessionId: ctx.sessionId }), + withContext: (patch) => makeClient({ ...ctx, ...patch }), + }); + const r = await bootDaemon( + [ + [ + IPromptService, + createPromptServiceOverride({ + submit: async (_sid, body) => ({ + prompt_id: 'prompt_from_stub', + user_message_id: 'msg_from_stub', + status: 'running', + content: body.content, + created_at: '2026-06-09T00:00:00.000Z', + }), + }), + ], + ], + makeClient({}), + ); + const sid = await createSession(r); + + const base64 = Buffer.from( + await new Jimp({ width: 3600, height: 400, color: 0x3366ccff }).getBuffer('image/png'), + ).toString('base64'); + const res = await appOf(r).inject({ + method: 'POST', + url: `/api/v1/sessions/${sid}/prompts`, + payload: { + content: [ + { type: 'image', source: { kind: 'base64', media_type: 'image/png', data: base64 } }, + ], + }, + }); + expect(envelopeOf(res.json()).code).toBe(0); + + const compress = events.filter((e) => e.event === 'image_compress'); + expect(compress).toHaveLength(1); + expect(compress[0]!.sessionId).toBe(sid); + }); + it('returns 40407 when prompt image file_id is unknown', async () => { let submitted = false; const r = await bootDaemon([