From cfd14a1fe287abb8af92d367fe6dc463886f25f5 Mon Sep 17 00:00:00 2001 From: liruifengv Date: Thu, 6 Aug 2026 19:09:32 +0800 Subject: [PATCH] feat(kap-server): accept attachments on skill activation (#2693) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(kap-server): accept attachments on skill activation The :activate endpoint only took {args?}, so REST clients (web/desktop composers) could not attach uploads to a /skill invocation — attachments were silently dropped at the edge. - activateSkillRequestSchema gains an optional attachments field carrying the image/video/file subset of the prompt content wire shape. - The skills route resolves them through the same edge pipeline as prompt submissions (validate file refs → materialize/compress → convert), extracted from routes/prompts.ts into lib/promptMedia.ts. - AgentSkillService.activate appends the resolved parts after the rendered skill prompt in the activation's user message; SkillActivationInput gains an optional content field. The native RPC/TUI path is unchanged. - Attachment failures map to 40407 file.not_found / 40001 validation.failed, mirroring the prompts route. * fix(kap-server): drop the unused parseKimiFileUrl import in promptMedia * refactor: address review — header-only comments in the skill domain, provider id on protocol URL sources - agent-core-v2 keeps comments solely in the top-of-file block (scoped guide): SkillActivationInput.content documented in the skill.ts header, the activate() note folded into the skillService.ts header. - packages/protocol's image/video URL source gains the optional provider-issued id, matching the kap-server wire schema so parsing the public contract no longer strips it. * fix(kap-server): validate the skill before materializing activation attachments An unknown or non-user-activatable skill name with attachments ran the media pipeline first, streaming bytes into the session/cache dirs and compressing images for a request that activate() would reject with 40415/40912. The route now checks the session catalog up front (the service still re-validates) so invalid activations leave no disk or CPU side effects. --- .../agent-core-v2/src/agent/skill/skill.ts | 12 + .../src/agent/skill/skillService.ts | 10 +- packages/kap-server/src/lib/promptMedia.ts | 430 ++++++++++++++++++ .../kap-server/src/protocol/rest-skill.ts | 19 + packages/kap-server/src/routes/prompts.ts | 401 +--------------- packages/kap-server/src/routes/skills.ts | 69 ++- packages/kap-server/test/skills.test.ts | 102 ++++- packages/protocol/src/message.ts | 8 +- packages/protocol/src/rest/skill.ts | 20 +- 9 files changed, 671 insertions(+), 400 deletions(-) create mode 100644 packages/kap-server/src/lib/promptMedia.ts diff --git a/packages/agent-core-v2/src/agent/skill/skill.ts b/packages/agent-core-v2/src/agent/skill/skill.ts index 82c282a0e..ed4eb3e93 100644 --- a/packages/agent-core-v2/src/agent/skill/skill.ts +++ b/packages/agent-core-v2/src/agent/skill/skill.ts @@ -1,10 +1,22 @@ +/** + * `skill` domain — user-slash skill activation contract. + * + * `SkillActivationInput` carries the slash name and raw args, plus optional + * edge-resolved attachment parts (`content`) that the activation appends after + * the rendered skill prompt in its user message. `IAgentSkillService` starts + * the activation turn (`activate`) and records model-tool activations without + * a turn (`recordModelToolActivation`). Bound at Agent scope. + */ + import { createDecorator } from "#/_base/di/instantiation"; import type { SkillActivationOrigin } from '#/agent/contextMemory/types'; import type { Turn } from '#/agent/loop/loop'; +import type { ContentPart } from '#/kosong/contract/message'; export interface SkillActivationInput { readonly name: string; readonly args?: string; + readonly content?: readonly ContentPart[]; } export interface IAgentSkillService { diff --git a/packages/agent-core-v2/src/agent/skill/skillService.ts b/packages/agent-core-v2/src/agent/skill/skillService.ts index 2e056b104..aed7efba2 100644 --- a/packages/agent-core-v2/src/agent/skill/skillService.ts +++ b/packages/agent-core-v2/src/agent/skill/skillService.ts @@ -5,10 +5,11 @@ * records the activation as a `skill.activate` fact through `wire.dispatch` * (a stateless, identity-apply Op), derives the `skill.activated` event * through the Op's `toEvent`, drives user-slash activations into a new turn via - * `prompt`, and reports `skill_invoked` / `flow_invoked` through `telemetry`. - * `wire.replay` reapplies the fact as a no-op, so neither the event nor - * telemetry fires on resume (matching the former `restoring` guard). Bound at - * Agent scope. + * `prompt` (attachment parts from the caller ride the same user message after + * the rendered prompt), and reports `skill_invoked` / `flow_invoked` through + * `telemetry`. `wire.replay` reapplies the fact as a no-op, so neither the + * event nor telemetry fires on resume (matching the former `restoring` guard). + * Bound at Agent scope. */ import { randomUUID } from 'node:crypto'; @@ -70,6 +71,7 @@ export class AgentSkillService extends Service implements IAgentSkillService { skillDir: skill.dir, }), }, + ...(input.content ?? []), ]; const turn = await this.recordActivation( diff --git a/packages/kap-server/src/lib/promptMedia.ts b/packages/kap-server/src/lib/promptMedia.ts new file mode 100644 index 000000000..2895cecaf --- /dev/null +++ b/packages/kap-server/src/lib/promptMedia.ts @@ -0,0 +1,430 @@ +/** + * Prompt media/attachment pipeline shared by the prompt-submission and + * skill-activation edges. + * + * Three stages, in the order both routes apply them: + * 1. `assertPromptFileRefs` — fail fast on stale or mis-kinded `file_id` + * references before anything session-scoped is resolved or mutated. + * 2. `resolvePromptMediaFiles` — materialize uploads into session-local + * copies: arbitrary files become path-referenced attachments (a text + * notice the model opens with the Read tool), images are format-gated + * and compressed, videos become internal `kimi-file://` references. + * 3. `contentToCoreParts` — project the resolved wire content onto engine + * `ContentPart`s. + * + * Extracted from `routes/prompts.ts` so `routes/skills.ts` can run the exact + * same pipeline for skill-activation attachments. + */ + +import { createHash } from 'node:crypto'; +import { createWriteStream } from 'node:fs'; +import { mkdir, stat, writeFile } from 'node:fs/promises'; +import { extname, join } from 'node:path'; +import { pipeline } from 'node:stream/promises'; + +import { + buildKimiFileUrl, + buildImageCompressionCaption, + buildUnsupportedImageNotice, + compressBase64ForModel, + compressImageForModel, + decodeBase64Prefix, + Error2, + isModelAcceptedImageMime, + normalizeImageMime, + persistOriginalImage, + resolveEffectiveImageMime, + unsupportedImageMimeFromUrl, + type ContentPart, + type GetResult, + type IFileService, + type ImageCompressionTelemetry, + type ITelemetryService, +} from '@moonshot-ai/agent-core-v2'; + +import type { PromptSubmission } from '../protocol/rest-prompt'; + +/** + * The content list these helpers walk. Routes pass their own wire content: + * the full prompt submission's `content`, or a skill activation's + * `attachments` (same `MessageContent` parts, minus the text-ish kinds). + */ +type WireContent = PromptSubmission['content']; + +const VIDEO_EXT_BY_MIME: Record = { + 'video/mp4': '.mp4', + 'video/quicktime': '.mov', + 'video/webm': '.webm', + 'video/x-msvideo': '.avi', + 'video/x-matroska': '.mkv', + 'video/mpeg': '.mpeg', +}; + +/** + * Fail fast on stale or mis-kinded file references before anything + * session-scoped happens: a bad `file_id` (unknown, or a real file used with + * the wrong media kind, e.g. a PDF submitted as a video) must reject the + * request without creating the prompt agent and without touching the + * session's model/thinking/permission. + */ +export async function assertPromptFileRefs(content: WireContent, store: IFileService): Promise { + for (const part of content) { + if (part.type === 'file') { + await store.get(part.file_id); + } else if ((part.type === 'image' || part.type === 'video') && part.source.kind === 'file') { + const file = await store.get(part.source.file_id); + assertMediaFile(file, part.type); + } + } +} + +export function contentToCoreParts(content: WireContent): ContentPart[] { + const parts: ContentPart[] = []; + for (const part of content) { + if (part.type === 'text') parts.push({ type: 'text', text: part.text }); + else if (part.type === 'image' && part.source.kind === 'url') parts.push({ type: 'image_url', imageUrl: { url: part.source.url, id: part.source.id } }); + else if (part.type === 'image' && part.source.kind === 'base64') parts.push({ type: 'image_url', imageUrl: { url: `data:${part.source.media_type};base64,${part.source.data}` } }); + else if (part.type === 'video' && part.source.kind === 'url') parts.push({ type: 'video_url', videoUrl: { url: part.source.url, id: part.source.id } }); + else if (part.type === 'video' && part.source.kind === 'base64') parts.push({ type: 'video_url', videoUrl: { url: `data:${part.source.media_type};base64,${part.source.data}` } }); + } + return parts; +} + +export interface ResolvePromptMediaOptions { + /** + * Lazily resolve the session's media-originals dir for persisting the + * pre-compression bytes of inline base64 images. Only invoked when an image + * was actually compressed; a failure or undefined result falls back to the + * shared temp-dir cache. + */ + readonly resolveOriginalsDir?: () => Promise; + /** + * Lazily resolve the session's attachments dir for materializing arbitrary + * file uploads (and image bytes the provider rejects) into a path the model + * can open with the Read tool. A failure or undefined result falls back to + * the shared cache dir. + */ + readonly resolveAttachmentsDir?: () => Promise; + /** Report an `image_compress` event per compressed prompt image. */ + readonly telemetry?: ITelemetryService; +} + +/** + * Resolve a wire content list's media/file references into their final wire + * form: uploaded files materialize to a session-local path notice, images are + * format-gated and compressed, videos materialize to a `kimi-file://` + * reference. Returns the input array unchanged when nothing needed resolving. + */ +export async function resolvePromptMediaFiles( + input: WireContent, + store: IFileService, + cacheDir: string, + options: ResolvePromptMediaOptions = {}, +): Promise { + let changed = false; + let originalsDir: string | undefined; + let originalsDirResolved = false; + const resolveOriginalsDir = async (): Promise => { + if (!originalsDirResolved) { + originalsDirResolved = true; + originalsDir = await options.resolveOriginalsDir?.().catch(() => undefined); + } + return originalsDir; + }; + let attachmentsDir: string | undefined; + let attachmentsDirResolved = false; + const resolveAttachmentsDir = async (): Promise => { + if (!attachmentsDirResolved) { + attachmentsDirResolved = true; + attachmentsDir = await options.resolveAttachmentsDir?.().catch(() => undefined); + } + return attachmentsDir ?? cacheDir; + }; + const telemetryFor = (source: string): ImageCompressionTelemetry | undefined => + options.telemetry === undefined ? undefined : { client: options.telemetry, source }; + const content: WireContent = []; + for (const part of input) { + // Inline base64 image: compress the payload in place. This mirrors the v1 + // server path for REST clients that submit an image without uploading it. + if (part.type === 'image' && part.source.kind === 'base64') { + // Formats the provider cannot accept must never enter the session + // history — one unsupported image_url makes every later request fail. + // The bytes are authoritative: an image labeled image/png that is + // actually AVIF is gated on the sniffed format, not the label. The + // bytes are still the user's content, though: persist them as a + // path-referenced attachment so the model can read and convert them + // itself (best effort — the plain notice stands in when persisting + // fails). Inline base64 has no original name, so the file is addressed + // by content hash with a name derived from the sniffed format. + const effectiveMime = resolveEffectiveImageMime( + part.source.media_type, + decodeBase64Prefix(part.source.data), + ); + if (!isModelAcceptedImageMime(effectiveMime)) { + const bytes = Buffer.from(part.source.data, 'base64'); + const name = `image.${imageExtensionForMime(effectiveMime)}`; + const persisted = await persistAttachmentBytes( + bytes, + `${createHash('sha256').update(bytes).digest('hex').slice(0, 32)}-${name}`, + await resolveAttachmentsDir(), + ); + content.push({ + type: 'text', + text: persisted === null + ? buildUnsupportedImageNotice(effectiveMime) + : buildAttachedFileNotice(name, effectiveMime, bytes.length, persisted), + }); + changed = true; + continue; + } + const canonicalMime = normalizeImageMime(effectiveMime); + const compressed = await compressBase64ForModel(part.source.data, canonicalMime, { + telemetry: telemetryFor('prompt_inline'), + }); + if (compressed.changed) { + const dir = await resolveOriginalsDir(); + const originalPath = await persistOriginalImage( + Buffer.from(part.source.data, 'base64'), + part.source.media_type, + { dir }, + ); + content.push({ + type: 'text', + text: buildImageCompressionCaption({ + original: { + width: compressed.originalWidth, + height: compressed.originalHeight, + byteLength: compressed.originalByteLength, + mimeType: part.source.media_type, + }, + final: { + width: compressed.width, + height: compressed.height, + byteLength: compressed.finalByteLength, + mimeType: compressed.mimeType, + }, + originalPath, + }), + }); + content.push({ + type: 'image', + source: { kind: 'base64', media_type: compressed.mimeType, data: compressed.base64 }, + }); + changed = true; + } else { + content.push(part); + } + continue; + } + + // Remote image URL: no bytes to sniff, so reject when its path extension + // names a format providers reject (e.g. a link ending in `.avif`) — the + // notice keeps the URL so the model can still fetch and convert the + // image. Extensionless / unknown URLs pass through to the provider and + // the 400 recovery. Image+URL parts that pass are re-emitted unchanged. + if (part.type === 'image' && part.source.kind === 'url') { + const extMime = unsupportedImageMimeFromUrl(part.source.url); + if (extMime !== null) { + content.push({ type: 'text', text: buildUnsupportedImageNotice(extMime, part.source.url) }); + changed = true; + continue; + } + content.push(part); + continue; + } + + // Arbitrary file attachment: materialize the uploaded bytes next to the + // session and replace the part with a path reference — the model opens it + // with the Read tool instead of receiving it as a media part. + if (part.type === 'file') { + const file = await store.get(part.file_id); + const attachedPath = await materializeAttachmentToDir(file, await resolveAttachmentsDir()); + content.push({ + type: 'text', + text: buildAttachedFileNotice(file.meta.name, file.meta.media_type, file.meta.size, attachedPath), + }); + changed = true; + continue; + } + + if ((part.type !== 'image' && part.type !== 'video') || part.source.kind !== 'file') { + content.push(part); + continue; + } + + const file = await store.get(part.source.file_id); + assertMediaFile(file, part.type); + if (part.type === 'image') { + const data = await readFileOrStream(file); + let mediaType = file.meta.media_type; + let bytes: Uint8Array = data; + // Same format gate as the inline path above, and again the bytes are + // authoritative: an upload whose Content-Type lies (AVIF bytes sent + // as image/png) is gated on the sniffed format. Like the inline path, + // keep the bytes as a path-referenced attachment instead of dropping + // them (best effort — the plain notice stands in when persisting + // fails). + mediaType = resolveEffectiveImageMime(mediaType, data); + if (!isModelAcceptedImageMime(mediaType)) { + const persisted = await persistAttachmentBytes( + data, + `${file.meta.id}-${sanitizeAttachmentName(file.meta.name)}`, + await resolveAttachmentsDir(), + ); + content.push({ + type: 'text', + text: persisted === null + ? buildUnsupportedImageNotice(mediaType, file.meta.name) + : buildAttachedFileNotice(file.meta.name, mediaType, file.meta.size, persisted), + }); + changed = true; + continue; + } + // Forward the canonical MIME (image/jpg → image/jpeg, case/whitespace) + // — strict provider whitelists reject the raw alias. + mediaType = normalizeImageMime(mediaType); + const compressed = await compressImageForModel(data, mediaType, { + telemetry: telemetryFor('prompt_file'), + }); + if (compressed.changed) { + const dir = await resolveOriginalsDir(); + const originalPath = await persistOriginalImage(data, mediaType, { dir }); + content.push({ + type: 'text', + text: buildImageCompressionCaption({ + original: { + width: compressed.originalWidth, + height: compressed.originalHeight, + byteLength: compressed.originalByteLength, + mimeType: mediaType, + }, + final: { + width: compressed.width, + height: compressed.height, + byteLength: compressed.finalByteLength, + mimeType: compressed.mimeType, + }, + originalPath, + }), + }); + } + bytes = compressed.data; + mediaType = compressed.mimeType; + content.push({ + type: 'image', + source: { + kind: 'base64', + media_type: mediaType, + data: Buffer.from(bytes).toString('base64'), + }, + }); + changed = true; + continue; + } + + // Uploaded video: materialize a local copy the model can open as a + // fallback, and carry the upload into context as an internal + // `kimi-file://?path=` reference. The engine + // resolves it to a provider form (upload / inline / `