mirror of
https://github.com/MoonshotAI/kimi-code.git
synced 2026-08-21 06:35:50 +00:00
fix(agent-core-v2): stop the legacy video resolver from shadowing the media resolver (#3053)
#2593 replaced AgentVideoResolverService with the image+video AgentMediaResolverService and reduced videoResolverService.ts to a pure deprecated alias with no DI registration. #2909's squash merge restored the pre-#2593 file wholesale, bringing back the legacy class and its registerScopedService call. Both classes then registered the same token ('agentVideoResolverService') at the Agent scope and the legacy video-only resolver won on the production import order, so image kimi-file:// references reached the provider unresolved. Gateways reject the unknown scheme with a 400 ("unsupported image url"), the media-strip fallback then hid the image from the model, and pasted images only worked on undo-resend via the inline base64 fallback. Delete the legacy alias files and their index exports, drop the stale alias assertion, and pin the behavior with a klient e2e regression: a kimi-file image prompt part must reach the provider as a data: URL, never verbatim.
This commit is contained in:
parent
c9c34ae5a8
commit
95cede82b4
6 changed files with 46 additions and 225 deletions
5
.changeset/fix-kimi-file-image-resolver.md
Normal file
5
.changeset/fix-kimi-file-image-resolver.md
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
"@moonshot-ai/kimi-code": patch
|
||||
---
|
||||
|
||||
Fix pasted images failing to reach the model on first send.
|
||||
|
|
@ -1 +0,0 @@
|
|||
export { IAgentMediaResolverService as IAgentVideoResolverService } from './mediaResolver';
|
||||
|
|
@ -1,213 +0,0 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { LifecycleScope } from '#/app/scopes';
|
||||
import { ScopeActivation, registerScopedService } from '#/_base/di/scope';
|
||||
import { IAgentStateService } from '#/agent/state/agentState';
|
||||
import { IFileService } from '#/app/file/fileService';
|
||||
import { ITelemetryService } from '#/app/telemetry/telemetry';
|
||||
import type { ContentPart, Message } from '#/kosong/contract/message';
|
||||
import type { ModelRequester } from '#/kosong/model/modelRequester';
|
||||
import { IBlobStore } from '#/persistence/interface/blobStore';
|
||||
|
||||
import { mediaResolvedKey } from './mediaResolverService';
|
||||
import { detectFileType, MEDIA_SNIFF_BYTES } from './file-type';
|
||||
import { type KimiFileRef, isKimiFileUrl, parseKimiFileUrl } from './kimiFileUrl';
|
||||
import { createVideoUploader } from './registerMediaTools';
|
||||
import {
|
||||
inlineVideoPart,
|
||||
inlineVideoSupportedForProtocol,
|
||||
isVideoUploadAuthError,
|
||||
isVideoUploadUnsupportedError,
|
||||
} from './videoUpload';
|
||||
import { IAgentVideoResolverService } from './videoResolver';
|
||||
|
||||
const CACHE_SCOPE = 'video-upload-cache';
|
||||
const PROVIDER_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
||||
const VIDEO_UNAVAILABLE_TEXT =
|
||||
'[video omitted: the uploaded file is no longer available]';
|
||||
|
||||
const textEncoder = new TextEncoder();
|
||||
const textDecoder = new TextDecoder();
|
||||
|
||||
export class AgentVideoResolverService implements IAgentVideoResolverService {
|
||||
declare readonly _serviceBrand: undefined;
|
||||
|
||||
constructor(
|
||||
@IFileService private readonly files: IFileService,
|
||||
@IBlobStore private readonly blobs: IBlobStore,
|
||||
@ITelemetryService private readonly telemetry: ITelemetryService,
|
||||
@IAgentStateService private readonly states: IAgentStateService,
|
||||
) {}
|
||||
|
||||
private get resolved(): Map<string, ContentPart> {
|
||||
return this.states.get(mediaResolvedKey);
|
||||
}
|
||||
|
||||
async resolve(
|
||||
messages: readonly Message[],
|
||||
requester: ModelRequester,
|
||||
signal?: AbortSignal,
|
||||
): Promise<readonly Message[]> {
|
||||
if (!messages.some(hasKimiFileVideoPart)) return messages;
|
||||
|
||||
let changed = false;
|
||||
const out: Message[] = [];
|
||||
for (const message of messages) {
|
||||
if (!hasKimiFileVideoPart(message)) {
|
||||
out.push(message);
|
||||
continue;
|
||||
}
|
||||
const content: ContentPart[] = [];
|
||||
for (const part of message.content) {
|
||||
const ref =
|
||||
part.type === 'video_url' ? parseKimiFileUrl(part.videoUrl.url) : undefined;
|
||||
content.push(ref === undefined ? part : await this.resolvePart(ref, requester, signal));
|
||||
}
|
||||
out.push({ ...message, content });
|
||||
changed = true;
|
||||
}
|
||||
return changed ? out : messages;
|
||||
}
|
||||
|
||||
private async resolvePart(
|
||||
ref: KimiFileRef,
|
||||
requester: ModelRequester,
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<ContentPart> {
|
||||
const model = requester.model;
|
||||
const providerKey = model.providerType ?? model.protocol;
|
||||
const cacheKey = `${ref.fileId}\0${providerKey}`;
|
||||
|
||||
const memoed = this.resolved.get(cacheKey);
|
||||
if (memoed !== undefined) return memoed;
|
||||
|
||||
const { part, memoize } = await this.resolveUncached(ref, requester, cacheKey, signal);
|
||||
if (memoize) this.resolved.set(cacheKey, part);
|
||||
return part;
|
||||
}
|
||||
|
||||
private async resolveUncached(
|
||||
ref: KimiFileRef,
|
||||
requester: ModelRequester,
|
||||
cacheKey: string,
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<{ part: ContentPart; memoize: boolean }> {
|
||||
const cachedLlmFileId = await this.readCachedUpload(cacheKey);
|
||||
if (cachedLlmFileId !== undefined) {
|
||||
return {
|
||||
part: { type: 'video_url', videoUrl: { url: `ms://${cachedLlmFileId}`, id: cachedLlmFileId } },
|
||||
memoize: true,
|
||||
};
|
||||
}
|
||||
|
||||
let bytes: Buffer;
|
||||
let filename: string;
|
||||
try {
|
||||
const file = await this.files.get(ref.fileId);
|
||||
bytes = await readStream(file.stream());
|
||||
filename = file.meta.name;
|
||||
} catch {
|
||||
return { part: tag(ref), memoize: true };
|
||||
}
|
||||
|
||||
const fileType = detectFileType(filename, bytes.subarray(0, MEDIA_SNIFF_BYTES), 'media');
|
||||
if (fileType.kind !== 'video') return { part: tag(ref), memoize: true };
|
||||
const mimeType = fileType.mimeType;
|
||||
|
||||
const model = requester.model;
|
||||
if (!model.capabilities.video_in) return { part: tag(ref), memoize: true };
|
||||
const inlineSupported = inlineVideoSupportedForProtocol(model.protocol);
|
||||
|
||||
const uploader = createVideoUploader(requester, {
|
||||
client: this.telemetry,
|
||||
props: {
|
||||
model: model.name,
|
||||
provider_type: model.providerType ?? model.protocol,
|
||||
protocol: model.protocol,
|
||||
},
|
||||
});
|
||||
if (uploader === undefined) {
|
||||
return {
|
||||
part: inlineSupported ? inlineVideoPart(bytes, mimeType) : tag(ref),
|
||||
memoize: true,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const uploaded = await uploader({ data: bytes, mimeType, filename }, { signal });
|
||||
const llmFileId = uploaded.videoUrl.id ?? msFileIdFromUrl(uploaded.videoUrl.url);
|
||||
if (llmFileId !== undefined) await this.writeCachedUpload(cacheKey, llmFileId);
|
||||
return { part: uploaded, memoize: true };
|
||||
} catch (error) {
|
||||
if (signal?.aborted) throw error;
|
||||
if (isVideoUploadAuthError(error)) throw error;
|
||||
if (isVideoUploadUnsupportedError(error)) {
|
||||
return {
|
||||
part: inlineSupported ? inlineVideoPart(bytes, mimeType) : tag(ref),
|
||||
memoize: true,
|
||||
};
|
||||
}
|
||||
return { part: tag(ref), memoize: false };
|
||||
}
|
||||
}
|
||||
|
||||
private async readCachedUpload(cacheKey: string): Promise<string | undefined> {
|
||||
const data = await this.blobs.get(CACHE_SCOPE, blobKey(cacheKey)).catch(() => undefined);
|
||||
if (data === undefined) return undefined;
|
||||
const llmFileId = textDecoder.decode(data);
|
||||
return PROVIDER_ID_RE.test(llmFileId) ? llmFileId : undefined;
|
||||
}
|
||||
|
||||
private async writeCachedUpload(cacheKey: string, llmFileId: string): Promise<void> {
|
||||
if (!PROVIDER_ID_RE.test(llmFileId)) return;
|
||||
await this.blobs.put(CACHE_SCOPE, blobKey(cacheKey), textEncoder.encode(llmFileId)).catch(
|
||||
() => undefined,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function hasKimiFileVideoPart(message: Message): boolean {
|
||||
return message.content.some(
|
||||
(part) => part.type === 'video_url' && isKimiFileUrl(part.videoUrl.url),
|
||||
);
|
||||
}
|
||||
|
||||
function tag(ref: KimiFileRef): ContentPart {
|
||||
if (ref.fileId.length === 0) {
|
||||
return { type: 'text', text: VIDEO_UNAVAILABLE_TEXT };
|
||||
}
|
||||
return { type: 'text', text: `<video path="${escapeAttribute(ref.fileId)}"></video>` };
|
||||
}
|
||||
|
||||
function msFileIdFromUrl(url: string): string | undefined {
|
||||
if (!url.startsWith('ms://')) return undefined;
|
||||
const id = url.slice('ms://'.length);
|
||||
return id.length > 0 ? id : undefined;
|
||||
}
|
||||
|
||||
function blobKey(cacheKey: string): string {
|
||||
return createHash('sha256').update(cacheKey).digest('hex');
|
||||
}
|
||||
|
||||
async function readStream(stream: NodeJS.ReadableStream): Promise<Buffer> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.from(chunk as string | Uint8Array));
|
||||
}
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
function escapeAttribute(value: string): string {
|
||||
return value
|
||||
.replaceAll('&', '&')
|
||||
.replaceAll('"', '"')
|
||||
.replaceAll('<', '<')
|
||||
.replaceAll('>', '>');
|
||||
}
|
||||
|
||||
registerScopedService(
|
||||
LifecycleScope.Agent,
|
||||
IAgentVideoResolverService,
|
||||
AgentVideoResolverService,
|
||||
ScopeActivation.OnScopeCreated,
|
||||
'media',
|
||||
);
|
||||
|
|
@ -653,8 +653,6 @@ export * from '#/agent/media/kimiFileUrl';
|
|||
export * from '#/agent/media/videoUpload';
|
||||
export * from '#/agent/media/mediaResolver';
|
||||
export * from '#/agent/media/mediaResolverService';
|
||||
export * from '#/agent/media/videoResolver';
|
||||
export * from '#/agent/media/videoResolverService';
|
||||
import '#/agent/media/configSection';
|
||||
export * from '#/agent/media/imageConfigBridge';
|
||||
import '#/agent/permissionMode/configSection';
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ import { buildKimiFileUrl } from '#/agent/media/kimiFileUrl';
|
|||
import { IAgentMediaResolverService } from '#/agent/media/mediaResolver';
|
||||
import { AgentMediaResolverService } from '#/agent/media/mediaResolverService';
|
||||
import { ISessionMediaStore } from '#/agent/media/sessionMediaStore';
|
||||
import { IAgentVideoResolverService } from '#/agent/media/videoResolver';
|
||||
import { IAgentStateService } from '#/agent/state/agentState';
|
||||
import { AgentStateService } from '#/agent/state/agentStateService';
|
||||
import { type GetResult, IFileService } from '#/app/file/fileService';
|
||||
|
|
@ -795,12 +794,4 @@ describe('AgentMediaResolverService scoped registration', () => {
|
|||
|
||||
expect(firstPart(out)).toEqual({ type: 'image_url', imageUrl: { url: PNG_DATA_URL } });
|
||||
});
|
||||
|
||||
it('resolves the legacy video-resolver alias to the same instance', () => {
|
||||
const agent = agentScope(new Map());
|
||||
|
||||
expect(agent.accessor.get(IAgentVideoResolverService)).toBe(
|
||||
agent.accessor.get(IAgentMediaResolverService),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -683,6 +683,47 @@ describe('image blocks with invalid data', () => {
|
|||
}, 30_000);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Daemon file references (kimi-file://): engine-side resolution before the
|
||||
// provider wire.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('daemon file references (kimi-file://)', () => {
|
||||
it('a kimi-file image reference reaches the provider as a data URL, never verbatim', async () => {
|
||||
// Regression for the duplicated resolver-token shadowing: the legacy
|
||||
// video-only resolver won the shared DI token on the production import
|
||||
// order, so image kimi-file refs leaked to the provider unchanged and
|
||||
// gateways rejected the unknown scheme with a 400 ("unsupported image
|
||||
// url"), which the media-strip fallback then mistook for a bad image.
|
||||
const cases = [
|
||||
{ label: 'kimifile-image-openai', model: M_OPENAI_VISION, reply: OK_OPENAI },
|
||||
{ label: 'kimifile-image-kimi', model: M_KIMI, reply: OK_OPENAI },
|
||||
] as const;
|
||||
for (const { label, model, reply } of cases) {
|
||||
const meta = await klient.global.files.save({
|
||||
data: new Uint8Array(Buffer.from(PNG_1X1_BASE64, 'base64')),
|
||||
filename: 'pasted-image.png',
|
||||
mimeType: 'image/png',
|
||||
expiresInSec: 3600,
|
||||
});
|
||||
const ctx = await newCase(model, label);
|
||||
resetMock(queueScript(reply));
|
||||
await promptAndWait(ctx, [
|
||||
{ type: 'image_url', imageUrl: { url: `kimi-file://${meta.id}` } },
|
||||
{ type: 'text', text: 'what is this?' },
|
||||
]);
|
||||
expect(requests, label).toHaveLength(1);
|
||||
expect(JSON.stringify(requests[0]?.json), label).not.toContain('kimi-file://');
|
||||
const content = openAiMessages(0).at(-1)?.['content'] as unknown[];
|
||||
const imagePart = content.find(
|
||||
(part) => (part as { type?: string }).type === 'image_url',
|
||||
) as { image_url?: { url?: string } } | undefined;
|
||||
expect(imagePart?.image_url?.url ?? '', label).toMatch(/^data:image\/png;base64,/);
|
||||
expect(ctx.payloads('prompt.completed')[0]?.['reason'], label).toBe('completed');
|
||||
}
|
||||
}, 60_000);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Video blocks: URL pass-through, upload capability, illegal video data.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue