diff --git a/.changeset/recover-from-context-overflow-413.md b/.changeset/recover-from-context-overflow-413.md new file mode 100644 index 000000000..6d331677a --- /dev/null +++ b/.changeset/recover-from-context-overflow-413.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Recover from provider 413 context overflows by compacting before retrying. diff --git a/.changeset/support-managed-anthropic-protocol.md b/.changeset/support-managed-anthropic-protocol.md new file mode 100644 index 000000000..156ed7e4a --- /dev/null +++ b/.changeset/support-managed-anthropic-protocol.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Support the Anthropic-compatible protocol for managed Kimi Code, including video input. diff --git a/apps/kimi-code/src/tui/controllers/session-event-handler.ts b/apps/kimi-code/src/tui/controllers/session-event-handler.ts index 2adc00a4c..a00dcb559 100644 --- a/apps/kimi-code/src/tui/controllers/session-event-handler.ts +++ b/apps/kimi-code/src/tui/controllers/session-event-handler.ts @@ -401,9 +401,10 @@ export class SessionEventHandler { private isAnthropicSessionActive(): boolean { const { state } = this.host; - const providerKey = state.appState.availableModels[state.appState.model]?.provider; - if (providerKey === undefined) return false; - return state.appState.availableProviders[providerKey]?.type === 'anthropic'; + const model = state.appState.availableModels[state.appState.model]; + if (model === undefined) return false; + if (model.protocol === 'anthropic') return true; + return state.appState.availableProviders[model.provider]?.type === 'anthropic'; } private handleStepInterrupted(event: TurnStepInterruptedEvent): void { diff --git a/packages/agent-core/src/agent/compaction/full.ts b/packages/agent-core/src/agent/compaction/full.ts index 31ff5c4e4..21b0fad5f 100644 --- a/packages/agent-core/src/agent/compaction/full.ts +++ b/packages/agent-core/src/agent/compaction/full.ts @@ -8,8 +8,10 @@ import { APIEmptyResponseError, isRetryableGenerateError, type GenerateResult, + type Message, type TokenUsage, APIContextOverflowError, + APIStatusError, createUserMessage, } from '@moonshot-ai/kosong'; @@ -23,6 +25,7 @@ import { renderPrompt } from '../../utils/render-prompt'; import { estimateTokens, estimateTokensForMessages, + estimateTokensForTools, } from '../../utils/tokens'; import { applyCompletionBudget, @@ -39,14 +42,9 @@ import { export const MAX_COMPACTION_RETRY_ATTEMPTS = 5; -/** - * Default hard cap on compaction output tokens when `maxOutputSize` is not - * configured on the model alias. Without this, compaction falls back to the - * full context window size, which exceeds the `max_tokens` ceiling enforced - * by many OpenAI-compatible providers. 128k matches the chat-completions - * ceiling applied by the OpenAI Legacy provider. - */ const DEFAULT_COMPACTION_MAX_COMPLETION_TOKENS = 128 * 1024; +const OVERFLOW_CONTEXT_SAFETY_RATIO = 0.85; +const OVERFLOW_STATUS_RECOVERY_RATIO = 0.5; class CompactionTruncatedError extends Error { constructor() { @@ -62,6 +60,7 @@ export class FullCompaction { promise: Promise; blockedByTurn: boolean; } | null = null; + private readonly observedMaxContextTokensByModel = new Map(); protected readonly strategy: CompactionStrategy; constructor( @@ -71,7 +70,7 @@ export class FullCompaction { this.strategy = strategy ?? new DefaultCompactionStrategy( - () => agent.config.modelCapabilities.max_context_tokens, + () => this.getEffectiveMaxContextTokens(), { ...DEFAULT_COMPACTION_CONFIG, reservedContextSize: @@ -85,6 +84,45 @@ export class FullCompaction { return this.compacting !== null; } + getEffectiveMaxContextTokens(): number { + const configured = this.agent.config.modelCapabilities.max_context_tokens; + const modelAlias = this.agent.config.modelAlias; + const observed = + modelAlias === undefined ? undefined : this.observedMaxContextTokensByModel.get(modelAlias); + if (observed === undefined) return configured; + if (configured <= 0) return observed; + return Math.min(configured, observed); + } + + estimateCurrentRequestTokens(): number { + return this.estimateRequestTokens(this.agent.context.messages); + } + + shouldRecoverFromContextOverflow( + error: unknown, + estimatedRequestTokens = this.estimateCurrentRequestTokens(), + ): boolean { + if (error instanceof APIContextOverflowError) return true; + if (!(error instanceof APIStatusError) || error.statusCode !== 413) return false; + const effectiveMax = this.getEffectiveMaxContextTokens(); + return ( + effectiveMax > 0 && estimatedRequestTokens >= effectiveMax * OVERFLOW_STATUS_RECOVERY_RATIO + ); + } + + observeContextOverflow(estimatedRequestTokens: number): void { + if (!Number.isFinite(estimatedRequestTokens) || estimatedRequestTokens <= 0) return; + const modelAlias = this.agent.config.modelAlias; + if (modelAlias === undefined) return; + const observed = Math.max( + 1, + Math.floor(estimatedRequestTokens * OVERFLOW_CONTEXT_SAFETY_RATIO), + ); + const current = this.getEffectiveMaxContextTokens(); + if (current > 0 && observed >= current) return; + this.observedMaxContextTokensByModel.set(modelAlias, observed); + } + begin(data: Readonly): void { if (this.compacting) return; if (data.source === 'manual') { @@ -145,6 +183,14 @@ export class FullCompaction { return this.agent.context.tokenCountWithPending; } + private estimateRequestTokens(messages: readonly Message[]): number { + return ( + estimateTokens(this.agent.config.systemPrompt) + + estimateTokensForTools(this.agent.tools.loopTools) + + estimateTokensForMessages(messages) + ); + } + resetForTurn(): void { this.compactionCountInTurn = 0; } @@ -300,6 +346,7 @@ export class FullCompaction { ...this.agent.context.project(messagesToCompact), createUserMessage(renderPrompt(compactionInstructionTemplate, { customInstruction: data.instruction ?? '' })), ]; + const estimatedCompactionRequestTokens = this.estimateRequestTokens(messages); try { const response = await this.agent.generate( provider, @@ -316,8 +363,15 @@ export class FullCompaction { summary = extractCompactionSummary(response); break; } catch (error) { + const isContextOverflow = this.shouldRecoverFromContextOverflow( + error, + estimatedCompactionRequestTokens, + ); + if (isContextOverflow) { + this.observeContextOverflow(estimatedCompactionRequestTokens); + } if ( - error instanceof APIContextOverflowError || + isContextOverflow || error instanceof CompactionTruncatedError || error instanceof APIEmptyResponseError // e.g. think-only ) { diff --git a/packages/agent-core/src/agent/tool/index.ts b/packages/agent-core/src/agent/tool/index.ts index f16577253..bab9491ac 100644 --- a/packages/agent-core/src/agent/tool/index.ts +++ b/packages/agent-core/src/agent/tool/index.ts @@ -537,8 +537,58 @@ export class ToolManager { const withAuth = this.agent.modelProvider?.resolveAuth?.(modelAlias, { log: this.agent.log, }); - if (withAuth === undefined) return (input) => uploadVideo(input); - return (input) => withAuth((auth) => uploadVideo(input, { auth })); + const baseProps = this.videoUploadTelemetryProps(modelAlias); + const upload = + withAuth === undefined + ? (input: b.VideoUploadInput) => uploadVideo(input) + : (input: b.VideoUploadInput) => withAuth((auth) => uploadVideo(input, { auth })); + + return async (input) => { + const startedAt = Date.now(); + const base = { + ...baseProps, + mime_type: input.mimeType, + size_bytes: input.data.length, + }; + const track = (props: Record): void => { + try { + this.agent.telemetry.track('video_upload', props); + } catch { + // Telemetry must never affect the upload outcome. + } + }; + try { + const part = await upload(input); + track({ ...base, success: true, latency_ms: Date.now() - startedAt }); + return part; + } catch (error) { + track({ + ...base, + success: false, + latency_ms: Date.now() - startedAt, + error: error instanceof Error ? error.message : String(error), + }); + throw error; + } + }; + } + + private videoUploadTelemetryProps(modelAlias: string): { + type?: string; + protocol?: string; + model: string; + } { + try { + const resolved = this.agent.modelProvider?.resolveProviderConfig(modelAlias); + if (resolved === undefined) return { model: modelAlias }; + return { + model: modelAlias, + type: resolved.type, + protocol: resolved.protocol ?? resolved.type, + }; + } catch { + return { model: modelAlias }; + } } get loopTools(): readonly ExecutableTool[] { diff --git a/packages/agent-core/src/agent/turn/index.ts b/packages/agent-core/src/agent/turn/index.ts index 6f8f30298..d8d2b85f5 100644 --- a/packages/agent-core/src/agent/turn/index.ts +++ b/packages/agent-core/src/agent/turn/index.ts @@ -455,7 +455,7 @@ export class TurnFlow { const telemetryMode = this.telemetryMode(); this.telemetryModeByTurn.set(turnId, telemetryMode); this.currentStepByTurn.set(turnId, 0); - this.agent.telemetry.track('turn_started', { mode: telemetryMode }); + this.agent.telemetry.track('turn_started', { mode: telemetryMode, ...this.requestProtocolProps() }); this.agent.fullCompaction.resetForTurn(); this.agent.usage.beginTurn(); this.agent.emitEvent({ type: 'turn.started', turnId, origin }); @@ -760,10 +760,19 @@ export class TurnFlow { return result.stopReason; } catch (error) { - if ( + const isContextOverflow = error instanceof APIContextOverflowError || - (isKimiError(error) && error.code === ErrorCodes.CONTEXT_OVERFLOW) + (isKimiError(error) && error.code === ErrorCodes.CONTEXT_OVERFLOW); + const estimatedRequestTokens = isContextOverflow + ? this.agent.fullCompaction.estimateCurrentRequestTokens() + : undefined; + if ( + isContextOverflow || + this.agent.fullCompaction.shouldRecoverFromContextOverflow(error, estimatedRequestTokens) ) { + this.agent.fullCompaction.observeContextOverflow( + estimatedRequestTokens ?? this.agent.fullCompaction.estimateCurrentRequestTokens(), + ); await this.agent.fullCompaction.handleOverflowError(signal, error); continue; // Retry with compacted context } @@ -921,6 +930,27 @@ export class TurnFlow { return this.agent.planMode.isActive ? 'plan' : 'agent'; } + /** + * Resolve the current model's provider wire type and any model-level protocol + * override for request telemetry. Never throws — telemetry must not break a + * turn over an unresolvable provider config (the step loop will surface that + * error on its own). + */ + private requestProtocolProps(): { type?: string; protocol?: string } { + const model = this.agent.config.modelAlias; + if (model === undefined) return {}; + try { + const resolved = this.agent.modelProvider?.resolveProviderConfig(model); + if (resolved === undefined) return {}; + return { + type: resolved.type, + protocol: resolved.protocol ?? resolved.type, + }; + } catch { + return {}; + } + } + private shouldTrackApiError(turnId: number): boolean { const failure = this.stepFailureByTurn.get(turnId); return failure?.reason === 'error' && failure.activeStep !== undefined; diff --git a/packages/agent-core/src/agent/turn/kosong-llm.ts b/packages/agent-core/src/agent/turn/kosong-llm.ts index edd501d7e..ef3e2b8bf 100644 --- a/packages/agent-core/src/agent/turn/kosong-llm.ts +++ b/packages/agent-core/src/agent/turn/kosong-llm.ts @@ -19,7 +19,9 @@ import { emptyUsage, generate as kosongGenerate, isRetryableGenerateError, + isUnknownCapability, type ChatProvider, + type ContentPart, type GenerateCallbacks, type Message, type ModelCapability, @@ -119,7 +121,7 @@ export class KosongLLM implements LLM { effectiveProvider, this.systemPrompt, [...params.tools], - params.messages, + downgradeUnsupportedMedia(params.messages, this.capability), callbacks, options, ); @@ -263,3 +265,50 @@ export function buildMessagesWithSystem(systemPrompt: string, history: Message[] ...history, ]; } + +export function downgradeUnsupportedMedia( + messages: readonly Message[], + capability: ModelCapability | undefined, +): Message[] { + if (capability === undefined || isUnknownCapability(capability)) return [...messages]; + const dropImage = !capability.image_in; + const dropVideo = !capability.video_in; + const dropAudio = !capability.audio_in; + if (!dropImage && !dropVideo && !dropAudio) return [...messages]; + + const drop = { dropImage, dropVideo, dropAudio }; + let changed = false; + const out: Message[] = []; + for (const message of messages) { + let nextContent: ContentPart[] | undefined; + for (let i = 0; i < message.content.length; i++) { + const part = message.content[i]!; + const placeholder = mediaPlaceholder(part, drop); + if (placeholder === undefined) { + nextContent?.push(part); + continue; + } + nextContent ??= message.content.slice(0, i); + nextContent.push({ type: 'text', text: placeholder }); + changed = true; + } + out.push(nextContent === undefined ? message : { ...message, content: nextContent }); + } + return changed ? out : [...messages]; +} + +function mediaPlaceholder( + part: ContentPart, + drop: { readonly dropImage: boolean; readonly dropVideo: boolean; readonly dropAudio: boolean }, +): string | undefined { + if (part.type === 'image_url' && drop.dropImage) { + return '[image omitted: current model has no image input]'; + } + if (part.type === 'video_url' && drop.dropVideo) { + return '[video omitted: current model has no video input]'; + } + if (part.type === 'audio_url' && drop.dropAudio) { + return '[audio omitted: current model has no audio input]'; + } + return undefined; +} diff --git a/packages/agent-core/src/config/schema.ts b/packages/agent-core/src/config/schema.ts index 9b3d11cf0..041bf8d6e 100644 --- a/packages/agent-core/src/config/schema.ts +++ b/packages/agent-core/src/config/schema.ts @@ -45,6 +45,7 @@ export const ModelAliasSchema = z.object({ capabilities: z.array(z.string()).optional(), displayName: z.string().optional(), reasoningKey: z.string().optional(), + protocol: z.literal('anthropic').optional(), // Explicitly declare adaptive-thinking support, overriding the kosong // model-name version inference. Needed for custom-named Anthropic endpoints // whose model name does not encode a parseable Claude version. diff --git a/packages/agent-core/src/session/provider-manager.ts b/packages/agent-core/src/session/provider-manager.ts index 34dc82091..44b24153d 100644 --- a/packages/agent-core/src/session/provider-manager.ts +++ b/packages/agent-core/src/session/provider-manager.ts @@ -1,7 +1,7 @@ import type { Logger } from '#/logging/types'; import type { ProviderConfig as KosongProviderConfig, ModelCapability, ProviderRequestAuth } from '@moonshot-ai/kosong'; import { APIStatusError, getModelCapability, UNKNOWN_CAPABILITY } from '@moonshot-ai/kosong'; -import type { KimiConfig, ModelAlias, OAuthRef, ProviderConfig } from '../config'; +import type { KimiConfig, ModelAlias, OAuthRef, ProviderConfig, ProviderType } from '../config'; import { ErrorCodes, isKimiError, KimiError } from '../errors'; export interface BearerTokenProvider { @@ -20,6 +20,10 @@ export interface ResolvedRuntimeProvider { /** Declared 'always_thinking' capability — the model cannot disable thinking. */ readonly alwaysThinking?: boolean; readonly maxOutputSize?: number; + /** Configured provider wire type (`provider.type`), before any model-level protocol override. */ + readonly type: ProviderType; + /** Model-level protocol override (`alias.protocol`); when set, takes precedence over `type` for transport selection. */ + readonly protocol: ModelAlias['protocol']; } interface ProviderManagerOptions { @@ -60,6 +64,8 @@ export class SingleModelProvider implements ModelProvider { modelCapabilities: this.modelCapabilities, providerName: 'single-model-provider', provider: this.providerConfig, + type: this.providerConfig.type, + protocol: undefined, }; } } @@ -107,6 +113,7 @@ export class ProviderManager implements ModelProvider { const provider = toKosongProviderConfig( providerConfig, alias.model, + alias.protocol, this.options.kimiRequestHeaders, alias.maxOutputSize, alias.reasoningKey, @@ -122,6 +129,8 @@ export class ProviderManager implements ModelProvider { (c) => c.trim().toLowerCase() === 'always_thinking', ), maxOutputSize: alias.maxOutputSize, + type: providerConfig.type, + protocol: alias.protocol, }; } @@ -219,23 +228,33 @@ function resolveModelCapabilities( function toKosongProviderConfig( provider: ProviderConfig, model: string, + modelProtocol: ModelAlias['protocol'], kimiRequestHeaders: Record | undefined, maxOutputSize: number | undefined, reasoningKey: string | undefined, promptCacheKey: string | undefined, adaptiveThinking: boolean | undefined, ): KosongProviderConfig { - switch (provider.type) { - case 'anthropic': + const effectiveType = modelProtocol === 'anthropic' ? 'anthropic' : provider.type; + switch (effectiveType) { + case 'anthropic': { + const baseUrl = providerValue(provider.baseUrl, provider.env, 'ANTHROPIC_BASE_URL'); return { type: 'anthropic', model, - baseUrl: providerValue(provider.baseUrl, provider.env, 'ANTHROPIC_BASE_URL'), + baseUrl: + modelProtocol === 'anthropic' && baseUrl !== undefined + ? baseUrl.replace(/\/v1\/?$/, '') + : baseUrl, apiKey: providerApiKey(provider), ...(maxOutputSize !== undefined ? { defaultMaxTokens: maxOutputSize } : {}), ...(adaptiveThinking !== undefined ? { adaptiveThinking } : {}), + // Session affinity: Anthropic's analog of OpenAI `prompt_cache_key` is + // `metadata.user_id` on the Messages API (cache-affinity / end-user id). + ...(promptCacheKey !== undefined ? { metadata: { user_id: promptCacheKey } } : {}), ...defaultHeadersField(provider.customHeaders), }; + } case 'openai': return { type: 'openai', @@ -280,7 +299,7 @@ function toKosongProviderConfig( }; } default: { - const exhaustive: never = provider.type; + const exhaustive: never = effectiveType; throw new KimiError( ErrorCodes.MODEL_CONFIG_INVALID, `Unsupported provider type: ${String(exhaustive)}`, diff --git a/packages/agent-core/test/agent/compaction/full.test.ts b/packages/agent-core/test/agent/compaction/full.test.ts index a1a630250..6eceacc31 100644 --- a/packages/agent-core/test/agent/compaction/full.test.ts +++ b/packages/agent-core/test/agent/compaction/full.test.ts @@ -1589,6 +1589,113 @@ describe('FullCompaction', () => { await ctx.expectResumeMatches(); }); + it('uses observed max from overflow to size compaction input', async () => { + const ctx = testAgent(); + ctx.configure({ + provider: CATALOGUED_PROVIDER, + modelCapabilities: { + ...CATALOGUED_MODEL_CAPABILITIES, + max_context_tokens: 1_000_000, + }, + }); + for (let i = 0; i < 20; i++) { + ctx.appendExchange( + i + 1, + `old user ${String(i)}`, + `old assistant ${String(i)} ${'x'.repeat(40_000)}`, + 20_000, + ); + } + ctx.agent.fullCompaction.observeContextOverflow(200_000); + const compacted = ctx.once('context.apply_compaction'); + const completed = ctx.once('compaction.completed'); + + ctx.mockNextResponse({ type: 'text', text: 'Observed max summary.' }); + await ctx.rpc.beginCompaction({}); + await compacted; + await completed; + + expect(ctx.agent.fullCompaction.getEffectiveMaxContextTokens()).toBe(170_000); + const compactionTokens = estimateTokensForMessages(ctx.llmCalls[0]?.history ?? []); + expect(compactionTokens).toBeLessThan(200_000); + expect(ctx.compactHistory()[0]).toEqual({ role: 'assistant', text: 'Observed max summary.' }); + await ctx.expectResumeMatches(); + }); + + it('recovers from plain 413 when estimated request is over effective max', async () => { + let callCount = 0; + const generate: GenerateFn = async (_provider, _system, _tools, _history, callbacks) => { + callCount += 1; + if (callCount === 1) { + throw new APIStatusError(413, 'Request Entity Too Large', 'req-plain-413'); + } + if (callCount === 2) { + return textResult('Plain 413 compacted summary.'); + } + await callbacks?.onMessagePart?.({ + type: 'text', + text: 'Recovered after plain 413 compaction.', + }); + return textResult('Recovered after plain 413 compaction.'); + }; + const ctx = testAgent({ generate }); + ctx.configure({ + provider: CATALOGUED_PROVIDER, + modelCapabilities: { + ...CATALOGUED_MODEL_CAPABILITIES, + max_context_tokens: 200_000, + }, + }); + ctx.appendExchange(1, 'old user one', `old assistant one ${'x'.repeat(600_000)}`, 150_000); + ctx.newEvents(); + + await ctx.rpc.prompt({ input: [{ type: 'text', text: 'Retry after plain 413' }] }); + const events = await ctx.untilTurnEnd(); + + expect(callCount).toBe(3); + expect(ctx.agent.fullCompaction.getEffectiveMaxContextTokens()).toBeLessThan(200_000); + expect(events).toContainEqual( + expect.objectContaining({ + event: 'compaction.started', + args: { trigger: 'auto' }, + }), + ); + expect(events).toContainEqual( + expect.objectContaining({ + event: 'turn.ended', + args: { turnId: 0, reason: 'completed' }, + }), + ); + await ctx.expectResumeMatches(); + }); + + it('does not compact plain 413 when estimated request is small', async () => { + const generate: GenerateFn = async () => { + throw new APIStatusError(413, 'Request Entity Too Large', 'req-small-413'); + }; + const ctx = testAgent({ generate }); + ctx.configure({ + provider: CATALOGUED_PROVIDER, + modelCapabilities: { + ...CATALOGUED_MODEL_CAPABILITIES, + max_context_tokens: 200_000, + }, + }); + ctx.appendExchange(1, 'old user one', 'old assistant one', 20); + ctx.newEvents(); + + await ctx.rpc.prompt({ input: [{ type: 'text', text: 'small prompt' }] }); + const events = await ctx.untilTurnEnd(); + + expect(eventIndex(events, 'compaction.started')).toBe(-1); + expect(events).toContainEqual( + expect.objectContaining({ + event: 'turn.ended', + args: expect.objectContaining({ turnId: 0, reason: 'failed' }), + }), + ); + }); + it('preserves thinking effort when compacting after provider context overflow', async () => { let callCount = 0; const records: TelemetryRecord[] = []; diff --git a/packages/agent-core/test/agent/kosong-llm.test.ts b/packages/agent-core/test/agent/kosong-llm.test.ts index 4337c9623..4855f1236 100644 --- a/packages/agent-core/test/agent/kosong-llm.test.ts +++ b/packages/agent-core/test/agent/kosong-llm.test.ts @@ -1,13 +1,19 @@ import { emptyUsage, + UNKNOWN_CAPABILITY, type ChatProvider, + type Message, type ModelCapability, type StreamedMessagePart, type ToolCall, } from '@moonshot-ai/kosong'; import { describe, expect, it } from 'vitest'; -import { KosongLLM, type GenerateFn } from '../../src/agent/turn/kosong-llm'; +import { + KosongLLM, + downgradeUnsupportedMedia, + type GenerateFn, +} from '../../src/agent/turn/kosong-llm'; import type { ToolCallDelta } from '../../src/loop'; const provider: ChatProvider = { @@ -227,3 +233,102 @@ function makeCapability(maxContextTokens: number): ModelCapability { max_context_tokens: maxContextTokens, }; } + +function mediaMessage(content: Message['content']): Message { + return { role: 'tool', content, toolCalls: [], toolCallId: 'call_media' }; +} + +describe('downgradeUnsupportedMedia', () => { + const imagePart = { type: 'image_url', imageUrl: { url: 'data:image/png;base64,AAA' } } as const; + const videoPart = { type: 'video_url', videoUrl: { url: 'ms://file-1', id: 'file-1' } } as const; + const audioPart = { type: 'audio_url', audioUrl: { url: 'data:audio/mpeg;base64,AAA' } } as const; + + it('replaces video parts when the model lacks video_in and keeps the rest', () => { + const capability: ModelCapability = { ...makeCapability(1000), image_in: true, audio_in: true }; + const input = [mediaMessage([{ type: 'text', text: '