diff --git a/packages/core/src/core/openaiContentGenerator/pipeline.test.ts b/packages/core/src/core/openaiContentGenerator/pipeline.test.ts index 59aa5ac61d..76c76a8edc 100644 --- a/packages/core/src/core/openaiContentGenerator/pipeline.test.ts +++ b/packages/core/src/core/openaiContentGenerator/pipeline.test.ts @@ -2973,10 +2973,11 @@ describe('ContentGenerationPipeline', () => { ); }); - it('should pass arbitrary samplingParams keys through verbatim (e.g. max_completion_tokens for GPT-5)', async () => { + it('should pass arbitrary samplingParams keys through verbatim when the window has room (e.g. max_completion_tokens for GPT-5)', async () => { // Arrange: user sets a GPT-5 / o-series shape in samplingParams. // None of these are typed fields; all must appear on the wire because - // samplingParams is the source of truth. + // samplingParams is the source of truth. maxOutputTokens (32000) leaves + // room above max_completion_tokens (4096), so the value is not clamped. mockContentGeneratorConfig.samplingParams = { max_completion_tokens: 4096, reasoning_effort: 'medium', @@ -2987,7 +2988,7 @@ describe('ContentGenerationPipeline', () => { const request: GenerateContentParameters = { model: 'test-model', contents: [{ parts: [{ text: 'Hello' }], role: 'user' }], - config: { maxOutputTokens: 999 }, + config: { maxOutputTokens: 32000 }, }; (mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue([]); (mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue( @@ -3001,8 +3002,9 @@ describe('ContentGenerationPipeline', () => { // Act await pipeline.execute(request, 'prompt-id'); - // Assert: the exact samplingParams keys reach the wire; max_tokens is NOT - // synthesized from request.config.maxOutputTokens. + // Assert: the exact samplingParams keys reach the wire unchanged; a + // separate max_tokens is NOT synthesized (that would double-specify the + // budget and o-series rejects the pair). const call = (mockClient.chat.completions.create as Mock).mock .calls[0][0]; expect(call).toMatchObject({ @@ -3013,6 +3015,45 @@ describe('ContentGenerationPipeline', () => { expect(call).not.toHaveProperty('max_tokens'); }); + it('should clamp a provider output-budget key to the window without injecting max_tokens', async () => { + // Arrange: max_completion_tokens (200000) exceeds the window's remaining + // room (maxOutputTokens 50000). The value must be clamped in place so + // `prompt + output ≤ window` holds — but NO max_tokens is injected + // (o-series rejects both keys together). + mockContentGeneratorConfig.samplingParams = { + max_completion_tokens: 200000, + reasoning_effort: 'high', + } as ContentGeneratorConfig['samplingParams']; + pipeline = new ContentGenerationPipeline(mockConfig); + + const request: GenerateContentParameters = { + model: 'test-model', + contents: [{ parts: [{ text: 'Hello' }], role: 'user' }], + config: { maxOutputTokens: 50000 }, + }; + (mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue([]); + (mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue( + new GenerateContentResponse(), + ); + (mockClient.chat.completions.create as Mock).mockResolvedValue({ + id: 'test', + choices: [{ message: { content: 'r' } }], + }); + + // Act + await pipeline.execute(request, 'prompt-id'); + + // Assert: provider key clamped to the window; other keys verbatim; no + // max_tokens added. + const call = (mockClient.chat.completions.create as Mock).mock + .calls[0][0]; + expect(call).toMatchObject({ + max_completion_tokens: 50000, + reasoning_effort: 'high', + }); + expect(call).not.toHaveProperty('max_tokens'); + }); + it('should inject the window-clamped max_tokens when samplingParams omits it and carries no provider output-budget key', async () => { // Arrange: samplingParams is set but specifies no output budget (no // max_tokens, no provider-specific key). The window clamp diff --git a/packages/core/src/core/openaiContentGenerator/pipeline.ts b/packages/core/src/core/openaiContentGenerator/pipeline.ts index ff68c60ca9..a9f9420f3f 100644 --- a/packages/core/src/core/openaiContentGenerator/pipeline.ts +++ b/packages/core/src/core/openaiContentGenerator/pipeline.ts @@ -131,6 +131,30 @@ function hasProviderOutputBudgetKey(samplingParams: { ); } +/** + * Clamp any provider-specific output-budget key (e.g. `max_completion_tokens`) + * to the window's remaining room, mutating and returning the passed object. + * An output budget is subject to `prompt + output ≤ window` regardless of the + * key it travels under, so we shrink the key's value to `requestMaxTokens` when + * it exceeds it — but we clamp the value in place rather than injecting a + * separate `max_tokens`, which would double-specify the budget and be rejected + * by endpoints like the o-series. When there is room (or no clamp value is + * available), the user's value passes through unchanged. + */ +function clampProviderOutputBudgetKeys( + samplingParams: { [key: string]: unknown }, + requestMaxTokens: number | undefined, +): { [key: string]: unknown } { + if (typeof requestMaxTokens !== 'number') return samplingParams; + for (const key of PROVIDER_OUTPUT_BUDGET_KEYS) { + const value = samplingParams[key]; + if (typeof value === 'number' && value > requestMaxTokens) { + samplingParams[key] = requestMaxTokens; + } + } + return samplingParams; +} + /** * Resolve the effective streaming inactivity timeout (ms). Precedence: * explicit `ContentGeneratorConfig.streamIdleTimeoutMs` (programmatic, wins — @@ -812,15 +836,18 @@ export class ContentGenerationPipeline { // When samplingParams is set, its keys pass through to the wire verbatim. // This lets users target provider-specific parameter names // (e.g. `max_completion_tokens` for GPT-5 / o-series) without a client release. - // max_tokens is a ceiling, not an exemption from the window clamp: when both - // a config max_tokens and the (clamped) request maxOutputTokens are present - // the smaller wins, and when samplingParams omits max_tokens the clamped - // request value is injected — so `prompt + max_tokens ≤ window` holds for - // samplingParams users too, matching the Anthropic path. The injection is - // suppressed when samplingParams already carries a provider-specific - // output-budget key (e.g. `max_completion_tokens`): adding `max_tokens` - // alongside it would double the budget and some endpoints (o-series) reject - // the pair, so those configs stay verbatim. + // No output budget escapes the window clamp, whatever key it travels under: + // - max_tokens is a ceiling, not an exemption — when both a config + // max_tokens and the (clamped) request maxOutputTokens are present the + // smaller wins; when samplingParams omits max_tokens the clamped request + // value is injected. + // - A provider-specific output-budget key (max_completion_tokens, + // max_new_tokens) is clamped in place to the window instead — we do NOT + // also inject max_tokens, since sending the pair double-specifies the + // budget and some endpoints (o-series) reject it. Its value only shrinks + // when the window is tight; when there is room it passes through as-is. + // So `prompt + max_tokens ≤ window` holds for samplingParams users too, + // matching the Anthropic path. if (configSamplingParams !== undefined) { const requestMaxTokens = request.config?.maxOutputTokens; const maxTokens = @@ -832,7 +859,13 @@ export class ContentGenerationPipeline { if (maxTokens !== undefined) { return { ...configSamplingParams, max_tokens: maxTokens }; } - return { ...configSamplingParams }; + // maxTokens is undefined only when a provider-specific output-budget key + // is in use (or there is nothing to clamp against). Clamp that key's + // value to the window rather than injecting a separate max_tokens. + return clampProviderOutputBudgetKeys( + { ...configSamplingParams }, + requestMaxTokens, + ); } const params: Record = {