From d96087dbd3a01dee21cd49ebb7d75baf14c8dbe6 Mon Sep 17 00:00:00 2001 From: tanzhenxin Date: Thu, 9 Jul 2026 11:26:53 +0800 Subject: [PATCH] fix(core): apply window clamp to samplingParams users who omit max_tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously a samplingParams config without a max_tokens key sent no max_tokens on the wire (OpenAI path), so those users bypassed the prompt + max_tokens <= window clamp — inconsistent with the Anthropic path, which always injects the clamped value. Mirror the Anthropic fallback (reconcile ?? config ?? request) so the clamped maxOutputTokens is injected when samplingParams omits max_tokens. Guard the injection: when samplingParams targets a provider-specific output-budget key (max_completion_tokens for GPT-5/o-series, max_new_tokens), leave it verbatim — adding max_tokens alongside double-specifies the budget and those endpoints reject the pair. --- .../openaiContentGenerator/pipeline.test.ts | 37 +++++++++++++++ .../core/openaiContentGenerator/pipeline.ts | 47 ++++++++++++++----- 2 files changed, 72 insertions(+), 12 deletions(-) diff --git a/packages/core/src/core/openaiContentGenerator/pipeline.test.ts b/packages/core/src/core/openaiContentGenerator/pipeline.test.ts index 790c6b40f5..960b6f0d09 100644 --- a/packages/core/src/core/openaiContentGenerator/pipeline.test.ts +++ b/packages/core/src/core/openaiContentGenerator/pipeline.test.ts @@ -2697,6 +2697,43 @@ describe('ContentGenerationPipeline', () => { expect(call).not.toHaveProperty('max_tokens'); }); + it('should inject the window-clamped max_tokens when samplingParams omits it and carries no provider output-budget key', async () => { + // Arrange: samplingParams is set but specifies no output budget (no + // max_tokens, no provider-specific key). The window clamp + // (request.config.maxOutputTokens) must still reach the wire as + // max_tokens so these users get the same `prompt + max_tokens ≤ window` + // protection as everyone else, matching the Anthropic path. + mockContentGeneratorConfig.samplingParams = { + temperature: 0.7, + } as ContentGeneratorConfig['samplingParams']; + pipeline = new ContentGenerationPipeline(mockConfig); + + const request: GenerateContentParameters = { + model: 'test-model', + contents: [{ parts: [{ text: 'Hello' }], role: 'user' }], + config: { maxOutputTokens: 777 }, + }; + (mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue([]); + (mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue( + new GenerateContentResponse(), + ); + (mockClient.chat.completions.create as Mock).mockResolvedValue({ + id: 'test', + choices: [{ message: { content: 'r' } }], + }); + + // Act + await pipeline.execute(request, 'prompt-id'); + + // Assert: clamped value injected as max_tokens; other keys pass through. + const call = (mockClient.chat.completions.create as Mock).mock + .calls[0][0]; + expect(call).toMatchObject({ + temperature: 0.7, + max_tokens: 777, + }); + }); + it('should preserve historical default behavior when samplingParams is absent', async () => { // Arrange: no samplingParams — request.config.maxOutputTokens must still // fall through to max_tokens on the wire (original behavior unchanged). diff --git a/packages/core/src/core/openaiContentGenerator/pipeline.ts b/packages/core/src/core/openaiContentGenerator/pipeline.ts index 43399ed406..dd6118ecc1 100644 --- a/packages/core/src/core/openaiContentGenerator/pipeline.ts +++ b/packages/core/src/core/openaiContentGenerator/pipeline.ts @@ -66,6 +66,23 @@ export class StreamInactivityTimeoutError extends Error { } } +/** + * Provider-specific output-budget keys that stand in for `max_tokens` on the + * wire (e.g. GPT-5 / o-series use `max_completion_tokens`). When a user's + * samplingParams already carries one of these, the window clamp must not also + * inject `max_tokens`: sending the pair double-specifies the output budget and + * some endpoints reject it. + */ +const PROVIDER_OUTPUT_BUDGET_KEYS = ['max_completion_tokens', 'max_new_tokens']; + +function hasProviderOutputBudgetKey(samplingParams: { + [key: string]: unknown; +}): boolean { + return PROVIDER_OUTPUT_BUDGET_KEYS.some( + (key) => samplingParams[key] !== undefined, + ); +} + /** * Resolve the effective streaming inactivity timeout (ms). Precedence: * explicit `ContentGeneratorConfig.streamIdleTimeoutMs` (programmatic, wins — @@ -684,19 +701,25 @@ export class ContentGenerationPipeline { // When samplingParams is set, its keys pass through to the wire verbatim. // This lets users target provider-specific parameter names // (e.g. `max_completion_tokens` for GPT-5 / o-series) without a client release. - // One exception: a user-set max_tokens is a ceiling, not an exemption from - // the window clamp — when the request carries a (clamped) maxOutputTokens - // and it is lower, it wins, so `prompt + max_tokens ≤ window` holds even - // for samplingParams users. When samplingParams omits max_tokens entirely, - // nothing is injected (the user opted into verbatim wire control). - // When absent, the historical default behavior applies. + // max_tokens is a ceiling, not an exemption from the window clamp: when both + // a config max_tokens and the (clamped) request maxOutputTokens are present + // the smaller wins, and when samplingParams omits max_tokens the clamped + // request value is injected — so `prompt + max_tokens ≤ window` holds for + // samplingParams users too, matching the Anthropic path. The injection is + // suppressed when samplingParams already carries a provider-specific + // output-budget key (e.g. `max_completion_tokens`): adding `max_tokens` + // alongside it would double the budget and some endpoints (o-series) reject + // the pair, so those configs stay verbatim. if (configSamplingParams !== undefined) { - const reconciled = reconcileMaxTokens( - configSamplingParams.max_tokens, - request.config?.maxOutputTokens, - ); - if (reconciled !== undefined) { - return { ...configSamplingParams, max_tokens: reconciled }; + const requestMaxTokens = request.config?.maxOutputTokens; + const maxTokens = + reconcileMaxTokens(configSamplingParams.max_tokens, requestMaxTokens) ?? + configSamplingParams.max_tokens ?? + (hasProviderOutputBudgetKey(configSamplingParams) + ? undefined + : requestMaxTokens); + if (maxTokens !== undefined) { + return { ...configSamplingParams, max_tokens: maxTokens }; } return { ...configSamplingParams }; }